storazzo 0.4.2 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +7 -4
  3. data/Makefile +20 -1
  4. data/README.md +23 -0
  5. data/Rakefile +35 -8
  6. data/VERSION +1 -1
  7. data/bin/hello-storazzo +5 -0
  8. data/bin/ricdisk-magic +50 -57
  9. data/bin/stats-with-md5 +268 -297
  10. data/lib/storazzo/colors.rb +43 -45
  11. data/lib/storazzo/common.rb +80 -17
  12. data/lib/storazzo/debug.rb +5 -8
  13. data/lib/storazzo/hashify.rb +44 -43
  14. data/lib/storazzo/main.rb +44 -40
  15. data/lib/storazzo/media/abstract_ric_disk.rb +163 -104
  16. data/lib/storazzo/media/gcs_bucket.rb +51 -15
  17. data/lib/storazzo/media/local_folder.rb +43 -46
  18. data/lib/storazzo/media/mount_point.rb +12 -2
  19. data/lib/storazzo/ric_disk.rb +223 -251
  20. data/lib/storazzo/ric_disk_config.rb +230 -193
  21. data/lib/storazzo/ric_disk_sample_config.rb +12 -16
  22. data/lib/storazzo/ric_disk_statsfile.rb +17 -16
  23. data/lib/storazzo/ric_disk_ugly.rb +35 -38
  24. data/lib/storazzo/version.rb +7 -7
  25. data/lib/storazzo.rb +34 -29
  26. data/storazzo.gemspec +22 -20
  27. data/test/media/test_abstract_ric_disk.rb +19 -0
  28. data/test/media/test_gcs_bucket.rb +58 -0
  29. data/test/media/test_local_folder.rb +145 -0
  30. data/test/media/test_mount_point.rb +25 -0
  31. data/test/test_ric_disk.rb +16 -0
  32. data/test/test_ric_disk_config.rb +20 -29
  33. data/test/test_ric_disk_stats_file.rb +13 -14
  34. data/test/test_storazzo.rb +26 -26
  35. data/var/dumps/file_stat.linux.yaml +15 -0
  36. data/var/dumps/file_stat.macosx.yaml +15 -0
  37. data/var/test/disks/disk02-full/Rakefile +13 -0
  38. data/var/test/disks/ricdisk_stats_v11.rds +11 -0
  39. metadata +38 -10
  40. data/test/test_gcs_bucket.rb +0 -70
  41. data/test/test_local_folder.rb +0 -121
@@ -4,62 +4,72 @@
4
4
  require 'digest'
5
5
 
6
6
  module Storazzo
7
- class Storazzo::RicDisk
8
-
7
+ class Storazzo::RicDisk
9
8
  include Hashify
10
- include Storazzo::Common
9
+ include Storazzo::Common
10
+ extend Storazzo::Common
11
11
  extend Storazzo::Colors
12
12
  require 'socket'
13
13
 
14
-
15
- ## Instance variables
14
+ ## Instance variables
16
15
 
17
16
  # in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
18
17
  # These are the files I do accept.
19
- ConfigFiles = %W{ ricdisk.yaml .ricdisk storazzo.yaml }
20
- DefaultConfigFile = "storazzo.yaml" # .ricdisk }
18
+ ConfigFiles = %W{ricdisk.yaml .ricdisk storazzo.yaml}
19
+ DefaultConfigFile = "storazzo.yaml" # .ricdisk }
21
20
  RicdiskVersion = '2.1'
22
21
  RicdiskHistory = [
23
22
  '2022-07-29 2.1 Added timestamp',
24
23
  '2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...',
25
- ]
24
+ ]
26
25
  DefaultGemfileTestDiskFolder = Storazzo.root + "/var/test/disks/" # was: @@default_gemfile_test_disks_folder
27
26
  # Immutable
28
- DefaultMediaFolders = %w{
29
- /Volumes/
30
- /mnt/
31
- }.append(DefaultGemfileTestDiskFolder ).append("/media/#{ENV["USER"]}/" )
32
-
33
- # # todo substitute with protobuf..
34
- attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
35
- :ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
27
+ DefaultMediaFolders = %w{
28
+ /Volumes/
29
+ /mnt/
30
+ }.append(DefaultGemfileTestDiskFolder).append("/media/#{ENV["USER"]}/")
31
+
32
+ # # todo substitute with protobuf..
33
+ attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
34
+ :ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
36
35
  :unique_hash # new 202207
37
36
 
37
+ ################################
38
+ ## INSTANCE methods
39
+ ################################
38
40
 
39
- ################################
40
- ## INSTANCE methods
41
- ################################
41
+ def initialize_old_way(path, opts = {})
42
+ raise "Now I dont want a string in input, I want an OBJECT < Storazzo::Media::AbstractRicDisk"
43
+ end
42
44
 
45
+ def initialize(ric_disk_object, opts = {})
46
+ verbose = opts.fetch :verbose, true
47
+ pverbose verbose,
48
+ "This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
49
+ raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
43
50
 
44
- def initialize(path, opts={})
51
+ # ok back to business, now path is a String :)
52
+ path = ric_disk_object.path
53
+ deb "RicDisk initialize.. path=#{path}"
45
54
  deb "RicDisk initialize.. path=#{path}"
46
55
  @local_mountpoint = File.expand_path(path)
56
+ @ard = ric_disk_object # AbstractRicDiskObject
47
57
  @description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
48
58
  @ricdisk_version = RicdiskVersion
49
59
  @ricdisk_file = compute_ricdisk_file() # Storazzo::RicDisk.get_ricdisk_file(path)
50
60
  @ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
51
61
  @label = path.split("/").last
52
62
  @name = path.split("/").last
53
- #@wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
54
- #@wr = writeable?
63
+ # @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
64
+ # @wr = writeable?
55
65
  @tags = ['ricdisk', 'storazzo']
56
- @size = `du -s '#{path}'`.split(/\s/)[0] # self.size
66
+ @size = RicDisk._compute_size_could_take_long(path)
57
67
  @unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
58
68
  @computation_hostname = Socket.gethostname
59
69
  @created_at = Time.now
60
70
 
61
71
  @ricdisk_file_empty = ricdisk_file_empty?
62
-
72
+
63
73
  # @config = RicDiskConfig.instance.get_config
64
74
  # #puts @config if @config
65
75
  # find_info_from_mount(path)
@@ -77,7 +87,6 @@ module Storazzo
77
87
  not ricdisk_file.nil?
78
88
  end
79
89
 
80
-
81
90
  def analyze_local_system()
82
91
  puts "TODO This should analyzze the WHOLE system. TODO(ricc): move to another object which has to do with the system/computer."
83
92
  puts "1. Interesting Mounts: #{green interesting_mount_points}"
@@ -86,85 +95,108 @@ module Storazzo
86
95
  # find_info_from_df()
87
96
  end
88
97
 
89
- def path
98
+ def path
90
99
  local_mountpoint
91
100
  end
92
101
 
93
- def to_s
94
- "RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version})"
102
+ def to_s
103
+ "RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version}, ard=#{@ard})"
95
104
  end
96
105
 
97
106
  # could take long..
98
- # def size
107
+ # def size
99
108
  # `du -s '#{path}'`.split(/\s/)[0]
100
109
  # end
110
+ def self._compute_size_could_take_long(my_path)
111
+ deb "Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds."
112
+ puts azure('could take long. Please take precautions like forking with timeout of 5sec')
113
+ `du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
114
+ end
101
115
 
102
- def writeable?()
103
- return @wr unless @wr.nil?
104
- # Otherwise I can do an EXPENSIVE calculation
105
- puts yellow("TODO(ricc): Do expensive calculation if this FS is writeable: #{path}")
106
- #@wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
116
+ def writeable?()
117
+ # memoize
118
+ return @wr unless @wr.nil?
119
+
120
+ # NOW: CALCULATE it
121
+ # Now I can do ONCE an EXPENSIVE calculation
122
+ puts yellow("[RicDisk.writeable] TODO(ricc): Do expensive calculation if this FS is writeable: #{path} and write/memoize it on @wr once and for all")
123
+ puts yellow("[RicDisk.writeable] I have a feeling this should be delegated to praecipuus Storazzo::Media::Object we refer to (WR is different on GCS vs Local):") # infinite loop dammit #{self.to_verbose_s}")
124
+ puts("Dir: #{azure path}")
125
+ puts("absolute_path: #{azure absolute_path}")
126
+ puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
127
+ bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
128
+ puts("bash_output: #{azure bash_output}")
129
+ # @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
107
130
  raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
131
+
108
132
  @wr = File.writable?(absolute_path) # rescue false
109
133
  return @wr
110
- #:boh_todo_fix_me_and_compute
111
- #false
134
+ # :boh_todo_fix_me_and_compute
135
+ # false
112
136
  end
113
137
 
114
- ################################
115
- ## CLASS methods
116
- ################################
138
+ def to_verbose_s
139
+ h = {}
140
+ h[:to_s] = self.to_s
141
+ h[:wr] = self.wr
142
+ h[:inspect] = self.inspect
143
+ h[:writeable] = self.writeable?
144
+ h[:ard] = @ard
145
+ return h
146
+ end
117
147
 
148
+ ################################
149
+ ## CLASS methods
150
+ ################################
118
151
 
119
152
  # All places where to find for something :)
120
153
  def self.default_media_folders
121
154
  DefaultMediaFolders # was DEFAULT_MEDIA_FOLDERS
122
155
  end
123
156
 
124
-
125
157
  def self.test # _localgem_disks
126
- d = RicDisk.new( DefaultGemfileTestDiskFolder)
158
+ d = RicDisk.new(DefaultGemfileTestDiskFolder)
127
159
  puts (d)
128
160
  puts "do something with it: #{d}"
129
- #d.find_active_dirs()
161
+ # d.find_active_dirs()
130
162
  end
131
163
 
132
164
  def absolute_path
133
- #@local_mountpoint + "/" + @ricdisk_file
165
+ # @local_mountpoint + "/" + @ricdisk_file
134
166
  "#{local_mountpoint}/#{ricdisk_file}"
135
167
  end
136
-
137
- def self.find_active_dirs(base_dirs=nil, also_mountpoints=true)
168
+
169
+ def self.find_active_dirs(base_dirs = nil, also_mountpoints = true)
138
170
  if base_dirs.nil?
139
- base_dirs = default_media_folders
171
+ base_dirs = default_media_folders
140
172
  puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
141
173
  end
142
174
  active_dirs = []
143
- base_dirs.each do |ugly_dir|
175
+ base_dirs.each do |ugly_dir|
144
176
  # https://stackoverflow.com/questions/1899072/getting-a-list-of-folders-in-a-directory#:~:text=Dir.chdir(%27/destination_directory%27)%0ADir.glob(%27*%27).select%20%7B%7Cf%7C%20File.directory%3F%20f%7D
145
177
  dir = File.expand_path(ugly_dir)
146
178
  begin
147
- x=[]
148
- # puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
179
+ x = []
180
+ # puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
149
181
  unless Dir.exists?(dir)
150
182
  deb "Dir doesnt exist, skipping: #{dir}"
151
- next
183
+ next
152
184
  end
153
185
  Dir.chdir(dir)
154
- x = Dir.glob('*').select {|f| File.directory? f}
155
- subdirs = x.map{|subdir| "#{dir}#{subdir}"}
156
- subdirs.each{|subdir|
157
- puts "DEB Subdir: #{subdir}"
158
- puts `ls -al "#{subdir}"`
186
+ x = Dir.glob('*').select { |f| File.directory? f }
187
+ subdirs = x.map { |subdir| "#{dir}#{subdir}" }
188
+ subdirs.each { |subdir|
189
+ deb "Subdir: #{subdir}"
190
+ puts `ls -al "#{subdir}"` # TODO refactor in exec
159
191
  active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
160
192
  }
161
- #puts(white x)
193
+ # puts(white x)
162
194
  rescue Exception => e # optionally: `rescue Exception => ex`
163
195
  puts "Exception: '#{e}'"
164
196
  ensure # will always get executed
165
- #deb 'Always gets executed.'
166
- #x = []
167
- end
197
+ # deb 'Always gets executed.'
198
+ # x = []
199
+ end
168
200
  end
169
201
  end
170
202
 
@@ -175,16 +207,15 @@ module Storazzo
175
207
  end
176
208
  deb "[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once"
177
209
  ConfigFiles.each do |papable_config_filename|
178
- #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
179
- #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
210
+ # return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
211
+ # return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
180
212
  return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
181
213
  end
182
214
  deb "File not found! Neither #{ConfigFiles} exist.."
183
- # return nil
215
+ # return nil
184
216
  return DefaultConfigFile
185
217
  end
186
218
 
187
-
188
219
  # def self.compute_ricdisk_file_by_path_once(path)
189
220
  # # unless @ricdisk_file.nil?
190
221
  # # deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
@@ -199,7 +230,6 @@ module Storazzo
199
230
  # return nil
200
231
  # end
201
232
 
202
-
203
233
  # # new
204
234
  # def self.get_ricdisk_file_obsolete(path)
205
235
  # if @ricdisk_file
@@ -215,9 +245,8 @@ module Storazzo
215
245
  # return nil
216
246
  # end
217
247
 
218
-
219
- def self.interesting_mount_points(opts={})
220
- #https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
248
+ def self.interesting_mount_points(opts = {})
249
+ # https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
221
250
  `mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
222
251
  end
223
252
 
@@ -227,23 +256,23 @@ module Storazzo
227
256
  deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{(Time.now - creation_time)} seconds ago")
228
257
  (Time.now - creation_time).to_i
229
258
  end
259
+
230
260
  # maybe move to a RiccFile class? Maybe even INHERIT from FILE?
231
261
  def obsolescence_days(file_path)
232
262
  return obsolescence_seconds(file_path) / 86400
233
263
  end
234
264
 
235
-
236
265
  # FORMER SBRODOLA, now write_config_yaml_to_disk
237
- #def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
238
- def write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
266
+ # def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
267
+ def write_config_yaml_to_disk(subdir, opts = {}) # sbrodola_ricdisk(subdir)
239
268
  # given a path, if .ricdisk exists i do stuff with it..
240
269
  disk_info = nil
241
270
  unless ok_dir? # self.ok_dir?(subdir)
242
- puts("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
243
- return
271
+ warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
272
+ return
244
273
  end
245
274
  ConfigFiles.each do |papable_configfile_name|
246
- if File.exists?( "#{subdir}/#{papable_configfile_name}") and File.empty?( "#{subdir}/#{papable_configfile_name}")
275
+ if File.exists?("#{subdir}/#{papable_configfile_name}") and File.empty?("#{subdir}/#{papable_configfile_name}")
247
276
  deb("Interesting. Empty file '#{papable_configfile_name}'! Now I write YAML with it.")
248
277
  disk_info = RicDisk.new(subdir, papable_configfile_name)
249
278
  end
@@ -258,7 +287,7 @@ module Storazzo
258
287
  # puts(yellow disk_info.to_yaml)
259
288
  # end
260
289
  if disk_info.is_a?(RicDisk)
261
- puts yellow("DEB disk_info.class: #{disk_info.class}")
290
+ deb yellow("disk_info.class: #{disk_info.class}")
262
291
  if File.empty?(disk_info.absolute_path) # and (disk_info.wr)
263
292
  puts(green("yay, we can now write the file '#{disk_info.absolute_path}' (which is R/W, I just checked!) with proper YAML content.."))
264
293
  if disk_info.wr
@@ -279,8 +308,8 @@ module Storazzo
279
308
  puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
280
309
  end
281
310
 
282
- #disk_info.absolute_path
283
- #if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
311
+ # disk_info.absolute_path
312
+ # if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
284
313
  # if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
285
314
  # puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
286
315
  # puts(white `cat "#{disk_info.absolute_path}"`)
@@ -292,191 +321,134 @@ module Storazzo
292
321
 
293
322
  # TODO obsolete this as i should NOT be calling it from clas, but from method.
294
323
  def self.ok_dir?(subdir)
295
- File.exists?( "#{subdir}/.ricdisk") or File.exists?( "#{subdir}/.ricdisk.yaml")
324
+ File.exists?("#{subdir}/.ricdisk") or File.exists?("#{subdir}/.ricdisk.yaml")
296
325
  end
297
326
 
298
- def compute_stats_files(opts={})
299
- puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy")
300
- #Storazzo::RicDisk.calculate_stats_files(path, opts)
301
- opts_upload_to_gcs = opts.fetch :upload_to_gcs, true
327
+ def compute_stats_files(opts = {})
328
+ # Storazzo::RicDisk.calculate_stats_files(path, opts)
329
+ opts_upload_to_gcs = opts.fetch :upload_to_gcs, false
330
+ opts_force_rewrite = opts.fetch :force, false
331
+ opts_stats_file = opts.fetch :stats_file, "ricdisk_stats_v11.rds" # default. TODO point to proper..
302
332
  dir = path
303
-
304
- full_file_path = "#{dir}/#{$stats_file}"
305
- #return "This refactor is for another day"
306
-
333
+ puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy. stats_file=#{opts_stats_file} dir=#{dir}")
334
+
335
+ full_file_path = "#{dir}/#{opts_stats_file}"
336
+ deb "This refactor is for another day. Actually no, TODAY "
337
+ pverbose true,
338
+ "TODO(ricc): you should compute more SMARTLY the full_file_path (#{full_file_path}): if its R/O it should be elsewhere.."
339
+ puts azure("- full_file_path: #{full_file_path}")
340
+ puts azure("- writeable?: #{writeable?}")
341
+
307
342
  puts("compute_stats_files(#{white dir}): #{white full_file_path}")
308
- puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
343
+ deb "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
344
+ raise "Directory doesnt exist: #{dir}" unless File.directory?(dir)
345
+
309
346
  Dir.chdir(dir)
310
- if File.exists?(full_file_path) and ($opts[:force] == false)
311
- puts "File '#{$stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
312
- # TODO check for file time...
313
- print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp }. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
314
- if obsolescence_days(full_file_path) > 7
315
- puts("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
347
+ puts azure `ls` # im curious
348
+ if File.exists?(full_file_path)
349
+ if opts_force_rewrite
350
+ # raise "TODO implement file exists and FORCE enabled"
351
+ RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ReWrite enabled")
352
+ else # File.exists?(full_file_path) and (opts_force)
353
+ puts "File '#{opts_stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
354
+ # TODO check for file time...
355
+ print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp}. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
356
+ if obsolescence_days(full_file_path) > 7
357
+ # puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
358
+ puts yellow("*** ACHTUNG *** FIle is pretty old. I'll force a rewrite")
359
+ RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
360
+ "File older than 7 days. Indeed: #{obsolescence_days(full_file_path)}")
361
+ end
362
+ upload_to_gcs(full_file_path) if opts_upload_to_gcs
316
363
  end
317
- upload_to_gcs(full_file_path) if opts_upload_to_gcs
318
364
  else
319
- puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]"
320
- command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
321
- puts("[#{`pwd`.chomp}] Executing: #{azure command}")
322
- ret = backquote_execute(command)
323
- puts "Done. #{ret.split("\n").count} files processed."
365
+ deb("File doesnt exist..")
366
+ RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ConfigFile doesn't exist")
324
367
  end
325
368
  end
326
-
327
-
328
-
329
- # Create RDS file.
330
- def self.calculate_stats_files(dir, opts={})
331
- opts_upload_to_gcs = opts.fetch :upload_to_gcs, true
332
-
333
- full_file_path = "#{dir}/#{$stats_file}"
334
- return "This refactor is for another day"
335
-
336
- puts("calculate_stats_files(#{white dir}): #{white full_file_path}")
337
- puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
338
- Dir.chdir(dir)
339
- if File.exists?(full_file_path) and ($opts[:force] == false)
340
- puts "File '#{$stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
341
- # TODO check for file time...
342
- print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp }. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
343
- if obsolescence_days(full_file_path) > 7
344
- puts("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
345
- end
346
- upload_to_gcs(full_file_path) if opts_upload_to_gcs
347
- else
348
- puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]"
369
+
370
+ def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts = {})
371
+ max_lines = opts.fetch :max_lines, 42 # todo move to nil or -1
372
+ # full_file_path = "#{dir}/#{stats_file}"
373
+ puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
374
+ if max_lines < 0 # infinite
349
375
  command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
350
- puts("[#{`pwd`.chomp}] Executing: #{azure command}")
351
- ret = backquote_execute command
352
- puts "Done. #{ret.split("\n").count} files processed."
376
+ else
377
+ # WOW! https://stackoverflow.com/questions/68599963/reliably-stop-bash-find-after-n-matches
378
+ # find . -type f -iname "*.txt" -print0 |
379
+ # head -z -n 10 |
380
+ # xargs -r0 myscript.sh
381
+ if mac?
382
+ puts red("Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)")
383
+ spannometric_lines = (max_lines / 10)
384
+ command = "find . -print0 | head -n '#{spannometric_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
385
+ else
386
+ command = "find . -print0 | head -z -n '#{max_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
387
+ end
353
388
  end
389
+ puts("[#{`pwd`.chomp}] Executing: #{azure command}")
390
+ ret = backquote_execute(command)
391
+ puts "Done. #{ret.split("\n").count} files processed."
354
392
  end
355
-
356
- # if also_mountpoints
357
- # =begin
358
- # Example output from mount:
359
-
360
- # devfs on /dev (devfs, local, nobrowse)
361
- # /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
362
- # /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
363
- # /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
364
- # /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
365
- # /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
366
- # /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
367
- # /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
368
- # map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
369
- # //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
370
- # //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
371
- # =end
372
- # # add directories from current mountpoints...
373
- # mount_table_lines = interesting_mount_points()
374
- # mount_table_lines.each{|line|
375
- # next if line =~ /^map /
376
- # dev, on, path, mode = line.split(/ /)
377
- # #puts line
378
- # #deb yellow(path)
379
- # active_dirs << path if self.ok_dir?(path)
380
- # }
381
- # end
382
- # active_dirs.uniq!
383
- # puts("find_active_dirs(): found dirs " + green(active_dirs))
384
- # return active_dirs
385
- # end
386
-
387
-
388
-
389
- end #/Class
390
- end #/Module
391
-
392
-
393
-
394
-
395
-
396
- # def initialize(path, ricdisk_file)
397
- # puts "[DEB] RicDisk initialize.. path=#{path}"
398
- # @local_mountpoint = path
399
- # @description = "This is an automated RicDisk description from v.#{VERSION}. Riccardo feel free to edit away with characteristicshs of this device.. Created on #{Time.now}'"
400
- # @ricdisk_version = VERSION
401
- # @ricdisk_file = ricdisk_file
402
- # #@questo_non_esiste = :sobenme
403
- # @label = path.split("/").last
404
- # @name = path.split("/").last
405
- # @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
406
- # @tags = 'ricdisk'
407
- # puts :beleza
408
- # @config = RicDiskConfig.instance.get_config
409
- # #puts @config if @config
410
- # find_info_from_mount(path)
411
- # find_info_from_df()
412
- # end
413
-
414
- # def ricdisk_absolute_path
415
- # @local_mountpoint + "/" + @ricdisk_file
416
- # end
417
-
418
- # def add_tag(tag)
419
- # @tags += ", #{tag}"
420
- # end
421
-
422
- # # might have other things in the future...
423
- # def find_info_from_mount(path)
424
- # mount_table_lines = interesting_mount_points()
425
- # mount_line = nil
426
- # mount_table_lines.each do |line|
427
- # next if line =~ /^map /
428
- # dev, on, mount_path, mode = line.split(/ /)
429
- # if mount_path==path
430
- # mount_line = line
431
- # else
432
- # @info_from_mount = false
433
- # end
434
- # end
435
- # @info_from_mount = ! (mount_line.nil?)
436
- # if @info_from_mount
437
- # #@mount_line = mount_line
438
- # @description += "\nMount line:\n" + mount_line
439
- # @remote_mountpoint = mount_line.split(/ /)[0]
440
- # @fstype = mount_line.split(/ /)[3].gsub(/[\(,]/, '')
441
- # add_tag(:synology) if @remote_mountpoint.match('1.0.1.10')
442
- # end
443
- # end
444
-
445
- # def find_info_from_df()
446
- # path = @local_mountpoint
447
- # df_info = `df -h "#{path}"`
448
- # @df_info = df_info
449
- # lines = df_info.split(/\n+/)
450
- # raise "I need exactly TWO lines! Or no info is served here..." unless lines.size == 2
451
- # mount, @size_readable, used_size, avail_size, @disk_utilization, other = lines[1].split(/\s+/) # second line..
452
- # end
453
-
454
-
455
-
456
-
457
-
458
-
459
- def backquote_execute(cmd, opts={})
460
- dryrun = opts.fetch :dryrun, false
461
- # executed a command wrapped by dryrun though
462
- return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
463
- `#{cmd}`
464
- end
465
-
466
- def upload_to_gcs(file, opts={})
467
- deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
468
- mount_name = file.split('/')[-2]
469
- filename = "#{mount_name}-#{File.basename file}"
470
- hostname = Socket.gethostname[/^[^.]+/]
471
- command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{ hostname }-#{filename}"
472
- deb("Command: #{command}")
473
- puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
474
- ret = backquote_execute(command, :dryrun => true)
475
- # if $opts[:debug] do
476
- # puts "+ Current list of files:"
477
- # ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
478
- # puts ret
479
- # end
480
- ret
393
+
394
+ def self.calculate_stats_files_DUPLICATE_STATIC(dir, opts = {})
395
+ raise "Please use object instead. If you cant, please move the object code to STATIC and dedupe code!"
481
396
  end
482
-
397
+
398
+ # if also_mountpoints
399
+ # =begin
400
+ # Example output from mount:
401
+
402
+ # devfs on /dev (devfs, local, nobrowse)
403
+ # /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
404
+ # /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
405
+ # /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
406
+ # /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
407
+ # /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
408
+ # /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
409
+ # /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
410
+ # map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
411
+ # //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
412
+ # //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
413
+ # =end
414
+ # # add directories from current mountpoints...
415
+ # mount_table_lines = interesting_mount_points()
416
+ # mount_table_lines.each{|line|
417
+ # next if line =~ /^map /
418
+ # dev, on, path, mode = line.split(/ /)
419
+ # #puts line
420
+ # #deb yellow(path)
421
+ # active_dirs << path if self.ok_dir?(path)
422
+ # }
423
+ # end
424
+ # active_dirs.uniq!
425
+ # puts("find_active_dirs(): found dirs " + green(active_dirs))
426
+ # return active_dirs
427
+ # end
428
+ end # /Class
429
+ end # /Module
430
+
431
+ def backquote_execute(cmd, opts = {})
432
+ dryrun = opts.fetch :dryrun, false
433
+ # executed a command wrapped by dryrun though
434
+ return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
435
+
436
+ `#{cmd}`
437
+ end
438
+
439
+ def upload_to_gcs(file, opts = {})
440
+ deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
441
+ mount_name = file.split('/')[-2]
442
+ filename = "#{mount_name}-#{File.basename file}"
443
+ hostname = Socket.gethostname[/^[^.]+/]
444
+ command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{hostname}-#{filename}"
445
+ deb("Command: #{command}")
446
+ puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
447
+ ret = backquote_execute(command, :dryrun => true)
448
+ # if $opts[:debug] do
449
+ # puts "+ Current list of files:"
450
+ # ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
451
+ # puts ret
452
+ # end
453
+ ret
454
+ end