storazzo 0.4.2 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +7 -4
  3. data/Makefile +20 -1
  4. data/README.md +23 -0
  5. data/Rakefile +35 -8
  6. data/VERSION +1 -1
  7. data/bin/hello-storazzo +5 -0
  8. data/bin/ricdisk-magic +50 -57
  9. data/bin/stats-with-md5 +268 -297
  10. data/lib/storazzo/colors.rb +43 -45
  11. data/lib/storazzo/common.rb +80 -17
  12. data/lib/storazzo/debug.rb +5 -8
  13. data/lib/storazzo/hashify.rb +44 -43
  14. data/lib/storazzo/main.rb +44 -40
  15. data/lib/storazzo/media/abstract_ric_disk.rb +163 -104
  16. data/lib/storazzo/media/gcs_bucket.rb +51 -15
  17. data/lib/storazzo/media/local_folder.rb +43 -46
  18. data/lib/storazzo/media/mount_point.rb +12 -2
  19. data/lib/storazzo/ric_disk.rb +223 -251
  20. data/lib/storazzo/ric_disk_config.rb +230 -193
  21. data/lib/storazzo/ric_disk_sample_config.rb +12 -16
  22. data/lib/storazzo/ric_disk_statsfile.rb +17 -16
  23. data/lib/storazzo/ric_disk_ugly.rb +35 -38
  24. data/lib/storazzo/version.rb +7 -7
  25. data/lib/storazzo.rb +34 -29
  26. data/storazzo.gemspec +22 -20
  27. data/test/media/test_abstract_ric_disk.rb +19 -0
  28. data/test/media/test_gcs_bucket.rb +58 -0
  29. data/test/media/test_local_folder.rb +145 -0
  30. data/test/media/test_mount_point.rb +25 -0
  31. data/test/test_ric_disk.rb +16 -0
  32. data/test/test_ric_disk_config.rb +20 -29
  33. data/test/test_ric_disk_stats_file.rb +13 -14
  34. data/test/test_storazzo.rb +26 -26
  35. data/var/dumps/file_stat.linux.yaml +15 -0
  36. data/var/dumps/file_stat.macosx.yaml +15 -0
  37. data/var/test/disks/disk02-full/Rakefile +13 -0
  38. data/var/test/disks/ricdisk_stats_v11.rds +11 -0
  39. metadata +38 -10
  40. data/test/test_gcs_bucket.rb +0 -70
  41. data/test/test_local_folder.rb +0 -121
@@ -4,62 +4,72 @@
4
4
  require 'digest'
5
5
 
6
6
  module Storazzo
7
- class Storazzo::RicDisk
8
-
7
+ class Storazzo::RicDisk
9
8
  include Hashify
10
- include Storazzo::Common
9
+ include Storazzo::Common
10
+ extend Storazzo::Common
11
11
  extend Storazzo::Colors
12
12
  require 'socket'
13
13
 
14
-
15
- ## Instance variables
14
+ ## Instance variables
16
15
 
17
16
  # in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
18
17
  # These are the files I do accept.
19
- ConfigFiles = %W{ ricdisk.yaml .ricdisk storazzo.yaml }
20
- DefaultConfigFile = "storazzo.yaml" # .ricdisk }
18
+ ConfigFiles = %W{ricdisk.yaml .ricdisk storazzo.yaml}
19
+ DefaultConfigFile = "storazzo.yaml" # .ricdisk }
21
20
  RicdiskVersion = '2.1'
22
21
  RicdiskHistory = [
23
22
  '2022-07-29 2.1 Added timestamp',
24
23
  '2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...',
25
- ]
24
+ ]
26
25
  DefaultGemfileTestDiskFolder = Storazzo.root + "/var/test/disks/" # was: @@default_gemfile_test_disks_folder
27
26
  # Immutable
28
- DefaultMediaFolders = %w{
29
- /Volumes/
30
- /mnt/
31
- }.append(DefaultGemfileTestDiskFolder ).append("/media/#{ENV["USER"]}/" )
32
-
33
- # # todo substitute with protobuf..
34
- attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
35
- :ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
27
+ DefaultMediaFolders = %w{
28
+ /Volumes/
29
+ /mnt/
30
+ }.append(DefaultGemfileTestDiskFolder).append("/media/#{ENV["USER"]}/")
31
+
32
+ # # todo substitute with protobuf..
33
+ attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
34
+ :ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
36
35
  :unique_hash # new 202207
37
36
 
37
+ ################################
38
+ ## INSTANCE methods
39
+ ################################
38
40
 
39
- ################################
40
- ## INSTANCE methods
41
- ################################
41
+ def initialize_old_way(path, opts = {})
42
+ raise "Now I dont want a string in input, I want an OBJECT < Storazzo::Media::AbstractRicDisk"
43
+ end
42
44
 
45
+ def initialize(ric_disk_object, opts = {})
46
+ verbose = opts.fetch :verbose, true
47
+ pverbose verbose,
48
+ "This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
49
+ raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
43
50
 
44
- def initialize(path, opts={})
51
+ # ok back to business, now path is a String :)
52
+ path = ric_disk_object.path
53
+ deb "RicDisk initialize.. path=#{path}"
45
54
  deb "RicDisk initialize.. path=#{path}"
46
55
  @local_mountpoint = File.expand_path(path)
56
+ @ard = ric_disk_object # AbstractRicDiskObject
47
57
  @description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
48
58
  @ricdisk_version = RicdiskVersion
49
59
  @ricdisk_file = compute_ricdisk_file() # Storazzo::RicDisk.get_ricdisk_file(path)
50
60
  @ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
51
61
  @label = path.split("/").last
52
62
  @name = path.split("/").last
53
- #@wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
54
- #@wr = writeable?
63
+ # @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
64
+ # @wr = writeable?
55
65
  @tags = ['ricdisk', 'storazzo']
56
- @size = `du -s '#{path}'`.split(/\s/)[0] # self.size
66
+ @size = RicDisk._compute_size_could_take_long(path)
57
67
  @unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
58
68
  @computation_hostname = Socket.gethostname
59
69
  @created_at = Time.now
60
70
 
61
71
  @ricdisk_file_empty = ricdisk_file_empty?
62
-
72
+
63
73
  # @config = RicDiskConfig.instance.get_config
64
74
  # #puts @config if @config
65
75
  # find_info_from_mount(path)
@@ -77,7 +87,6 @@ module Storazzo
77
87
  not ricdisk_file.nil?
78
88
  end
79
89
 
80
-
81
90
  def analyze_local_system()
82
91
  puts "TODO This should analyzze the WHOLE system. TODO(ricc): move to another object which has to do with the system/computer."
83
92
  puts "1. Interesting Mounts: #{green interesting_mount_points}"
@@ -86,85 +95,108 @@ module Storazzo
86
95
  # find_info_from_df()
87
96
  end
88
97
 
89
- def path
98
+ def path
90
99
  local_mountpoint
91
100
  end
92
101
 
93
- def to_s
94
- "RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version})"
102
+ def to_s
103
+ "RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version}, ard=#{@ard})"
95
104
  end
96
105
 
97
106
  # could take long..
98
- # def size
107
+ # def size
99
108
  # `du -s '#{path}'`.split(/\s/)[0]
100
109
  # end
110
+ def self._compute_size_could_take_long(my_path)
111
+ deb "Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds."
112
+ puts azure('could take long. Please take precautions like forking with timeout of 5sec')
113
+ `du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
114
+ end
101
115
 
102
- def writeable?()
103
- return @wr unless @wr.nil?
104
- # Otherwise I can do an EXPENSIVE calculation
105
- puts yellow("TODO(ricc): Do expensive calculation if this FS is writeable: #{path}")
106
- #@wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
116
+ def writeable?()
117
+ # memoize
118
+ return @wr unless @wr.nil?
119
+
120
+ # NOW: CALCULATE it
121
+ # Now I can do ONCE an EXPENSIVE calculation
122
+ puts yellow("[RicDisk.writeable] TODO(ricc): Do expensive calculation if this FS is writeable: #{path} and write/memoize it on @wr once and for all")
123
+ puts yellow("[RicDisk.writeable] I have a feeling this should be delegated to praecipuus Storazzo::Media::Object we refer to (WR is different on GCS vs Local):") # infinite loop dammit #{self.to_verbose_s}")
124
+ puts("Dir: #{azure path}")
125
+ puts("absolute_path: #{azure absolute_path}")
126
+ puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
127
+ bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
128
+ puts("bash_output: #{azure bash_output}")
129
+ # @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
107
130
  raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
131
+
108
132
  @wr = File.writable?(absolute_path) # rescue false
109
133
  return @wr
110
- #:boh_todo_fix_me_and_compute
111
- #false
134
+ # :boh_todo_fix_me_and_compute
135
+ # false
112
136
  end
113
137
 
114
- ################################
115
- ## CLASS methods
116
- ################################
138
+ def to_verbose_s
139
+ h = {}
140
+ h[:to_s] = self.to_s
141
+ h[:wr] = self.wr
142
+ h[:inspect] = self.inspect
143
+ h[:writeable] = self.writeable?
144
+ h[:ard] = @ard
145
+ return h
146
+ end
117
147
 
148
+ ################################
149
+ ## CLASS methods
150
+ ################################
118
151
 
119
152
  # All places where to find for something :)
120
153
  def self.default_media_folders
121
154
  DefaultMediaFolders # was DEFAULT_MEDIA_FOLDERS
122
155
  end
123
156
 
124
-
125
157
  def self.test # _localgem_disks
126
- d = RicDisk.new( DefaultGemfileTestDiskFolder)
158
+ d = RicDisk.new(DefaultGemfileTestDiskFolder)
127
159
  puts (d)
128
160
  puts "do something with it: #{d}"
129
- #d.find_active_dirs()
161
+ # d.find_active_dirs()
130
162
  end
131
163
 
132
164
  def absolute_path
133
- #@local_mountpoint + "/" + @ricdisk_file
165
+ # @local_mountpoint + "/" + @ricdisk_file
134
166
  "#{local_mountpoint}/#{ricdisk_file}"
135
167
  end
136
-
137
- def self.find_active_dirs(base_dirs=nil, also_mountpoints=true)
168
+
169
+ def self.find_active_dirs(base_dirs = nil, also_mountpoints = true)
138
170
  if base_dirs.nil?
139
- base_dirs = default_media_folders
171
+ base_dirs = default_media_folders
140
172
  puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
141
173
  end
142
174
  active_dirs = []
143
- base_dirs.each do |ugly_dir|
175
+ base_dirs.each do |ugly_dir|
144
176
  # https://stackoverflow.com/questions/1899072/getting-a-list-of-folders-in-a-directory#:~:text=Dir.chdir(%27/destination_directory%27)%0ADir.glob(%27*%27).select%20%7B%7Cf%7C%20File.directory%3F%20f%7D
145
177
  dir = File.expand_path(ugly_dir)
146
178
  begin
147
- x=[]
148
- # puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
179
+ x = []
180
+ # puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
149
181
  unless Dir.exists?(dir)
150
182
  deb "Dir doesnt exist, skipping: #{dir}"
151
- next
183
+ next
152
184
  end
153
185
  Dir.chdir(dir)
154
- x = Dir.glob('*').select {|f| File.directory? f}
155
- subdirs = x.map{|subdir| "#{dir}#{subdir}"}
156
- subdirs.each{|subdir|
157
- puts "DEB Subdir: #{subdir}"
158
- puts `ls -al "#{subdir}"`
186
+ x = Dir.glob('*').select { |f| File.directory? f }
187
+ subdirs = x.map { |subdir| "#{dir}#{subdir}" }
188
+ subdirs.each { |subdir|
189
+ deb "Subdir: #{subdir}"
190
+ puts `ls -al "#{subdir}"` # TODO refactor in exec
159
191
  active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
160
192
  }
161
- #puts(white x)
193
+ # puts(white x)
162
194
  rescue Exception => e # optionally: `rescue Exception => ex`
163
195
  puts "Exception: '#{e}'"
164
196
  ensure # will always get executed
165
- #deb 'Always gets executed.'
166
- #x = []
167
- end
197
+ # deb 'Always gets executed.'
198
+ # x = []
199
+ end
168
200
  end
169
201
  end
170
202
 
@@ -175,16 +207,15 @@ module Storazzo
175
207
  end
176
208
  deb "[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once"
177
209
  ConfigFiles.each do |papable_config_filename|
178
- #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
179
- #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
210
+ # return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
211
+ # return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
180
212
  return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
181
213
  end
182
214
  deb "File not found! Neither #{ConfigFiles} exist.."
183
- # return nil
215
+ # return nil
184
216
  return DefaultConfigFile
185
217
  end
186
218
 
187
-
188
219
  # def self.compute_ricdisk_file_by_path_once(path)
189
220
  # # unless @ricdisk_file.nil?
190
221
  # # deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
@@ -199,7 +230,6 @@ module Storazzo
199
230
  # return nil
200
231
  # end
201
232
 
202
-
203
233
  # # new
204
234
  # def self.get_ricdisk_file_obsolete(path)
205
235
  # if @ricdisk_file
@@ -215,9 +245,8 @@ module Storazzo
215
245
  # return nil
216
246
  # end
217
247
 
218
-
219
- def self.interesting_mount_points(opts={})
220
- #https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
248
+ def self.interesting_mount_points(opts = {})
249
+ # https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
221
250
  `mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
222
251
  end
223
252
 
@@ -227,23 +256,23 @@ module Storazzo
227
256
  deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{(Time.now - creation_time)} seconds ago")
228
257
  (Time.now - creation_time).to_i
229
258
  end
259
+
230
260
  # maybe move to a RiccFile class? Maybe even INHERIT from FILE?
231
261
  def obsolescence_days(file_path)
232
262
  return obsolescence_seconds(file_path) / 86400
233
263
  end
234
264
 
235
-
236
265
  # FORMER SBRODOLA, now write_config_yaml_to_disk
237
- #def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
238
- def write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
266
+ # def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
267
+ def write_config_yaml_to_disk(subdir, opts = {}) # sbrodola_ricdisk(subdir)
239
268
  # given a path, if .ricdisk exists i do stuff with it..
240
269
  disk_info = nil
241
270
  unless ok_dir? # self.ok_dir?(subdir)
242
- puts("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
243
- return
271
+ warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
272
+ return
244
273
  end
245
274
  ConfigFiles.each do |papable_configfile_name|
246
- if File.exists?( "#{subdir}/#{papable_configfile_name}") and File.empty?( "#{subdir}/#{papable_configfile_name}")
275
+ if File.exists?("#{subdir}/#{papable_configfile_name}") and File.empty?("#{subdir}/#{papable_configfile_name}")
247
276
  deb("Interesting. Empty file '#{papable_configfile_name}'! Now I write YAML with it.")
248
277
  disk_info = RicDisk.new(subdir, papable_configfile_name)
249
278
  end
@@ -258,7 +287,7 @@ module Storazzo
258
287
  # puts(yellow disk_info.to_yaml)
259
288
  # end
260
289
  if disk_info.is_a?(RicDisk)
261
- puts yellow("DEB disk_info.class: #{disk_info.class}")
290
+ deb yellow("disk_info.class: #{disk_info.class}")
262
291
  if File.empty?(disk_info.absolute_path) # and (disk_info.wr)
263
292
  puts(green("yay, we can now write the file '#{disk_info.absolute_path}' (which is R/W, I just checked!) with proper YAML content.."))
264
293
  if disk_info.wr
@@ -279,8 +308,8 @@ module Storazzo
279
308
  puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
280
309
  end
281
310
 
282
- #disk_info.absolute_path
283
- #if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
311
+ # disk_info.absolute_path
312
+ # if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
284
313
  # if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
285
314
  # puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
286
315
  # puts(white `cat "#{disk_info.absolute_path}"`)
@@ -292,191 +321,134 @@ module Storazzo
292
321
 
293
322
  # TODO obsolete this as i should NOT be calling it from clas, but from method.
294
323
  def self.ok_dir?(subdir)
295
- File.exists?( "#{subdir}/.ricdisk") or File.exists?( "#{subdir}/.ricdisk.yaml")
324
+ File.exists?("#{subdir}/.ricdisk") or File.exists?("#{subdir}/.ricdisk.yaml")
296
325
  end
297
326
 
298
- def compute_stats_files(opts={})
299
- puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy")
300
- #Storazzo::RicDisk.calculate_stats_files(path, opts)
301
- opts_upload_to_gcs = opts.fetch :upload_to_gcs, true
327
+ def compute_stats_files(opts = {})
328
+ # Storazzo::RicDisk.calculate_stats_files(path, opts)
329
+ opts_upload_to_gcs = opts.fetch :upload_to_gcs, false
330
+ opts_force_rewrite = opts.fetch :force, false
331
+ opts_stats_file = opts.fetch :stats_file, "ricdisk_stats_v11.rds" # default. TODO point to proper..
302
332
  dir = path
303
-
304
- full_file_path = "#{dir}/#{$stats_file}"
305
- #return "This refactor is for another day"
306
-
333
+ puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy. stats_file=#{opts_stats_file} dir=#{dir}")
334
+
335
+ full_file_path = "#{dir}/#{opts_stats_file}"
336
+ deb "This refactor is for another day. Actually no, TODAY "
337
+ pverbose true,
338
+ "TODO(ricc): you should compute more SMARTLY the full_file_path (#{full_file_path}): if its R/O it should be elsewhere.."
339
+ puts azure("- full_file_path: #{full_file_path}")
340
+ puts azure("- writeable?: #{writeable?}")
341
+
307
342
  puts("compute_stats_files(#{white dir}): #{white full_file_path}")
308
- puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
343
+ deb "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
344
+ raise "Directory doesnt exist: #{dir}" unless File.directory?(dir)
345
+
309
346
  Dir.chdir(dir)
310
- if File.exists?(full_file_path) and ($opts[:force] == false)
311
- puts "File '#{$stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
312
- # TODO check for file time...
313
- print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp }. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
314
- if obsolescence_days(full_file_path) > 7
315
- puts("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
347
+ puts azure `ls` # im curious
348
+ if File.exists?(full_file_path)
349
+ if opts_force_rewrite
350
+ # raise "TODO implement file exists and FORCE enabled"
351
+ RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ReWrite enabled")
352
+ else # File.exists?(full_file_path) and (opts_force)
353
+ puts "File '#{opts_stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
354
+ # TODO check for file time...
355
+ print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp}. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
356
+ if obsolescence_days(full_file_path) > 7
357
+ # puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
358
+ puts yellow("*** ACHTUNG *** FIle is pretty old. I'll force a rewrite")
359
+ RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
360
+ "File older than 7 days. Indeed: #{obsolescence_days(full_file_path)}")
361
+ end
362
+ upload_to_gcs(full_file_path) if opts_upload_to_gcs
316
363
  end
317
- upload_to_gcs(full_file_path) if opts_upload_to_gcs
318
364
  else
319
- puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]"
320
- command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
321
- puts("[#{`pwd`.chomp}] Executing: #{azure command}")
322
- ret = backquote_execute(command)
323
- puts "Done. #{ret.split("\n").count} files processed."
365
+ deb("File doesnt exist..")
366
+ RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ConfigFile doesn't exist")
324
367
  end
325
368
  end
326
-
327
-
328
-
329
- # Create RDS file.
330
- def self.calculate_stats_files(dir, opts={})
331
- opts_upload_to_gcs = opts.fetch :upload_to_gcs, true
332
-
333
- full_file_path = "#{dir}/#{$stats_file}"
334
- return "This refactor is for another day"
335
-
336
- puts("calculate_stats_files(#{white dir}): #{white full_file_path}")
337
- puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
338
- Dir.chdir(dir)
339
- if File.exists?(full_file_path) and ($opts[:force] == false)
340
- puts "File '#{$stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
341
- # TODO check for file time...
342
- print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp }. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
343
- if obsolescence_days(full_file_path) > 7
344
- puts("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
345
- end
346
- upload_to_gcs(full_file_path) if opts_upload_to_gcs
347
- else
348
- puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]"
369
+
370
+ def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts = {})
371
+ max_lines = opts.fetch :max_lines, 42 # todo move to nil or -1
372
+ # full_file_path = "#{dir}/#{stats_file}"
373
+ puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
374
+ if max_lines < 0 # infinite
349
375
  command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
350
- puts("[#{`pwd`.chomp}] Executing: #{azure command}")
351
- ret = backquote_execute command
352
- puts "Done. #{ret.split("\n").count} files processed."
376
+ else
377
+ # WOW! https://stackoverflow.com/questions/68599963/reliably-stop-bash-find-after-n-matches
378
+ # find . -type f -iname "*.txt" -print0 |
379
+ # head -z -n 10 |
380
+ # xargs -r0 myscript.sh
381
+ if mac?
382
+ puts red("Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)")
383
+ spannometric_lines = (max_lines / 10)
384
+ command = "find . -print0 | head -n '#{spannometric_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
385
+ else
386
+ command = "find . -print0 | head -z -n '#{max_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
387
+ end
353
388
  end
389
+ puts("[#{`pwd`.chomp}] Executing: #{azure command}")
390
+ ret = backquote_execute(command)
391
+ puts "Done. #{ret.split("\n").count} files processed."
354
392
  end
355
-
356
- # if also_mountpoints
357
- # =begin
358
- # Example output from mount:
359
-
360
- # devfs on /dev (devfs, local, nobrowse)
361
- # /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
362
- # /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
363
- # /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
364
- # /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
365
- # /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
366
- # /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
367
- # /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
368
- # map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
369
- # //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
370
- # //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
371
- # =end
372
- # # add directories from current mountpoints...
373
- # mount_table_lines = interesting_mount_points()
374
- # mount_table_lines.each{|line|
375
- # next if line =~ /^map /
376
- # dev, on, path, mode = line.split(/ /)
377
- # #puts line
378
- # #deb yellow(path)
379
- # active_dirs << path if self.ok_dir?(path)
380
- # }
381
- # end
382
- # active_dirs.uniq!
383
- # puts("find_active_dirs(): found dirs " + green(active_dirs))
384
- # return active_dirs
385
- # end
386
-
387
-
388
-
389
- end #/Class
390
- end #/Module
391
-
392
-
393
-
394
-
395
-
396
- # def initialize(path, ricdisk_file)
397
- # puts "[DEB] RicDisk initialize.. path=#{path}"
398
- # @local_mountpoint = path
399
- # @description = "This is an automated RicDisk description from v.#{VERSION}. Riccardo feel free to edit away with characteristicshs of this device.. Created on #{Time.now}'"
400
- # @ricdisk_version = VERSION
401
- # @ricdisk_file = ricdisk_file
402
- # #@questo_non_esiste = :sobenme
403
- # @label = path.split("/").last
404
- # @name = path.split("/").last
405
- # @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
406
- # @tags = 'ricdisk'
407
- # puts :beleza
408
- # @config = RicDiskConfig.instance.get_config
409
- # #puts @config if @config
410
- # find_info_from_mount(path)
411
- # find_info_from_df()
412
- # end
413
-
414
- # def ricdisk_absolute_path
415
- # @local_mountpoint + "/" + @ricdisk_file
416
- # end
417
-
418
- # def add_tag(tag)
419
- # @tags += ", #{tag}"
420
- # end
421
-
422
- # # might have other things in the future...
423
- # def find_info_from_mount(path)
424
- # mount_table_lines = interesting_mount_points()
425
- # mount_line = nil
426
- # mount_table_lines.each do |line|
427
- # next if line =~ /^map /
428
- # dev, on, mount_path, mode = line.split(/ /)
429
- # if mount_path==path
430
- # mount_line = line
431
- # else
432
- # @info_from_mount = false
433
- # end
434
- # end
435
- # @info_from_mount = ! (mount_line.nil?)
436
- # if @info_from_mount
437
- # #@mount_line = mount_line
438
- # @description += "\nMount line:\n" + mount_line
439
- # @remote_mountpoint = mount_line.split(/ /)[0]
440
- # @fstype = mount_line.split(/ /)[3].gsub(/[\(,]/, '')
441
- # add_tag(:synology) if @remote_mountpoint.match('1.0.1.10')
442
- # end
443
- # end
444
-
445
- # def find_info_from_df()
446
- # path = @local_mountpoint
447
- # df_info = `df -h "#{path}"`
448
- # @df_info = df_info
449
- # lines = df_info.split(/\n+/)
450
- # raise "I need exactly TWO lines! Or no info is served here..." unless lines.size == 2
451
- # mount, @size_readable, used_size, avail_size, @disk_utilization, other = lines[1].split(/\s+/) # second line..
452
- # end
453
-
454
-
455
-
456
-
457
-
458
-
459
- def backquote_execute(cmd, opts={})
460
- dryrun = opts.fetch :dryrun, false
461
- # executed a command wrapped by dryrun though
462
- return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
463
- `#{cmd}`
464
- end
465
-
466
- def upload_to_gcs(file, opts={})
467
- deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
468
- mount_name = file.split('/')[-2]
469
- filename = "#{mount_name}-#{File.basename file}"
470
- hostname = Socket.gethostname[/^[^.]+/]
471
- command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{ hostname }-#{filename}"
472
- deb("Command: #{command}")
473
- puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
474
- ret = backquote_execute(command, :dryrun => true)
475
- # if $opts[:debug] do
476
- # puts "+ Current list of files:"
477
- # ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
478
- # puts ret
479
- # end
480
- ret
393
+
394
+ def self.calculate_stats_files_DUPLICATE_STATIC(dir, opts = {})
395
+ raise "Please use object instead. If you cant, please move the object code to STATIC and dedupe code!"
481
396
  end
482
-
397
+
398
+ # if also_mountpoints
399
+ # =begin
400
+ # Example output from mount:
401
+
402
+ # devfs on /dev (devfs, local, nobrowse)
403
+ # /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
404
+ # /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
405
+ # /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
406
+ # /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
407
+ # /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
408
+ # /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
409
+ # /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
410
+ # map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
411
+ # //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
412
+ # //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
413
+ # =end
414
+ # # add directories from current mountpoints...
415
+ # mount_table_lines = interesting_mount_points()
416
+ # mount_table_lines.each{|line|
417
+ # next if line =~ /^map /
418
+ # dev, on, path, mode = line.split(/ /)
419
+ # #puts line
420
+ # #deb yellow(path)
421
+ # active_dirs << path if self.ok_dir?(path)
422
+ # }
423
+ # end
424
+ # active_dirs.uniq!
425
+ # puts("find_active_dirs(): found dirs " + green(active_dirs))
426
+ # return active_dirs
427
+ # end
428
+ end # /Class
429
+ end # /Module
430
+
431
+ def backquote_execute(cmd, opts = {})
432
+ dryrun = opts.fetch :dryrun, false
433
+ # executed a command wrapped by dryrun though
434
+ return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
435
+
436
+ `#{cmd}`
437
+ end
438
+
439
+ def upload_to_gcs(file, opts = {})
440
+ deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
441
+ mount_name = file.split('/')[-2]
442
+ filename = "#{mount_name}-#{File.basename file}"
443
+ hostname = Socket.gethostname[/^[^.]+/]
444
+ command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{hostname}-#{filename}"
445
+ deb("Command: #{command}")
446
+ puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
447
+ ret = backquote_execute(command, :dryrun => true)
448
+ # if $opts[:debug] do
449
+ # puts "+ Current list of files:"
450
+ # ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
451
+ # puts ret
452
+ # end
453
+ ret
454
+ end