storazzo 0.3.7 → 0.4.2

data/lib/storazzo/ric_disk.rb

@@ -16,8 +16,13 @@ module Storazzo
 
     # in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
     # These are the files I do accept.
-    ConfigFiles = %W{ ricdisk.yaml .ricdisk  } 
-    RicdiskVersion = '2.0'
+    ConfigFiles = %W{ ricdisk.yaml .ricdisk storazzo.yaml } 
+    DefaultConfigFile = "storazzo.yaml" # .ricdisk } 
+    RicdiskVersion = '2.1'
+    RicdiskHistory = [
+      '2022-07-29 2.1 Added timestamp',
+      '2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...',
+  ]
     DefaultGemfileTestDiskFolder = Storazzo.root + "/var/test/disks/" # was: @@default_gemfile_test_disks_folder
     # Immutable
     DefaultMediaFolders = %w{ 
@@ -26,7 +31,7 @@ module Storazzo
      }.append(DefaultGemfileTestDiskFolder ).append("/media/#{ENV["USER"]}/" )
 
      #     # todo substitute with protobuf..
-    attr_accessor :name, :description, :ricdisk_file,:ricdisk_file_full, :local_mountpoint, :wr, :path, 
+    attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path, 
                   :ricdisk_file_empty, :size, :active_dirs, :ricdisk_version, 
                   :unique_hash # new 202207
 
@@ -38,7 +43,6 @@ module Storazzo
 
     def initialize(path, opts={})
       deb "RicDisk initialize.. path=#{path}"
-#      @local_mountpoint = path
       @local_mountpoint = File.expand_path(path)
       @description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
       @ricdisk_version = RicdiskVersion
@@ -52,6 +56,7 @@ module Storazzo
       @size = `du -s '#{path}'`.split(/\s/)[0] # self.size
       @unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) #   hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
       @computation_hostname = Socket.gethostname
+      @created_at = Time.now
 
       @ricdisk_file_empty = ricdisk_file_empty?
       
@@ -97,8 +102,10 @@ module Storazzo
     def writeable?() 
       return @wr unless @wr.nil? 
       # Otherwise I can do an EXPENSIVE calculation
-      puts "TODO(ricc): Do expensive calculation if this FS is writeable: #{path}"
-      @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
+      puts yellow("TODO(ricc): Do expensive calculation if this FS is writeable: #{path}")
+      #@wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
+      raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
+      @wr = File.writable?(absolute_path) # rescue false
       return @wr
       #:boh_todo_fix_me_and_compute
       #false
@@ -123,9 +130,9 @@ module Storazzo
     end
 
     def absolute_path
-      @local_mountpoint + "/" +  @ricdisk_file
+      #@local_mountpoint + "/" +  @ricdisk_file
+      "#{local_mountpoint}/#{ricdisk_file}"
     end
-
   
     def self.find_active_dirs(base_dirs=nil, also_mountpoints=true)
       if base_dirs.nil?
@@ -166,15 +173,32 @@ module Storazzo
         deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
         return @ricdisk_file
       end
-      warn "RICC_WARNING This requires cmputation I wanna do it almost once"
+      deb "[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once"
       ConfigFiles.each do |papable_config_filename|
         #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
         #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
         return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
       end
-      return nil
+      deb "File not found! Neither #{ConfigFiles} exist.."
+#      return nil
+      return DefaultConfigFile
     end
 
+
+    # def self.compute_ricdisk_file_by_path_once(path)
+    #   # unless @ricdisk_file.nil?
+    #   #   deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
+    #   #   return @ricdisk_file
+    #   # end
+    #   warn "RICC_WARNING This requires cmputation I wanna do it almost once"
+    #   ConfigFiles.each do |papable_config_filename|
+    #     #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
+    #     #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
+    #     return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
+    #   end
+    #   return nil
+    # end
+
     
     # # new
     # def self.get_ricdisk_file_obsolete(path)
@@ -198,13 +222,13 @@ module Storazzo
     end
 
     # maybe move to a RiccFile class? Maybe even INHERIT from FILE?
-    def self.obsolescence_seconds file_path
+    def obsolescence_seconds(file_path)
       creation_time = File.stat(file_path).ctime
       deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{(Time.now - creation_time)} seconds ago")
       (Time.now - creation_time).to_i
     end
     # maybe move to a RiccFile class? Maybe even INHERIT from FILE?
-    def self.obsolescence_days(file_path)
+    def obsolescence_days(file_path)
       return obsolescence_seconds(file_path) / 86400
     end
 
@@ -234,11 +258,16 @@ module Storazzo
       #   puts(yellow disk_info.to_yaml)
       # end
       if disk_info.is_a?(RicDisk)
-        deb("disk_info.class: #{disk_info.class}")
-        if File.empty?(disk_info.absolute_path) and (disk_info.wr)
+        puts yellow("DEB disk_info.class: #{disk_info.class}")
+        if File.empty?(disk_info.absolute_path) # and (disk_info.wr)
           puts(green("yay, we can now write the file '#{disk_info.absolute_path}' (which is R/W, I just checked!) with proper YAML content.."))
-          ret = File.write(disk_info.absolute_path, disk_info.to_yaml)
-          puts("Written file! ret=#{ret}")
+          if disk_info.wr
+            ret = File.write(disk_info.absolute_path, disk_info.obj_to_yaml)
+            puts green("Written file! ret=#{ret}")
+          else
+            raise "TODO_IMPLEMENT: write in proper place in config dir"
+            puts red("TODO implement me")
+          end
         else
           puts(red("Something not right here: either file is NOT empty or disk is NOT writeable.. #{File.empty?(disk_info.absolute_path)}"))
           puts("File size: #{File.size(disk_info.absolute_path)}")
@@ -246,30 +275,63 @@ module Storazzo
           puts(disk_info.obj_to_hash)
           puts(disk_info.obj_to_yaml)
         end
-      else 
+      else # not a RicDisk..
         puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
       end
-      if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
-        puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
-        puts(white `cat "#{subdir}/.ricdisk"`)
-      else
-        puts "WRITING NOW. disk_info.obj_to_yaml .. to #{compute_ricdisk_file}"
-        File.open(ricdisk_file_full, 'w').write(disk_info.obj_to_yaml)
-      end
+
+      #disk_info.absolute_path
+      #if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
+      # if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
+      #   puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
+      #   puts(white `cat "#{disk_info.absolute_path}"`)
+      # else
+      #   puts "WRITING NOW. [I BELIEVE THIS IS DUPE CODE - see a few lines above!] disk_info.obj_to_yaml .. to #{compute_ricdisk_file}"
+      #   File.open(ricdisk_file_full, 'w').write(disk_info.obj_to_yaml)
+      # end
     end
 
     # TODO obsolete this as i should NOT be calling it from clas, but from method.
     def self.ok_dir?(subdir)
       File.exists?( "#{subdir}/.ricdisk") or File.exists?( "#{subdir}/.ricdisk.yaml")
     end
+
+    def compute_stats_files(opts={})
+      puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy")
+      #Storazzo::RicDisk.calculate_stats_files(path, opts)
+      opts_upload_to_gcs = opts.fetch :upload_to_gcs, true
+      dir = path
+      
+      full_file_path = "#{dir}/#{$stats_file}"
+      #return "This refactor is for another day"
+  
+      puts("compute_stats_files(#{white dir}): #{white full_file_path}")
+      puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
+      Dir.chdir(dir)
+      if File.exists?(full_file_path) and ($opts[:force] == false)
+        puts "File '#{$stats_file}' exists already." #  - now should see if its too old, like more than 1 week old"
+        # TODO check for file time...
+        print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp }. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
+        if obsolescence_days(full_file_path) > 7 
+          puts("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
+        end
+        upload_to_gcs(full_file_path) if opts_upload_to_gcs
+      else
+        puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]" 
+        command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
+        puts("[#{`pwd`.chomp}] Executing: #{azure command}")
+        ret = backquote_execute(command)
+        puts "Done. #{ret.split("\n").count} files processed."
+      end
+    end
   
 
 
     # Create RDS file.
     def self.calculate_stats_files(dir, opts={})
-      opts_upload_to_gcs = opts.fetch :upload_to_gcs, true 
+      opts_upload_to_gcs = opts.fetch :upload_to_gcs, true
+      
       full_file_path = "#{dir}/#{$stats_file}"
-      return "This refacgtor is for another day"
+      return "This refactor is for another day"
   
       puts("calculate_stats_files(#{white dir}): #{white full_file_path}")
       puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
@@ -284,7 +346,7 @@ module Storazzo
         upload_to_gcs(full_file_path) if opts_upload_to_gcs
       else
         puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]" 
-        command = "find . -print0 | xargs -0 stats-with-md5.rb --no-color | tee '#{full_file_path}'"
+        command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
         puts("[#{`pwd`.chomp}] Executing: #{azure command}")
         ret = backquote_execute command
         puts "Done. #{ret.split("\n").count} files processed."
@@ -394,25 +456,27 @@ end  #/Module
 
 
   
-#     def self.backquote_execute(cmd)
-#       # executed a command wrapped by dryrun though
-#       return "DRYRUN backquote_execute(#{cmd})" if $opts[:dryrun]
-#       `#{cmd}`
-#     end
+    def backquote_execute(cmd, opts={})
+      dryrun = opts.fetch :dryrun, false
+      # executed a command wrapped by dryrun though
+      return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
+      `#{cmd}`
+    end
     
-#     def self.upload_to_gcs(file, opts={})
-#       deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
-#       mount_name = file.split('/')[-2]
-#       filename = "#{mount_name}-#{File.basename file}"
-#       hostname = Socket.gethostname[/^[^.]+/]
-#       command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{ hostname }-#{filename}"
-#       deb("Command: #{command}")
-#       ret = backquote_execute(command)
-#       # if $opts[:debug] do
-#       #   puts "+ Current list of files:"
-#       #   ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
-#       #   puts ret
-#       # end
-#       ret
-#     end
+    def upload_to_gcs(file, opts={})
+      deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
+      mount_name = file.split('/')[-2]
+      filename = "#{mount_name}-#{File.basename file}"
+      hostname = Socket.gethostname[/^[^.]+/]
+      command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{ hostname }-#{filename}"
+      deb("Command: #{command}")
+      puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
+      ret = backquote_execute(command, :dryrun => true)
+      # if $opts[:debug] do
+      #   puts "+ Current list of files:"
+      #   ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
+      #   puts ret
+      # end
+      ret
+    end
   

data/lib/storazzo/ric_disk_config.rb

@@ -41,7 +41,7 @@ public
             verbose = opts.fetch :verbose, false
 
             if already_loaded? # and not self.config.nil?
-                puts "[#{self.class}] load: already loaded"
+                puts "[#{self.class}] VERBOSE load: already loaded" if verbose
                 return self.config
             end
 
@@ -116,6 +116,18 @@ public
             "POLY_#{self.class}_(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
         end
 
+        def to_verbose_s
+            h = {}
+            h[:description] = "This is a Verbose Hash describing a RicDiskConfig or its child RicDiskSampleConfig to understand why it keeps failing.."
+            h[:to_s] = self.to_s 
+            h[:class] = self.class 
+            h[:file] = __FILE__ 
+            h[:id] = self.object_id
+            h[:get_bucket_paths] = self.get_bucket_paths()
+            h[:get_local_folders] = self.get_local_folders()
+            return h
+        end
+
         def get_config(opts={})
             return load(opts) if @config.nil?
             @config
@@ -160,27 +172,26 @@ public
             #         RicDisk.write_config_yaml_to_disk(dir)
             #         RicDisk.calculate_stats_files(dir) # dir is inutile
             #     } # TODO refactor in option sbrodola_afterwards=true. :)
-            # else
-                raise "Wrong input: #{files_list} " unless files_list.is_a?(Array)
-   
-                puts "iterate_through_file_list_for_disks(): I consider files_list as a list of directories to parse :)" if verbose
-
-                #dirs = RicDisk.find_active_dirs()
-                files_list.each do |dir| 
-                    dir = File.expand_path(dir)
-                    if File.directory?(dir)
-                    #if dirs.include?(dir)
-                        puts "iterate_through_file_list_for_disks() Legit dir: #{green dir}" if verbose
-                        rd = RicDisk.new(dir)
-                        rd.write_config_yaml_to_disk(dir)
-                        #RicDisk.write_config_yaml_to_disk(dir)
-                        #RicDisk.calculate_stats_files (CLASS) => will become OBJECT compute_stats_files
-                        compute_stats_files(dir) # dir is inutile # TODO
-                    else
-                        deb red("Doesnt seem a legit dir to me: #{dir}")
-                    #     deb "Figghiu ri buttana: doesnt exist #{red dir}" 
-                    end
+        # else
+            raise "Wrong input, I need an array here: #{files_list} " unless files_list.is_a?(Array)
+            puts "iterate_through_file_list_for_disks(): I consider files_list as a list of directories to parse :)" if verbose
+
+            #dirs = RicDisk.find_active_dirs()
+            files_list.each do |dir| 
+                dir = File.expand_path(dir)
+                if File.directory?(dir)
+                #if dirs.include?(dir)
+                    puts "iterate_through_file_list_for_disks() Legit dir: #{green dir}" if verbose
+                    rd = RicDisk.new(dir)
+                    puts "RicDisk: #{rd}"
+                    rd.write_config_yaml_to_disk(dir)
+                    #RicDisk.write_config_yaml_to_disk(dir)
+                    #RicDisk.calculate_stats_files (CLASS) => will become OBJECT compute_stats_files
+                    rd.compute_stats_files() # dir is inutile # TODO
+                else
+                    raise("Doesnt seem a dir to me, quitting: #{dir}")
                 end
+            end
             #end
         end #/iterate_through_file_list_for_disks
 

data/lib/storazzo/ric_disk_sample_config.rb

@@ -21,7 +21,7 @@ module Storazzo
 
         public
         def load # _sample_version
-            puts white("[RicDiskSampleConfig] Wheew 1! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
+            deb("[RicDiskSampleConfig] Wheew 1! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
 #            super.load DefaultGemLocationForTests #super.load(DefaultGemLocationForTests, :verbose => true )
             super(DefaultGemLocationForTests, :verbose => false )
         end