storage_visualizer 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88c10ac5e6c03634ed7e26436ab61cefd18de7ec
-  data.tar.gz: da9c280b9296d6886df43cc010a3cf0942317824
+  metadata.gz: dafd6c489f0809c633d222e2f2e46a1f5f1f087c
+  data.tar.gz: 6c6b72560460adb5b4c13398fa26adc9b209e85b
 SHA512:
-  metadata.gz: 4aa75292140cbf9594d6a14181608d69795440c1fbecb83b98daba489028946eba958b989b3f3caaf4eba4f26f4c452f7e0dada095e022719cfd6133c580ac7a
-  data.tar.gz: 96b1a48b87221021a8746a70d2cbee8375660ad39c0050bf5ea13097c5f54de739716082d98175080a4d7f566928f9ee51c7d3e912a9d47a07e6e99e1fa81c80
+  metadata.gz: 496b3d800a10b5331980c9acd22e2e416f27ac628f2b260a17b723edde95845df949829f10fe99b8daac803a323a134ad86729e09ff2815f16e6a0f8819e5662
+  data.tar.gz: 0fc8ac60e6df0ecd7a5566fbda1e136e6e497a77843a55b26e7997a9e81f13768d47d3dd96ac8eb819d37d73db54fb4cc2ad7576c4f5b11f8157a97765edf722

data/lib/storage_visualizer.rb

@@ -20,6 +20,30 @@
 require 'pp'
 require 'yaml'
 require 'date'
+require 'uri'
+require 'cgi'
+require 'json'
+
+
+class DirNode
+  attr_accessor :parent
+  attr_accessor :dir_name
+  attr_accessor :dir_short
+  attr_accessor :size_gb
+  attr_accessor :children
+  
+  def initialize(parent_in, dir_name_in, dir_short_in, size_gb_in)
+    self.parent = parent_in
+    self.dir_name = dir_name_in
+    self.dir_short = dir_short_in
+    self.size_gb = size_gb_in
+    self.children = []
+  end
+  
+end
+
+
+
 
 class StorageVisualizer
 
@@ -85,16 +109,15 @@ class StorageVisualizer
   
   
   # To do:
-  # x Make it work on linux
+  # x Make it work on mac & linux (CentOS & Ubuntu)
   # x Specify blocksize and do not assume 512 bytes (use the -k flag, which reports blocks as KB)
-  # - Enable for filesystems not mounted at the root '/'
+  # x Enable for filesystems not mounted at the root '/'
   # - Allow the threshold to be specified (default is 5%)
   # - Allow output filename to be specified
   # Maybe:
-  # - Prevent paths on the graph from crossing
+  # x Prevent paths on the graph from crossing (dirs with the same name become the same node)
   # - See if it would be cleaner to use the googlecharts gem (gem install googlecharts)
-  # - Create an installer that sets up cron scheduling and add polling to the webpage
-  # - What to do about directories with the same name under different parents
+
 
   # disk Bytes
   attr_accessor :capacity
@@ -110,13 +133,18 @@ class StorageVisualizer
   attr_accessor :tree_formatted
   attr_accessor :diskhash
   attr_accessor :threshold_pct
+  attr_accessor :target_volume
+  attr_accessor :dupe_counter
+
+  # this is the root DirNode object 
+  attr_accessor :dir_tree
 
   # Constructor
   def initialize(target_dir_passed = nil)
 
     if (target_dir_passed != nil)
       expanded = File.expand_path(target_dir_passed)
-      puts "Target dir: #{expanded}"
+      # puts "Target dir: #{expanded}"
       if (Dir.exist?(expanded))
         self.target_dir = expanded
       else
@@ -127,25 +155,44 @@ class StorageVisualizer
       self.target_dir = File.expand_path('~')
     end
     
-    
+    # how much space is considered worthy of noting on the chart
     self.threshold_pct = 0.05
     self.diskhash = {}
     self.tree = []
     self.tree_formatted = ''
+    self.dupe_counter = 0
   end
   
   
   
   def format_data_for_the_chart
-    working_string = "[\n"
+  
+    # Build the list of nodes
+    nodes = []
+    nodes.push(self.dir_tree)
+    comparison_list = []
+    while true
+      if (nodes.length == 0)
+        break
+      end
+      node = nodes.shift
+      comparison_list.push(node)
+      nodes.concat(node.children)
+    end
     
-    self.tree.each_with_index do |entry, index|
-      if(index == self.tree.length - 1)
+
+    # format the data for the chart
+    working_string = "[\n"
+    comparison_list.each_with_index do |entry, index|
+      if (entry.parent == nil)
+        next
+      end
+      if(index == comparison_list.length - 1)
         # this is the next to last element, it gets no comma
-        working_string << "[ '#{entry[0]}', '#{entry[1]}', #{entry[2]} ]\n"
+        working_string << "[ '#{entry.parent.dir_short}', '#{entry.dir_short}', #{entry.size_gb} ]\n"
       else
         # mind the comma
-        working_string << "[ '#{entry[0]}', '#{entry[1]}', #{entry[2]} ],\n"
+        working_string << "[ '#{entry.parent.dir_short}', '#{entry.dir_short}', #{entry.size_gb} ],\n"
       end
     end
     working_string << "]\n"
@@ -154,6 +201,7 @@ class StorageVisualizer
   end
   
   
+  
   def write_storage_report
   
     the_html = %q|<html>
@@ -204,7 +252,7 @@ class StorageVisualizer
         var data = new google.visualization.DataTable();
         data.addColumn('string', 'From');
         data.addColumn('string', 'To');
-        data.addColumn('number', 'Weight');
+        data.addColumn('number', 'Size (GB)');
         data.addRows( | + self.tree_formatted + %q|);
 
         // Set chart options
@@ -213,11 +261,12 @@ class StorageVisualizer
               width: 1000,
               sankey: {
                 iterations: 32,
-                node: { label: { fontName: 'Arial',
-                                 fontSize: 10,
-                                 color: '#871b47',
-                                 bold: false,
-                                 italic: true } } },
+                node: { label: { 
+                          fontName: 'Arial',
+                          fontSize: 10,
+                          color: '#871b47',
+                          bold: false,
+                          italic: true } } }, 
             };
             
             
@@ -246,7 +295,7 @@ class StorageVisualizer
     # df -l gets info about locally-mounted filesystems
     output = `df -lk`
     
-    # OSX
+    # OSX:
     #   Filesystem                          1024-blocks  Used      Available   Capacity   iused     ifree       %iused  Mounted on
     #   /dev/disk1                          975912960    349150592 626506368   36%        87351646  156626592   36%     /
     #   localhost:/QwnJE6UBvlR1EvqouX6gMM   975912960    975912960         0   100%        0         0          100%    /Volumes/MobileBackups
@@ -275,8 +324,10 @@ class StorageVisualizer
     #   {"capacity"=>498876809216, "used"=>498876809216, "available"=>0}
     # }
 
+    # get each mount's capacity & utilization
     output.lines.each_with_index do |line, index|
       if (index == 0)
+        # skip the header line
         next
       end
       cols = line.split
@@ -304,45 +355,62 @@ class StorageVisualizer
 
     # puts "Disk mount info:"
     # pp diskhash
-    self.capacity = self.diskhash['/']['capacity']
-    self.used = self.diskhash['/']['used']
-    self.available = self.diskhash['/']['available']
 
 
+    # find the (self.)target_volume 
+    # look through diskhash keys, to find the one that most matches target_dir
+    val_of_min = 1000
+    # puts "Determining which volume contains the target directory.."
+    self.diskhash.keys.each do |volume|
+      result = self.target_dir.gsub(volume, '')
+      diskhash['match_amt'] = result.length
+      # puts "Considering:\t#{volume}, \t closeness: #{result.length}, \t (#{result})"
+      if (result.length < val_of_min)
+        # puts "Candidate: #{volume}"
+        val_of_min = result.length
+        self.target_volume = volume
+      end
+    end    
+    
+    puts "Target volume is #{self.target_volume}"
+    
 
-    free_space = (self.available).to_i
-    free_space_gb = "#{'%.0f' % (free_space / 1024 / 1024)}"
-    free_space_array = ['/', 'Free Space', free_space_gb]
-    self.tree.push(free_space_array)
+    self.capacity   = self.diskhash[self.target_volume]['capacity']
+    self.used       = self.diskhash[self.target_volume]['used']
+    self.available  = self.diskhash[self.target_volume]['available']
 
     self.capacity_gb  = "#{'%.0f' % (self.capacity.to_i / 1024 / 1024)}"
     self.used_gb      = "#{'%.0f' % (self.used.to_i / 1024 / 1024)}"
     self.available_gb = "#{'%.0f' % (self.available.to_i / 1024 / 1024)}"
 
-
+    self.dir_tree = DirNode.new(nil, self.target_volume, self.target_volume, self.capacity)
+    self.dir_tree.children.push(DirNode.new(self.dir_tree, 'Free Space', 'Free Space', self.available_gb))
 
   end
   
   
-  def analyze_dirs(dir_to_analyze)
+  # Crawl the dirs recursively, beginning with the target dir
+  def analyze_dirs(dir_to_analyze, parent)
 
-    # bootstrap case
-    if (dir_to_analyze == '/')
 
-      # run on all child dirs
+    # bootstrap case
+    # don't create an entry for the root because there's nothing to link to yet, scan the subdirs
+    if (dir_to_analyze == self.target_volume)
+      # puts "Dir to analyze is the target volume"
+      # run on all child dirs, not this dir
       Dir.entries(dir_to_analyze).reject {|d| d.start_with?('.')}.each do |name|
         # puts "\tentry: >#{file}<"
         full_path = File.join(dir_to_analyze, name)
-        if (Dir.exist?(full_path))
+        if (Dir.exist?(full_path) && !File.symlink?(full_path))
           # puts "Contender: >#{full_path}<"
-          analyze_dirs(full_path)
+          analyze_dirs(full_path, self.dir_tree)
         end
       end
       return
     end
 
-
-    cmd = "du -sxk \"#{dir_to_analyze}\""
+    # use "P" to help prevent following any symlinks
+    cmd = "du -sxkP \"#{dir_to_analyze}\""
     puts "\trunning #{cmd}"
     output = `#{cmd}`.strip().split("\t")
     # puts "Du output:"
@@ -350,69 +418,108 @@ class StorageVisualizer
     size = output[0].to_i 
     size_gb = "#{'%.0f' % (size.to_f / 1024 / 1024)}"
     # puts "Size: #{size}\nCapacity: #{self.diskhash['/']['capacity']}"
-    
-    occupancy = (size.to_f / self.capacity.to_f)
+
+    # Occupancy as a fraction of total space
+    # occupancy = (size.to_f / self.capacity.to_f)
+
+    # Occupancy as a fraction of USED space
+    occupancy = (size.to_f / self.used.to_f)
+
     occupancy_pct = "#{'%.0f' % (occupancy * 100)}"
-    
     capacity_gb = "#{'%.0f' % (self.capacity.to_f / 1024 / 1024)}"
     
     # if this dir contains more than 5% of disk space, add it to the tree
+
+    if (dir_to_analyze == self.target_dir)
+      # puts "Dir to analyze is the target dir, space used outside this dir.."
+      # account for space used outside of target dir
+      other_space = self.used - size
+      other_space_gb = "#{'%.0f' % (other_space / 1024 / 1024)}"
+      parent.children.push(DirNode.new(parent, self.target_volume, self.target_volume, other_space_gb))
+    end
     
     
     if (occupancy > self.threshold_pct)
-      puts "Dir contains more than 5% of disk space: #{dir_to_analyze} \n\tsize:\t#{size_gb} / \ncapacity:\t#{capacity_gb} = #{occupancy_pct}%"
-      # push this dir's info
-      
-      if (dir_to_analyze == self.target_dir)
-        
-        other_space = self.used - size
-        other_space_gb = "#{'%.0f' % (other_space / 1024 / 1024)}"
-        other_space_array = ['/', 'Other', other_space_gb]
-
-        short_target_dir = self.target_dir.split('/').reverse[0]
-        short_target_dir = (short_target_dir == nil) ? self.target_dir : short_target_dir
-
-        comparison = ['/', short_target_dir, size_gb]
-        
-        # add them to the array
-        self.tree.push(other_space_array)
-        self.tree.push(comparison)
+      # puts "Dir contains more than 5% of disk space: #{dir_to_analyze} \n\tsize:\t#{size_gb} / \ncapacity:\t#{capacity_gb} = #{occupancy_pct}%"
+      puts "Dir contains more than 5% of used disk space: #{dir_to_analyze} \n\tsize:\t\t#{size_gb} / \n\toccupancy:\t#{self.used_gb} = #{occupancy_pct}% of used space"
 
+      # puts "Dir to analyze (#{dir_to_analyze}) is not the target dir (#{self.target_dir})"
+      dirs = dir_to_analyze.split('/')
+      
+      short_dir = dirs.pop().gsub("'","\\\\'")
+      full_parent = dirs.join('/')
+      if (dir_to_analyze == self.target_dir || full_parent == self.target_volume)
+        # puts "Either this dir is the target dir, or the parent is the target volume, make parent the full target volume"
+        short_parent = self.target_volume.gsub("'","\\\\'")
       else
-        # get parent dir and add to the tree
-        short_parent = dir_to_analyze.split('/').reverse[1]
-
-        # short_parent = (short_parent == nil) ? parent : short_parent
-        # case for when parent is '/'
-        short_parent = (short_parent == '') ? '/' : short_parent
-        
-        short_dir = dir_to_analyze.split('/').reverse[0]
-        
-        # array_to_push = [parent, dir_to_analyze, size_gb]
-        array_to_push = [short_parent, short_dir, size_gb]
-        self.tree.push(array_to_push)
+        # puts "Neither this dir or parent is the target dir, making parent short"
+        short_parent = dirs.pop().gsub("'","\\\\'")
       end
+      
+
+      this_node = DirNode.new(parent, dir_to_analyze, short_dir, size_gb)
+      parent.children.push(this_node)
 
       # run on all child dirs
       Dir.entries(dir_to_analyze).reject {|d| d.start_with?('.')}.each do |name|
-        # puts "\tentry: >#{file}<"
-        
         full_path = File.join(dir_to_analyze, name)
-        
-        if (Dir.exist?(full_path))
+        # don't follow any symlinks
+        if (Dir.exist?(full_path) && !File.symlink?(full_path))
           # puts "Contender: >#{full_path}<"
-          analyze_dirs(full_path)
+          analyze_dirs(full_path, this_node)
         end
       end
       
-    end
+    end # occupancy > threshold
+      
+  end # function
+  
+
+  
+  def traverse_tree_and_remove_duplicates
+    puts "\nHandling duplicate entries.."
+    nodes = []
+    nodes.push(self.dir_tree)
+    comparison_list = []
+    while true
+      if (nodes.length == 0)
+        break
+      end
       
+      node = nodes.shift
+      comparison_list.push(node)
+      # pp node
+      if node.parent == nil
+        # puts "\tparent: no parent \n\tdir:    #{node.dir_name} \n\tshort:  #{node.dir_short} \n\tsize:   #{node.size_gb}"
+      else 
+        # puts "\tparent: #{node.parent.dir_short.to_s} \n\tdir:    #{node.dir_name} \n\tshort:  #{node.dir_short} \n\tsize:   #{node.size_gb}"
+      end
+      nodes.concat(node.children)
+    end
+    # puts "Done building node list"
+    
+    
+    
+    for i in 0..comparison_list.length do
+      for j in 0..comparison_list.length do
+        if (comparison_list[i] != nil && comparison_list[j] != nil)
+          if (i != j && comparison_list[i].dir_short == comparison_list[j].dir_short)
+            puts "\t#{comparison_list[i].dir_short} is the same as #{comparison_list[j].dir_short}, changing to #{comparison_list[j].dir_short}*"
+            comparison_list[j].dir_short = "#{comparison_list[j].dir_short}*"
+          end
+        end
+      end
+    end
+    puts "Duplicate handling complete"
+    
   end
   
+
   
   def run
     self.get_basic_disk_info
-    self.analyze_dirs(self.target_dir)
+    self.analyze_dirs(self.target_dir, self.dir_tree)
+    self.traverse_tree_and_remove_duplicates
     self.format_data_for_the_chart
     self.write_storage_report
     
@@ -438,12 +545,12 @@ def run
     vs = StorageVisualizer.new()
   end
 
-  puts "\nRunning visualization"
+  # puts "\nRunning visualization"
   vs.run()
   
   # puts "dumping tree: "
   # pp vs.tree
-  puts "Formatted tree\n#{vs.tree_formatted}"
+  # puts "Formatted tree\n#{vs.tree_formatted}"
   
 end
 

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: storage_visualizer
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Terry Case
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-07 00:00:00.000000000 Z
+date: 2015-10-05 00:00:00.000000000 Z
 dependencies: []
 description: This tool helps visualize which directories are occupying the most storage.
-  Any directory that occupies more than 5% of disk space is added to a visual hierarchichal
-  storage report in the form of a Google Sankey diagram. The storage data is gathered
-  using the linux `du` utility. It has been tested on Mac OSX, should work on linux
-  systems, will not work on Windows. Run as sudo if analyzing otherwise inaccessible
-  directories. May take a while to run.
+  Any directory that occupies more than 5% of used disk space is added to a visual
+  hierarchichal storage report in the form of a Google Sankey diagram. The storage
+  data is gathered using the linux `du` utility. It has been tested on Mac OSX and
+  linux systems (Ubuntu & CentOS), will not work on Windows. Run as sudo if analyzing
+  otherwise inaccessible directories. May take a while to run.
 email: terrylcase@gmail.com
 executables: []
 extensions: []