scout-essentials 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 635a27a40f5897e83e1632e43a1cfa9343edd5c53e7eee522fc99f4a74e03171
-  data.tar.gz: 19bb00a9be1cc0ab21661cedc46bfd1d21118cd0fed9f0d240b205580e8a733a
+  metadata.gz: 01ebe0cbbfda369f1d1b3de5009566eed0749cdb0a418c19c663a21ec68214c1
+  data.tar.gz: 2303029301ea7b4ef1f9d7cf44bd906cb8db637b31a5147ae2950e4eb1c6e37e
 SHA512:
-  metadata.gz: 11813ec485609e4275258e12aee416663bb70e99c453dc9e7ec2363a864370c07506c74c38b20ac43426a12e7872456eb37e9e368fc10e3f2d3a5b1083d12281
-  data.tar.gz: da326b8fa64b0850fd2d4e50ab1e0e13d4a881039b4945503a56debcd800a8c5850b5b09f6f63d2631746ae177dcce16ce235ae9deb803b095e8b7d7e588c0a0
+  metadata.gz: 707278b8bdd09658049035622021bd822d8b53ecef993f4136b550f2acb47be30cf515a460b07f5c45ed51769d782639b06e128947dab80832a4de3b6b66f574
+  data.tar.gz: d8a5ce01ca753d159734871447f961a39d48992af934f7ad69a0f95ff708794d00962cd36fbabbcd535a721611f5d0fa9194f87d5662cd8ab66ed80d8abeed38

data/.vimproject

@@ -5,7 +5,12 @@ scout-essentials=/$PWD filter="*.rb *.txt *.md *.conf *.yaml" {
   scout-essentials.rb
   scout=scout{
    exceptions.rb
-   meta_extension.rb
+   annotation.rb
+   annotation=annotation{
+    array.rb
+    annotated_object.rb
+    annotation_module.rb
+   }
    misc.rb
    misc=misc{
     digest.rb
@@ -15,6 +20,7 @@ scout-essentials=/$PWD filter="*.rb *.txt *.md *.conf *.yaml" {
     insist.rb
     monitor.rb
     system.rb
+    math.rb
    }
    named_array.rb
    indiferent_hash.rb
@@ -74,5 +80,31 @@ scout-essentials=/$PWD filter="*.rb *.txt *.md *.conf *.yaml" {
  }
  test=test{
   test_helper.rb
+  scout=scout{
+   test_annotation.rb
+   annotation=annotation{
+    test_annotated_object.rb
+    test_array.rb
+    }
+   test_cmd.rb
+   test_concurrent_stream.rb
+   test_config.rb
+   test_indiferent_hash.rb
+   test_log.rb
+   test_misc.rb
+   test_named_array.rb
+   test_open.rb
+   test_path.rb
+   test_persist.rb
+   test_resource.rb
+   test_tmpfile.rb
+   misc=misc{
+    test_digest.rb
+    test_filesystem.rb
+    test_helper.rb
+    test_insist.rb
+    test_system.rb
+    }
+   }
  }
 }

data/VERSION

@@ -1 +1 @@
-1.3.0
+1.4.0

data/lib/scout/annotation/annotated_object.rb

@@ -0,0 +1,69 @@
+module Annotation
+  module AnnotatedObject
+    def annotations
+      @annotations ||= []
+    end
+
+    def annotation_types
+      @annotation_types ||= []
+    end
+
+    def annotation_hash
+      attr_hash = {}
+      @annotations.each do |name|
+        attr_hash[name] = self.instance_variable_get("@#{name}")
+      end if @annotations
+      attr_hash
+    end
+
+    def annotation_info
+      annotation_hash.merge(annotation_types: annotation_types, annotated_array: (AnnotatedArray === self))
+    end
+
+    def self.serialize(obj)
+      Annotation.purge(obj.annotation_info.merge(literal: obj))
+    end
+
+    def serialize
+      AnnotatedObject.serialize(self)
+    end
+
+    def annotation_id
+      Misc.digest([self, annotation_info])
+    end
+
+    alias id annotation_id
+
+    def annotate(other)
+      annotation_types.each do |type|
+        type.setup(other, annotation_hash)
+      end
+      other
+    end
+
+    def purge
+      new = self.dup
+
+      if new.instance_variables.include?(:@annotations)
+        new.instance_variable_get(:@annotations).each do |a|
+          var_name = "@#{a}".to_sym
+          new.remove_instance_variable(var_name) if new.instance_variables.include? var_name
+        end
+        new.remove_instance_variable(:@annotations)
+      end
+
+      if new.instance_variables.include?(:@annotation_types)
+        new.remove_instance_variable(:@annotation_types)
+      end
+
+      new
+    end
+
+    def make_array
+      new = [self]
+      self.annotate(new)
+      new.extend AnnotatedArray
+      new
+    end
+  end
+end

data/lib/scout/annotation/annotation_module.rb

@@ -0,0 +1,59 @@
+module Annotation
+  module AnnotationModule
+    def annotation(*attrs)
+      self.instance_variable_get("@annotations").concat attrs
+      attrs.each do |a|
+        self.attr_accessor a
+      end
+    end
+
+    def annotations
+      @annotations ||= []
+    end
+
+    def included(mod)
+      mod.instance_variable_set(:@annotations, []) unless mod.instance_variables.include?(:@annotations)
+      mod.instance_variable_get(:@annotations).concat self.instance_variable_get(:@annotations)
+    end
+
+    def extended(obj)
+      attrs = self.instance_variable_get("@annotations")
+
+      obj.instance_variable_set(:@annotations, []) unless obj.instance_variables.include?(:@annotations)
+      obj.annotation_types << self
+
+      annotations = obj.instance_variable_get(:@annotations)
+      annotations.concat attrs
+    end
+
+    def setup(*args,&block)
+      if block_given?
+        obj, rest = block, args
+      else
+        obj, *rest = args
+      end
+      obj = block if obj.nil?
+      return nil if obj.nil?
+      obj.extend self unless self === obj
+      attrs = self.instance_variable_get("@annotations")
+
+      return obj if attrs.nil? || attrs.empty?
+
+      if rest.length == 1 && Hash === (rlast = rest.last) && 
+          ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
+           (! attrs.length != 1 ))
+
+        pairs = rlast
+      else
+        pairs = attrs.zip(rest)
+      end
+
+      pairs.each do |name,value|
+        next if name.to_sym === :annotation_types
+        obj.instance_variable_set("@#{name}", value) unless value.nil?
+      end
+
+      obj
+    end
+  end
+end

data/lib/scout/annotation/array.rb

@@ -0,0 +1,74 @@
+module AnnotatedArray
+
+  module AnnotatedArrayItem
+    attr_accessor :container, :container_index
+  end
+  
+  def self.is_contained?(obj)
+    AnnotatedArrayItem === obj
+  end
+
+  def annotate_item(obj, position = nil)
+    obj = obj.dup if obj.frozen?
+    obj.extend AnnotatedArray if Array === obj
+    obj.extend AnnotatedArrayItem
+    obj.container = self
+    obj.container_index = position
+    self.annotate(obj)
+  end
+
+  def [](pos, clean = false)
+    item = super(pos)
+    return item if item.nil? or clean
+    annotate_item(item, pos)
+  end
+
+  def first
+    annotate_item(super, 0)
+  end
+
+  def last
+    annotate_item(super, self.length - 1)
+  end
+
+  def each_with_index(&block)
+    super do |item,i|
+      block.call annotate_item(item, i)
+    end
+  end
+
+  def each(&block)
+    i = 0
+    super do |item|
+      block.call annotate_item(item, i)
+      i += 1
+    end
+  end
+
+  def inject(acc, &block)
+    each do |item|
+      acc = block.call acc, item
+    end
+    acc
+  end
+
+  def collect(&block)
+    if block_given?
+      inject([]){|acc,item| acc.push(block.call(item)); acc }
+    else
+      inject([]){|acc,item| acc.push(item); acc }
+    end
+  end
+
+  %w(compact uniq flatten reverse sort_by).each do |method|
+
+    self.define_method(method) do |*args|
+      res = super(*args)
+
+      annotate(res)
+      res.extend AnnotatedArray
+
+      res
+    end
+  end
+end

data/lib/scout/annotation.rb

@@ -0,0 +1,45 @@
+require_relative 'annotation/array'
+require_relative 'annotation/annotated_object'
+require_relative 'annotation/annotation_module'
+
+module Annotation
+
+  def self.setup(obj, annotation_types, annotation_hash)
+    return nil if obj.nil?
+    annotation_types = annotation_types.split("|") if String === annotation_types
+    annotation_types = [annotation_types] unless Array === annotation_types
+    annotation_types.each do |type|
+      type = Kernel.const_get(type) if String === type
+      type.setup(obj, annotation_hash)
+    end
+    obj
+  end
+
+  def self.extended(base)
+    base.instance_variable_set(:@annotations, []) unless base.instance_variables.include?(:@annotations)
+    base.include Annotation::AnnotatedObject
+    base.extend Annotation::AnnotationModule
+  end
+
+  def self.is_annotated?(obj)
+    obj.instance_variables.include?(:@annotation_types)
+  end
+
+  def self.purge(obj)
+    case obj
+    when nil
+      nil
+    when Array
+      obj = obj.purge if is_annotated?(obj)
+      obj.collect{|e| purge(e) }
+    when Hash
+      new = {}
+      obj.each do |k,v|
+        new[purge(k)] = purge(v)
+      end
+      new
+    else
+      is_annotated?(obj) ? obj.purge : obj
+    end
+  end
+end

data/lib/scout/concurrent_stream.rb

@@ -223,7 +223,10 @@ module ConcurrentStream
         self.join if ! @stream_exception && (self.closed? || self.eof?)
       end
     else
-      super(*args)
+      begin
+        super(*args)
+      rescue IOError
+      end unless self.closed?
     end
   end
 

data/lib/scout/config.rb

@@ -17,16 +17,16 @@ module Scout::Config
 
   def self.load_file(file)
     Log.debug "Loading config file: #{ file }"
-    TSV.traverse file, :type => :array do |line|
+    Open.read(file).split("\n").each do |line|
       next if line =~ /^#/
-      key, value, *tokens = line.strip.split(/\s/)
+      key, value, *tokens = line.strip.split(/\s+/)
 
       self.add_entry(key, value, tokens) if key
     end
   end
 
   def self.load_config
-    Scout.etc.config.find_all.reverse.each do |file|
+    Path.setup("etc").config.find_all.reverse.each do |file|
       self.load_file(file)
     end
   end

data/lib/scout/exceptions.rb

@@ -54,8 +54,9 @@ class DontClose < Exception
   end
 end
 
+class DontPersist < Exception; end
 
-class KeepLocked < Exception
+class KeepLocked < DontPersist
   attr_accessor :payload
   def initialize(payload)
     @payload = payload

data/lib/scout/indiferent_hash/options.rb

@@ -28,7 +28,8 @@ module IndiferentHash
   end
 
   def self.pull_keys(hash, prefix)
-    new = {}
+    IndiferentHash.setup(hash)
+    new = hash.include?("#{prefix}_options") ? hash.delete("#{prefix}_options") : {}
     prefix = prefix.to_s
     hash.keys.each do |key|
       if key.to_s =~ /#{ prefix }_(.*)/

data/lib/scout/log/color.rb

@@ -203,7 +203,7 @@ module Log
     str = str.to_s unless str.nil?
     return str if Symbol === color
     color_str = reset ? Term::ANSIColor.reset : ""
-    color_str << color
+    color_str << color if color
     if str.nil?
       color_str
     else

data/lib/scout/log.rb

@@ -109,20 +109,19 @@ module Log
 
   def self.logfile(file=nil)
     if file.nil?
-      @logfile ||= nil
+      @@logfile = nil
     else
       case file
       when String
-        @logfile = File.open(file, :mode => 'a')
-        @logfile.sync = true
+        @@logfile = File.open(file, :mode => 'a')
+        @@logfile.sync = true
       when IO, File
-        @logfile = file
+        @@logfile = file
       else
         raise "Unkown logfile format: #{file.inspect}"
       end
     end
   end
-
   def self.up_lines(num = 1)
     nocolor ? "" : "\033[#{num+1}F\033[2K"
   end
@@ -142,26 +141,26 @@ module Log
   MUTEX = Mutex.new
   def self.log_write(str)
     MUTEX.synchronize do
-      if logfile.nil?
+      if defined?(@@logfile) && @@logfile
+        @@logfile.write str
+      else
         begin
           STDERR.write str
         rescue
         end
-      else
-        logfile.write str
       end
     end
   end
 
   def self.log_puts(str)
     MUTEX.synchronize do
-      if logfile.nil?
+      if defined?(@@logfile) && @@logfile
+        @@logfile.puts str
+      else
         begin
           STDERR.puts str
         rescue
         end
-      else
-        logfile.puts str
       end
     end
   end
@@ -309,6 +308,7 @@ module Log
         end
       end
     end
+    Log::LAST.replace "log"
   end
 
   def self.stack(stack)

data/lib/scout/misc/digest.rb

@@ -5,14 +5,24 @@ module Misc
       obj.digest_str
     else
       case obj
-      when String
-        #'\'' << obj << '\''
-        if Path === obj || ! Open.exists?(obj)
-          '\'' << obj << '\''
-        elsif File.directory?(obj)
+      when Path
+        case
+        when File.directory?(obj)
           "Directory MD5: #{digest_str(Dir.glob(File.join(obj, "*")))}"
+        when obj.located? && File.exist?(obj)
+          "File MD5: #{Misc.digest_file(obj)}"
         else
-          "File MD5: #{Misc.file_md5(obj)}"
+          '\'' << obj << '\''
+        end
+      when String
+        if Path.is_filename?(obj) && Open.exists?(obj)
+          if File.directory?(obj)
+            "Directory MD5: #{digest_str(Dir.glob(File.join(obj, "*")))}"
+          else
+            "File MD5: #{Misc.digest_file(obj)}"
+          end
+        else
+          obj.dup
         end
       when Integer, Symbol
         obj.to_s
@@ -57,8 +67,29 @@ module Misc
   def self.file_md5(file)
     file = file.find if Path === file
     file = File.expand_path(file)
-    Persist.persist("MD5:#{file}", :string) do
-      Digest::MD5.file(file).hexdigest
+    Digest::MD5.file(file).hexdigest
+  end
+
+  def self.fast_file_md5(file, sample = 3_000_000)
+    size = File.size(file)
+    sample_txt = size.to_s << ":" 
+    File.open(file) do |f|
+      sample_txt << f.read(sample)
+      f.seek(size/2)
+      sample_txt << f.read(sample)
+      f.seek(size - sample - 1)
+      sample_txt << f.read(sample)
+    end
+    Digest::MD5.hexdigest(sample_txt)
+  end
+
+  def self.digest_file(file)
+    file = file.find if Path === file
+    file = File.expand_path(file)
+    if File.size(file) > 10_000_000
+      fast_file_md5(file)
+    else
+      file_md5(file)
     end
   end
 end

data/lib/scout/misc/filesystem.rb

@@ -22,4 +22,27 @@ module Misc
       return nil
     end
   end
+
+  def self.tarize(path, dest = nil)
+    Misc.in_dir(path) do
+      if dest
+        CMD.cmd("tar cvfz '#{dest}' '.'")
+      else
+        CMD.cmd("tar cvfz - '.'", :pipe => true)
+      end
+    end
+  end
+
+  def self.untar(file, target = '.')
+    target = target.find if Path === target
+    file = file.find if Path === file
+    Misc.in_dir target do
+      if IO === file
+        CMD.cmd("tar xvfz -", in: file)
+      else
+        CMD.cmd("tar xvfz '#{file}'")
+      end
+    end
+  end
+
 end

data/lib/scout/misc/format.rb

@@ -252,4 +252,36 @@ module Misc
     end
     values
   end
+
+  def self.timespan(str, default = "s")
+
+    return - timespan(str[1..-1], default) if str[0] == "-"
+
+    if str.include?(":")
+      seconds, minutes, hours = str.split(":").reverse
+      return seconds.to_i + minutes.to_i * 60 + hours.to_i * 60 * 60
+    end
+
+    tokens = {
+      "s" => (1),
+      "sec" => (1),
+      "m" => (60),
+      "min" => (60),
+      "''" => (1),
+      "'" => (60),
+      "h" => (60 * 60),
+      "d" => (60 * 60 * 24),
+      "w" => (60 * 60 * 24 * 7),
+      "mo" => (60 * 60 * 24 * 31),
+      "y" => (60 * 60 * 24 * 365),
+    }
+
+    tokens[nil] = tokens[default]
+    tokens[""] = tokens[default]
+    time = 0
+    str.scan(/(\d+)(\w*)/).each do |amount, measure|
+      time += amount.to_i * tokens[measure]
+    end
+    time
+  end
 end

data/lib/scout/misc/helper.rb

@@ -28,4 +28,41 @@ module Misc
     counts
   end
 
+  def self.chunk(array, num)
+    total = array.length
+    current = 0
+    while current < total
+      last = current + num - 1
+      yield array[current..last]
+      current = last + 1
+    end
+  end
+
+  # Divides the array into +num+ chunks of the same size by placing one
+  # element in each chunk iteratively.
+  def self.divide(array, num)
+    num = 1 if num == 0
+    chunks = []
+    num.to_i.times do chunks << [] end
+    array.each_with_index{|e, i|
+      c = i % num
+      chunks[c] << e
+    }
+    chunks
+  end
+
+  # Divides the array into chunks of +num+ same size by placing one
+  # element in each chunk iteratively.
+  def self.ordered_divide(array, num)
+    last = array.length - 1
+    chunks = []
+    current = 0
+    while current <= last
+      next_current = [last, current + num - 1].min
+      chunks << array[current..next_current]
+      current = next_current + 1
+    end
+    chunks
+  end
+
 end