feldtruby 0.3.16 → 0.3.18

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1c76bb7e979470ff22539fc9543ddf008dfe34a0
+  data.tar.gz: b9215aa49df906e914bd69611fefd0eddd46ae03
+SHA512:
+  metadata.gz: ff95c9ebbca79724d25133f970d3a638d118f624eb72b6601605f697a53c1132d7b7923c670319b547d70006b6dd8f81416aba5498ec4dc75d85e6fe8c16ce63
+  data.tar.gz: 78b95306f0af7db720989945452304e4bd6977209cb3eb84d4d7faf43807c72b28776bc140be95ea1c5bcb9d9857ecffb0bb5fbf3b04f68fb162715479812496

data/Gemfile.lock

@@ -1,16 +1,23 @@
 PATH
   remote: .
   specs:
-    feldtruby (0.3.11)
+    feldtruby (0.3.18)
+      bson_ext
       json
+      mongo
       nokogiri
       rinruby
 
 GEM
   remote: https://rubygems.org/
   specs:
+    bson (1.8.4)
+    bson_ext (1.8.4)
+      bson (~> 1.8.4)
     json (1.7.7)
-    nokogiri (1.5.6)
+    mongo (1.8.4)
+      bson (~> 1.8.4)
+    nokogiri (1.5.9)
     rinruby (2.0.3)
 
 PLATFORMS

data/Rakefile

@@ -13,6 +13,14 @@ def run_tests(testFiles)
   psys "ruby -Ilib:. -e '#{require_files}' --"
 end
 
+desc "Run tests separately to identify problematic ones"
+task :test_sep do
+  Dir["test/**/test*.rb"].each do |fn|
+    puts "RUNNING: #{fn}"
+    run_tests [fn]
+  end
+end
+
 desc "Run all tests"
 task :test do
   run_tests Dir["test/**/test*.rb"]

data/feldtruby.gemspec

@@ -22,4 +22,10 @@ Gem::Specification.new do |gem|
   gem.add_dependency('json')
 
   gem.add_dependency('nokogiri')
+
+  # For mongodb_logger:
+  gem.add_dependency('mongo')
+
+  # bson_ext does not work with macruby but good for MongoDB performance so:
+  gem.add_dependency('bson_ext')
 end

data/lib/feldtruby/annotations.rb

@@ -0,0 +1,10 @@
+module FeldtRuby
+
+# A generic way to annotate Ruby objects/classes.
+module Annotateable
+  def _annotations
+    @_annotations ||= Hash.new
+  end
+end
+
+end

data/lib/feldtruby/array/basic_stats.rb

@@ -131,7 +131,9 @@ module BasicStatistics
 
 	# Return summary stats for an array of numbers.
 	def summary_stats
- 		"%.3f (min = %.1f, max = %.1f, median = %.1f, stdev = %.2f)" % [mean, self.min, self.max, median, stdev]
+		return "" if length == 0
+		vals = [mean, self.min, self.max, median, stdev]
+ 		"%.3g (min = %.3g, max = %.3g, median = %.3g, stdev = %.3g)" % vals
  	end
 end
 

data/lib/feldtruby/array/permutations_and_subsets.rb

@@ -8,4 +8,21 @@ class Array
     rest.map {|e| [x, e]} + rest.all_pairs
 
   end
+
+  # Create all combinations of values from an array of sub-arrays, with
+  # each combination picking one value from each sub-array.
+  #
+  # Examples: 
+  #  [[1,2], [3]].all_combinations_one_from_each => [[1,3], [2,3]]
+  #
+  #  [[1,2], [3, 7]].all_combinations_one_from_each => [[1,3], [2,3], [1,7], [2,7]]
+  def all_combinations_one_from_each
+    return [] if length == 0
+
+    return self.first.map {|v| [v]} if length == 1
+
+    self[1..-1].all_combinations_one_from_each.map do |c|
+      self.first.map {|v| [v] + c}
+    end.flatten(1)
+  end
 end

data/lib/feldtruby/float.rb

@@ -14,4 +14,27 @@ class Numeric
 	def ratio_diff_vs(other)
 		(self - other).protected_division_with(other)
 	end
+
+	# Change to a float with a given number of significant digits.
+	def signif(numDigits = 3)
+		self.to_f.signif(numDigits)
+	end
+
+	# Change to a float with a given number of significant digits.
+	def to_significant_digits(numDigits = 3)
+		self.to_f.to_significant_digits(numDigits)
+	end
+end
+
+class Float
+	# Change to a float with a given number of significant digits.
+	def signif(numDigits = 3)
+		return self if self == INFINITY || self == -INFINITY
+		Float("%.#{numDigits}g" % self)
+	end
+
+	# Change to a float with a given number of significant digits.
+	def to_significant_digits(numDigits = 3)
+		signif(numDigits)
+	end
 end

data/lib/feldtruby/logger.rb

@@ -1,62 +1,248 @@
-require 'stringio'
+require 'time'
+require 'feldtruby/array/basic_stats'
+require 'feldtruby/time'
+require 'feldtruby/float'
 
 module FeldtRuby
 
+# Simplest possible logger only prints to STDOUT.
 class Logger
-  def initialize(io = STDOUT)
+  DefaultParams = {
+    :verbose => true,
+    :print_frequency => 0.3  # Minimum seconds between consecutive messages printed for the same event type
+  }
 
-    @io = io
+  UnixEpoch = Time.at(0)
 
-    # We save a unique array of events for each type.
-    @events = Hash.new {|h,k| h[k] = Array.new}
+  attr_reader :start_time
 
+  def initialize(io = STDOUT, params = DefaultParams)
+
+    @start_time = Time.now
+
+    @params = DefaultParams.clone.update(params)
+
+    self.verbose = @params[:verbose]
+
+    self.print_frequency = @params[:print_frequency]
+
+    @ios = []
+
+    add_io io
+
+    setup_data_store
+
+    @last_time_printed_for_event_type = Hash.new(UnixEpoch)
+
+  end
+
+  # Set up the internal data store.
+  def setup_data_store
+    # Nothing is saved by this simplest Logger, we just count them
+    @counts = Hash.new(0)
   end
 
-  # Return the number of events of type _eventType_.
+  # Number of events of _eventType_ we have seen so far.
   def num_events eventType = nil
-    @events[eventType].length
+    if eventType == nil
+      @counts.values.sum
+    else
+      @counts[eventType]
+    end
   end
 
-  Event = Struct.new(:description, :time_stamp)
+  # Return the elapsed time since the logger was started.
+  def elapsed_time t = Time.now
+    t - @start_time
+  end
 
-  # Count an event of type _eventType_.
-  def log_event eventType = nil, description = ""
-    @events[eventType] << Event.new(description, Time.now)
+  def verbose=(flag)
+    @verbose = @params[:verbose] = flag
   end
 
-  # Return all the events, i.e. descriptions and time stamp, for a given _eventType_.
-  def events(eventType = nil)
-    @events[eventType]
+  # Set the minimum time between printing successive messages of the same type.
+  def print_frequency=(seconds = 1.0)
+    @print_frequency = @params[:print_frequency] = seconds
   end
 
-  # Return all the event descriptions for a given _eventType_.
-  def event_descriptions(eventType = nil)
-    @events[eventType].map {|e| e.description}
+  # Add one more _io_ stream to which events are logged.
+  def add_io io
+    @ios << io
+    @ios.uniq
   end
 
-  # Log an event described by a string _str_ optionally with an event type
-  # _eventType_.
-  def log str, eventType = nil
+  def add_output_file(filename)
+    @output_ios ||= []
+    @output_ios << File.open(filename, "w")
+    add_io @output_ios.last
+    ObjectSpace.define_finalizer(self) do
+      @output_ios.each {|fh| fh.close}
+    end
+  end
 
-    description = log_entry_description(str, eventType)
+  # Events:
+  #
+  # An event is a hash with the keys:
+  #  "t" => time of event in UTC
+  #  "v" => optional value of the event, this is singled out for perf reasons, 
+  #           it could also be saved in the data ("d")
+  #  "d" => optional hash with additional data for the event
+  #
+  # An event which has a value but no data is called a value event.
+  # A value event where the value is a number is called a number event.
+  # An event with data is called a data event.
+  # An event with only a message is called a message event. This is saved
+  #   as a data event of type "___default___", with the message in e["d"]["m"].
+  # An event can also be a counter event. Counter events are not logged, we just
+  #   count how many they are.
+
+  # Log a counter event, i.e. update the (local) count of how many times
+  # this event has happened.
+  def log_counter eventType, message = nil
+    if message
+      log_event eventType, nil, message
+    else
+      # We count it even if should not log it
+      @counts[eventType] += 1
+    end
+  end
 
-    log_event eventType, description
+  # Log a value event.
+  def log_value eventType, value, message = nil
+    log_event eventType, {"v" => value}, message
+  end
 
-    io_puts description
+  # Log a data event.
+  def log_data eventType, data, message = nil
+    log_event eventType, {"d" => data}, message
+  end
 
+  # Log a message event.
+  def log message
+    log_event "___default___", {"d" => {"m" => message}}, message
   end
 
-  # Puts the given _str_ on the io stream.
-  def io_puts str
-    @io.puts str
+  # Log the event and print the message, if any. This simplest logger
+  # only prints, it never saves the event.
+  def log_event eventType, event, message = nil
+
+    @counts[eventType] += 1
+
+    if message
+      print_message_if_needed message, eventType, (eventType == "___default___")
+    end
+
+    event
+
   end
 
-  # Map a string and event type to a log string.
-  def log_entry_description str, eventType = nil
+  def print_message_if_needed message, eventType, skipCheck = false
+    time = Time.now.utc
+
+    # We only print if enough time since last time we printed. This way
+    # we avoid "flooding" the user with log messages of the same type.
+    if skipCheck || (time - @last_time_printed_for_event_type[eventType]) >= @print_frequency
+
+      io_puts message, time
+
+      @last_time_printed_for_event_type[eventType] = time
+
+    end
+  end
+
+  # Puts the given _message_ on the io stream(s) stamped with the given time.
+  def io_puts message, time = Time.now
+
+    return unless @verbose
+
+    elapsed_str = Time.human_readable_timestr elapsed_time(time)
+
+    s = time.strftime("\n%H:%M.%S%3N (#{elapsed_str}), ") + message
+
+    @ios.each {|io| io.puts s}
+
+  end
+
+end
+
+# A simple logging interface front end for classes that need basic logging.
+# Just include and call log methods on logger. Uses a single common logger
+# unless a new one is been explicitly specified..
+module Logging
+  attr_accessor :logger
+
+  def setup_logger_and_distribute_to_instance_variables(logger = nil)
+
+    # Precedence for loggers if several has been setup:
+    #  1. One specified as parameter to this method
+    #  2. One that has already been set on this object
+    #  3. First one found on an instance var
+    #  4. Create a new standard one
+    self.logger = logger || self.logger || __find_logger_set_on_instance_vars() ||
+      new_default_logger()
+
+    # Now distribute the preferred logger to all instance vars, recursively.
+    self.instance_variables.each do |ivar_name|
+
+      ivar = self.instance_variable_get ivar_name
+
+      if ivar.respond_to?(:setup_logger_and_distribute_to_instance_variables)
+        ivar.setup_logger_and_distribute_to_instance_variables self.logger 
+      end
+
+    end
+
+  end
+
+  # Override to use another logger as default if no logger is found.
+  def new_default_logger
+    FeldtRuby::Logger.new
+  end
+
+  # Find a logger if one has been set on any of my instance vars or their
+  # instance vars (recursively).
+  def __find_logger_set_on_instance_vars
+
+    # First see if we find it in the immediate ivars
+    self.instance_variables.each do |ivar_name|
+
+      ivar = self.instance_variable_get ivar_name
+
+      if ivar.respond_to?(:logger)
+
+        begin
+
+          l = ivar.send(:logger)
+          return l if l.is_a?(FeldtRuby::Logger)
+
+        rescue Exception => e
+        end
+
+      end
+
+    end
+
+    # If we come here it means we did NOT find a logger in immediate
+    # ivar's. So we recurse.
+    self.instance_variables.each do |ivar_name|
+
+      ivar = self.instance_variable_get ivar_name
+
+      if ivar.respond_to?(:find_logger_set_on_instance_vars)
+
+        begin
+
+          l = ivar.send(:find_logger_set_on_instance_vars)
+          return l if l.is_a?(FeldtRuby::Logger)
+
+        rescue Exception => e
+        end
+
+      end
 
-    event_tag = eventType ? "{#{eventType.to_s}}: " : ""
+    end
 
-    event_tag + str
+    nil
 
   end
 end

data/lib/feldtruby/minitest_extensions.rb

@@ -6,7 +6,6 @@ module MiniTest::Assertions
   # Ensure that that are (statistically) the same number of each type
   # of value in an array.
   def assert_similar_proportions(values, expectedPValue = 0.01, msg = nil)
-    #pvalue = FeldtRuby.probability_of_same_proportions(values)
     pvalue = FeldtRuby.chi_squared_test(values)
     assert(pvalue > expectedPValue, msg || "Proportions differ! p-value is #{pvalue} (<0.05), counts: #{values.counts.inspect}")
   end

data/lib/feldtruby/mongodb.rb

@@ -0,0 +1,16 @@
+require 'mongo'
+require 'bson'
+
+
+module FeldtRuby
+
+def self.is_mongo_running?
+  begin
+    Mongo::MongoClient.new("localhost", 27017)
+    return true
+  rescue Exception => e
+    return false
+  end
+end
+
+end

data/lib/feldtruby/mongodb_logger.rb

@@ -0,0 +1,245 @@
+require 'feldtruby/version'
+require 'feldtruby/logger'
+require 'feldtruby/mongodb'
+
+if FeldtRuby.is_mongo_running?
+
+module FeldtRuby
+
+# This is a class to access the main directory of all MongoDBLogger db's
+# saved in a mongodb.
+class AllMongoDBLoggers
+  def initialize host, port
+    @host, @port = host, port
+
+    @client = Mongo::MongoClient.new("localhost", 27017)
+
+    @main_db = @client.db("MongoDBLoggers")
+  end
+
+  def all_logs
+    @main_db["all_logs"]
+  end
+
+  # Add a logger to the main dir of logger dbs.
+  def add_logger(logger)
+
+    db_logger_data = {
+      "db_name" => logger.db_name,
+      "logger_class" => logger.class.inspect,
+      "gem" => "FeldtRuby",
+      "gem_version" => FeldtRuby::VERSION,
+      "start_time" => logger.start_time
+    }
+
+    all_logs.insert db_logger_data
+
+  end
+
+  def all_log_infos
+    all_logs.find.to_a
+  end
+
+  def delete_logger_dbs skipLatestDb = true
+
+    infos = all_log_infos.sort_by {|i| i["start_time"]}
+
+    if skipLatestDb
+      infos = infos - [infos.last]
+    end
+
+    infos.each {|i| drop_db_named(i["db_name"])}
+
+  end
+
+  # Drop the logger db with given name.
+  def drop_db_named name
+
+    @client.drop_database name
+
+    all_logs.remove( {"db_name" => name} )
+
+  end
+end
+
+# This is an EventLogger that logs to a MongoDB database. It caches
+# the last two events per event type for quicker access to them.
+class MongoDBLogger < Logger
+  OurParams = {
+    :host => "localhost",
+    :port => 27017,
+  }
+
+  DefaultParams = FeldtRuby::Logger::DefaultParams.clone.update(OurParams)
+
+  # I think this is needed since we have redefined DefaultParams but should
+  # investigate...
+  def initialize(io = STDOUT, params = DefaultParams)
+    super
+  end
+
+  def unique_mongodb_name
+    "MongoDBLogger_" + @start_time.utc.strftime("%Y%m%d_%H%M%S") + "_#{object_id}"
+  end
+
+  # Reader methods for the db and mongo client and db name.
+  attr_reader :db, :mongo_client, :db_name
+
+  # Set up the internal data store.
+  def setup_data_store
+
+    super
+
+    # To handle the main "directory" of logger dbs in the mongodb.
+    @all_dbs = FeldtRuby::AllMongoDBLoggers.new @params[:host], @params[:port]
+
+    @mongo_client = Mongo::MongoClient.new @params[:host], @params[:port]
+
+    @db_name = unique_mongodb_name()
+
+    # Always creates a new db based on unique timestamp and object_id. The
+    # latter is used to ensure each db has a unique name.
+    @db = @mongo_client.db(@db_name)
+
+    @all_dbs.add_logger self
+
+    # Will map each event type to a collection in the db were we save
+    # events for that type.
+    @collections = Hash.new
+
+    # Caches for the last and second_last events per type so we need
+    # not lookup in db for most common requests.
+    @cache_last = Hash.new
+    @cache_2ndlast = Hash.new
+
+  end
+
+  # Reader methods for the main db and its all_logs collection. The latter is
+  # the main "directory" for listing all available logs.
+  attr_reader :main_db, :all_logs
+
+  # Events are saved in the mongodb db, using one collection per event type.
+  # The type itself is not saved in the event since it is implicit in the
+  # collection in which the event is saved.
+  def save_event event, type
+
+    collection_for_type(type).insert event
+
+    update_cache event, type
+
+  end
+
+  def update_cache event, type
+
+    @cache_2ndlast[type] = @cache_last[type]
+
+    @cache_last[type] = event
+
+  end
+
+  # Number of events of _eventType_ we have seen so far.
+  def num_events eventType
+    @counts[eventType]
+  end
+
+  # Log the event and print the message, if any.
+  def log_event eventType, event, message = nil
+
+    @counts[eventType] += 1
+
+    if event
+
+      event["t"] = Time.now.utc
+
+      save_event(event, eventType)
+
+      if message
+        print_message_if_needed message, eventType, (eventType == "___default___")
+      end
+
+    end
+
+    event
+
+  end
+
+  def collection_for_type(t)
+    ts = t || "___default___"
+    @collections[ts] ||= @db[ts.to_s]
+  end
+
+  def last_event eventType
+    @cache_last[eventType]
+  end
+
+  def prev_event eventType
+    @cache_2ndlast[eventType]
+  end
+
+  def last_value eventType
+    current_value eventType
+  end
+
+  # Return the current (latest) value for a given eventType and metric. 
+  # Return nil if no value has been set.
+  def previous_value eventType, metric = "v"
+    @cache_2ndlast[eventType][metric]
+  end
+
+  # Return the current (latest) value for a given eventType and metric. 
+  # Return nil if no value has been set.
+  def current_value eventType, metric = "v"
+    @cache_last[eventType][metric]
+  end
+
+  # Return all the events for a given _eventType_.
+  def events(eventType = nil)
+    c = collection_for_type eventType
+    c.find.to_a
+  end
+
+  # Return all events, for a given _eventType_, between the _start_ and _stop_
+  # times. If _includePreEvent_ is true we include the event that comes directly
+  # before the start time.
+  def events_between start, stop, eventType = nil, includePreEvent = false
+
+    all_events = events(eventType)
+
+    es = all_events.select do |e|
+
+      t = e.time
+
+      t >= start && t <= stop
+
+    end
+
+    if includePreEvent
+
+      index_to_first_selected_event = all_events.index(es.first)
+
+      if index_to_first_selected_event && index_to_first_selected_event > 0
+        # There is a pre-event so add it
+        es.unshift all_events[index_to_first_selected_event - 1]
+      end
+
+    end
+
+    es
+
+  end
+
+  # Get an array of values for the metric named _metric_ in events
+  # of type _eventType_.
+  def values_for_event_and_metric eventType, metric = "v"
+    events(eventType).map {|e| e.data[metric]}
+  end
+
+  # Shortcut method to get the value saved for a certain _eventType_.
+  def values_for eventType
+    values_for_event_and_metric eventType
+  end
+end
+
+end
+
+end