dohruby 0.2.1 → 0.3

data/lib/doh/data/human.rb

@@ -3,124 +3,132 @@ require 'doh/data/bulk'
 
 module DohData
 
-def self.random_first_name
+def random_first_name
   random_element(@@first_names)
 end
 
-def self.random_last_name
+def random_last_name
   random_element(@@last_names)
 end
 
-def self.random_generation
+def random_generation
   random_element(@@generation_options)
 end
 
-def self.random_us_state_abbreviation
+def random_us_state_abbreviation
   random_element(@@us_state_abbreviations)
 end
 
-def self.random_us_state_long_name
+def random_us_state_long_name
   random_element(@@us_state_long_names)
 end
 
-def self.random_occupation
+def random_occupation
   random_element(@@occupations)
 end
 
-def self.random_us_state
-  (rand(2) == 0) ? DohData::random_us_state_abbreviation : DohData::random_us_state_long_name
+def random_us_state
+  (random_int(2) == 0) ? DohData::random_us_state_abbreviation : DohData::random_us_state_long_name
 end
 
-def self.random_us_zip
-  random_digits(5) + ((rand(4) == 0) ? '-' + random_digits(4) : '')
+def random_us_zip
+  random_digits(5) + ((random_int(4) == 0) ? '-' + random_digits(4) : '')
 end
 
-def self.random_us_phone
+def random_us_phone
   random_range(200,999).to_s + '-' + random_digits(3) + '-' + random_digits(4)
 end
 
-def self.random_full_name
+def random_full_name
   random_first_name + ' ' + random_last_name
 end
 
-def self.random_email(first_name = nil, last_name = nil)
+def random_email(first_name = nil, last_name = nil)
+  domain = '@' + random_element(@@email_domains) + '.' + random_element(@@email_tlds)
+  max_name_size = 50 - domain.size
+
   if first_name || last_name
     name = ''
-    name += first_name.slice(0, random_element(1, 1, 1, 1, first_name.size)) if first_name
-    name += random_element('', '.') if first_name && last_name
-    name += last_name.slice(0, random_element(1, last_name.size, last_name.size)) if last_name
-    name += random_range(11, 358).to_s
+    name += first_name.slice(0, random_range(1, first_name.size)) if first_name
+    name += random_element('', '.', '_') if first_name && last_name
+    name += last_name.slice(0, random_range(1, last_name.size)) if last_name
+    max_num_size = max_name_size - name.size
+    name += random_digits(random_range([7, max_num_size].min, [12, max_num_size].min))
   else
-    name = random_letters(random_range(3, 12))
+    name_size = random_range([12, max_name_size].min, [16, max_name_size].min)
+    name = random_letters(name_size)
   end
-  name + '@' + random_element(@@email_domains) + '.' + random_element(@@email_tlds)
+  name + domain
+end
+
+def ssn_areas
+  @@ssn_areas ||= @@ssn_area_highgroups.keys
 end
 
-def self.random_ssn
-  random_row = random_array_element(@@ssn_highgroups)
-  highgroup = random_row[0]
-  area = random_element(random_row[1])
+def random_ssn
+  area = random_element(ssn_areas)
+  highgroup = @@ssn_area_highgroups[area]
   highgroup_index = @@ssn_group_order.index(highgroup)
   possible_groups = @@ssn_group_order.slice(0..highgroup_index)
   group = random_element(possible_groups)
   area.to_s.rjust(3, '0') + group.to_s.rjust(2, '0') + random_range(1, 9999).to_s.rjust(4, '0')
 end
 
-def self.random_city
+def random_city
   random_element(@@cities)
 end
 
-def self.random_street_name
+def random_street_name
   random_element(@@street_names)
 end
 
-def self.random_street_type
+def random_street_type
   random_element(@@street_types)
 end
 
-def self.random_street_number
-  weight = rand(10)
+def random_street_number
+  weight = random_int(10)
   if weight > 8
-    rand(50000)
+    random_int(50000)
   elsif weight > 5
-    rand(10000)
+    random_int(10000)
   elsif weight > 3
-    rand(1000)
+    random_int(1000)
   else
-    rand(100)
+    random_int(100)
   end
 end
 
-def self.random_street
+def random_street
   random_street_number.to_s + ' ' + random_element('N','E','S','W') + ' ' + random_street_name + ' ' + random_street_type
 end
 
-def self.random_birthday
-  Date.new(1928 + rand(56), rand(12) + 1, rand(27) + 1)
+def random_birthday
+  Date.new(1928 + random_int(56), random_int(12) + 1, random_int(27) + 1)
 end
 
-def self.random_bank_routing_number
+def random_bank_routing_number
   random_element(@@bank_routing_numbers)
 end
 
-def self.random_bank_account_number(max_size = 17)
+def random_bank_account_number(max_size = 17)
   random_digits(random_range(5, max_size))
 end
 
-def self.random_employer_name
+def random_employer_name
   random_element(@@employers)
 end
 
-def self.random_relation
+def random_relation
   random_element(['cousin', 'mother', 'father', 'aunt', 'uncle', 'brother', 'sister', 'son', 'daughter'])
 end
 
-def self.random_country_code
+def random_country_code
   random_element(['USA', 'MEX', 'CAN'])
 end
 
-def self.random_ip_address
-  "#{rand(256)}.#{rand(256)}.#{rand(256)}.#{rand(256)}"
+def random_ip_address
+  "#{random_int(256)}.#{random_int(256)}.#{random_int(256)}.#{random_int(256)}"
 end
 
 end

data/lib/doh/data/make_global.rb

@@ -0,0 +1,3 @@
+class Object
+  include DohData
+end

data/lib/doh/data/require_datagen.rb

@@ -1,7 +1,8 @@
 module DohData
 
 def self.require_datagen
-  require('datagen') if File.exist?(File.join(DohApp::home, 'lib/datagen.rb'))
+  lib_datagen_file = File.join(DohApp::home, 'lib/datagen.rb')
+  require(lib_datagen_file) if File.exist?(lib_datagen_file)
 end
 
 end

data/lib/doh/home.rb

@@ -0,0 +1,16 @@
+module Doh
+  def self.home
+    @@home ||= find_file_in_parents(File.dirname(__FILE__), 'dohapp_home')
+  end
+
+  def self.find_file_in_parents(dir, filename, max_tries = 20)
+    path = File.expand_path(dir)
+    raise "unable to find #{filename.inspect}" if (path == '/') || (max_tries <= 0)
+
+    if File.exist?(File.join(path, filename))
+      return path
+    else
+      return find_file_in_parents(File.join(path, '..'), filename, max_tries - 1)
+    end
+  end
+end

data/lib/doh/logger/email_acceptor.rb

@@ -1,13 +1,14 @@
 require 'net/smtp'
+require 'socket'
 
 module DohLogger
 
 def self.exception_email_format
-  "%severity: %msg\nlogfile_name: #@logfile_name\nsource_ip: %source_ip\nexception: %exception\nstack:\n%call_stack"
+  "%severity (%location): %msg\nlogfile_name: %logfile_name\nsource_ip: %source_ip\nhostname: %hostname (%internal_ip)\nexception: %exception\nstack:\n%call_stack"
 end
 
 def self.exceptionless_email_format
-  "%severity: %msg\nlogfile_name: #@logfile_name\nsource_ip: %source_ip\n"
+  "%severity (%location): %msg\nlogfile_name: %logfile_name\nsource_ip: %source_ip\nhostname: %hostname (%internal_ip)\n"
 end
 
 class EmailAcceptor
@@ -18,16 +19,18 @@ class EmailAcceptor
     @logfile_name = logfile_name
     @subject_formatter = Formatter.new(subject_format || "%severity - %msg")
     @body_formatter = Formatter.new(body_format)
+    @body_formatter.register_format('%logfile_name') {|event| @logfile_name}
   end
 
   def log(event)
     subject = @subject_formatter.replace(event)
     body = @body_formatter.replace(event)
-    msg = "Subject:#{subject}\nContent-Type: text/plain\n\n#{body}"
+    msg = "Subject:#{subject}\nFrom:#{@from_address}\nTo:#{@to_addresses.join(', ')}\nContent-Type: text/plain\n\n#{body}"
     Net::SMTP.start(@smtp_server) do |smtp|
       begin
         smtp.send_message(msg, @from_address, @to_addresses)
       rescue Exception => excpt
+        puts "got exception: #{excpt.inspect}"
       end
     end
   end

data/lib/doh/logger/event.rb

@@ -4,20 +4,33 @@ require 'ostruct'
 
 module DohLogger
 
-class Event < OpenStruct
-  def initialize(severity, msg, excpt = nil)
-    msg = msg.gsub(/\n/, '\n').gsub(/\r/, '\r').gsub(/\t/, '\t').firstn(4096)
-    super('severity' => severity, 'msg' => msg, 'exception' => excpt, 'time' => Time.now)
+class Event
+  attr_accessor :severity, :msg, :location, :exception, :time, :pid
+
+  def initialize(severity, msg, location = '', excpt = nil, literal = false)
+    msg = msg.gsub(/\n/, '\n').gsub(/\r/, '\r').gsub(/\t/, '\t') if !literal
+    @severity = severity
+    @msg = msg.firstn(4096)
+    @location = location
+    @exception = excpt
+    @time = Time.now
   end
 
   def call_stack
-    excpt = self.exception
-    excpt ? excpt.backtrace : caller
+    @exception ? @exception.backtrace : caller
   end
 
   def source_ip
     Doh::source_ip
   end
+
+  def inspect
+    result = "{'severity' => #{@severity.inspect}, 'msg' => #{@msg.inspect}, 'location' => #{@location.inspect} 'exception' => #{@exception.inspect} 'time' => #{@time.inspect}}"
+    if @exception
+      result += ", backtrace: #{@exception.backtrace.inspect}"
+    end
+    result
+  end
 end
 
 end

data/lib/doh/logger/formatter.rb

@@ -1,25 +1,18 @@
 require 'doh/logger/severity'
+require 'doh/util/internal_ip'
 
 module DohLogger
 
+class OptimizedStandardFormatter
+  def replace(event)
+    "#{event.time.strftime("%Y-%m-%d %H:%M:%S.") << "%03d" % (event.time.usec / 1000)} [#{DohLogger::severity_string(event.severity)}] (#{event.location}) : #{event.msg}"
+  end
+end
+
 class Formatter
   def initialize(template)
     @template = template
     @formats = {}
-    register_format('%severity') {|event| DohLogger::severity_string(event.severity)}
-    register_format('%msg') {|event| event.msg}
-    register_format('%time') {|event| event.time.strftime("%H:%M:%S.") << "%03d" % (event.time.usec / 1000)}
-    register_format('%datetime') {|event| event.time.strftime("%Y-%m-%d %H:%M:%S.") << "%03d" % (event.time.usec / 1000)}
-    register_format('%call_stack') {|event| event.call_stack.join("\n")}
-    register_format('%source_ip') {|event| event.source_ip}
-
-    register_format('%exception') do |event|
-      if event.exception.nil?
-        ''
-      else
-        event.exception.class.to_s + ' -- ' + event.exception.to_s
-      end
-    end
   end
 
   def register_format(field, &proc)
@@ -27,9 +20,22 @@ class Formatter
   end
 
   def replace(event)
-    @formats.inject(@template) do |sum, elem|
-      sum = sum.gsub(elem[0], elem[1].call(event))
+    result = @template.dup
+    result.gsub!('%severity', DohLogger::severity_string(event.severity)) if result =~ /%severity/
+    result.gsub!('%msg', event.msg) if result =~ /%msg/
+    result.gsub!('%location', event.location) if result =~ /%location/
+    result.gsub!('%time', event.time.strftime("%H:%M:%S.") << "%03d" % (event.time.usec / 1000)) if result =~ /%time/
+    result.gsub!('%datetime', event.time.strftime("%Y-%m-%d %H:%M:%S.") << "%03d" % (event.time.usec / 1000)) if result =~ /%datetime/
+    result.gsub!('%call_stack', event.call_stack.join("\n")) if result =~ /%call_stack/
+    result.gsub!('%source_ip', event.source_ip) if result =~ /%source_ip/
+    result.gsub!('%hostname', Socket.gethostname) if result =~ /%hostname/
+    result.gsub!('%internal_ip', Doh::internal_ip) if result =~ /%internal_ip/
+    result.gsub!('%exception', event.exception.nil? ? '' : "#{event.exception.class} -- #{event.exception}") if result =~ /%exception/
+
+    @formats.each do |elem|
+      result.gsub!(elem[0]) {elem[1].call(event)}
     end
+    result
   end
 end
 

data/lib/doh/logger/interface.rb

@@ -11,4 +11,17 @@ def self.log
   @@logger_interface
 end
 
+@@save_logger_interface = nil
+def self.disable_logging
+  return if @@logger_interface.is_a?(DohLogger::NullInterface)
+  @@save_logger_interface = @@logger_interface
+  @@logger_interface = DohLogger::NullInterface.new
+end
+
+def self.enable_logging
+  return if @@save_logger_interface.nil? || @@save_logger_interface.is_a?(DohLogger::NullInterface)
+  @@logger_interface = @@save_logger_interface
+  @@save_logger_interface = nil
+end
+
 end

data/lib/doh/logger/iostream_acceptor.rb

@@ -3,10 +3,11 @@ require 'doh/logger/formatter'
 module DohLogger
 
 class IOStreamAcceptor
+  attr_reader :ios
   def initialize(ios = nil, format = nil, error_format = nil)
     @ios = ios || STDOUT
-    @standard_formatter = Formatter.new(format || "%datetime [%severity] : %msg")
-    @error_formatter = Formatter.new(error_format || format || "%datetime [%severity] : %msg\nexception: %exception\nstack:\n%call_stack")
+    @standard_formatter = format ? Formatter.new(format) : OptimizedStandardFormatter.new
+    @error_formatter = Formatter.new(error_format || format || "%datetime [%severity] (%location) : %msg\nexception: %exception\nstack:\n%call_stack")
   end
 
   def flush_always(flag = true)

data/lib/doh/logger/null_interface.rb

@@ -1,6 +1,7 @@
 module DohLogger
 
 class NullInterface
+  attr_accessor :literal
   def fatal(*ignore) end
   def error(*ignore) end
   def notify(*ignore) end

data/lib/doh/logger/proxy.rb

@@ -0,0 +1,53 @@
+require 'doh/logger/interface'
+
+module DohLogger
+
+class Proxy
+  def initialize(location)
+    @location = location
+  end
+
+  def fatal(msg, excpt = nil)
+    Doh::log.fatal(msg, excpt, @location)
+  end
+
+  def error(msg, excpt = nil)
+    Doh::log.error(msg, excpt, @location)
+  end
+
+  def notify(msg)
+    Doh::log.notify(msg, @location)
+  end
+
+  def warn(msg)
+    Doh::log.warn(msg, @location)
+  end
+
+  def info(msg)
+    Doh::log.info(msg, @location)
+  end
+
+  def debug(msg)
+    Doh::log.debug(msg, @location)
+  end
+
+  def literal
+    Doh::log.literal = true
+    yield
+    Doh::log.literal = false
+  end
+end
+
+end
+
+class Module
+  def dohlog
+    @dohlog ||= DohLogger::Proxy.new(self.to_s)
+  end
+end
+
+class Object
+  def dohlog
+    self.class.dohlog
+  end
+end

data/lib/doh/logger/socket_acceptor.rb

@@ -0,0 +1,53 @@
+require 'socket'
+require 'doh/core/socket'
+require 'yaml'
+require 'thread'
+
+module DohLogger
+
+# this class accepts log events, and posts them to a socket
+# this is typically used in conjunction with socket_viewer.rb
+# to do remote debugging
+class SocketAcceptor
+  def initialize(host = '127.0.0.1', port = SocketAcceptor::default_port)
+    @host = host
+    @port = port
+    @last_connect_attempt = nil
+  end
+
+  def self.default_port
+    2377
+  end
+
+  def five_seconds
+    5.0/86440
+  end
+
+  # don't attempt to connect more than once every 5 seconds
+  def connect_socket(how_often = five_seconds)
+    if @sock || @last_connect_attempt && @last_connect_attempt >= (DateTime.now - how_often)
+      return
+    end
+    @last_connect_attempt = DateTime.now
+    @sock ||= TCPSocket.new(@host, @port)
+  end
+
+  def log(event)
+    begin
+      connect_socket
+      return if !@sock
+      event.pid = Process.pid
+      if event.exception
+        newevent = event.dup
+        newevent.exception = Exception.new("Exception(#{event.exception.class}): " + event.exception.to_s)
+        newevent.exception.set_backtrace(event.exception.backtrace)
+        event = newevent
+      end
+      @sock.write_object(Marshal::dump(event))
+    rescue Errno::ECONNREFUSED => e
+      #ignore silently?
+    end
+  end
+end
+
+end

data/lib/doh/logger/socket_viewer.rb

@@ -0,0 +1,64 @@
+require 'doh/core/socket'
+require 'ostruct'
+require 'thread'
+require 'yaml'
+require 'doh/logger/socket_acceptor'
+require 'doh/logger/event'
+
+module DohLogger
+
+# this class accepts log events through potentially multiple sockets
+# combining the events from the multiple sockets as they're received into a single event stream
+# this is typically used in conjunction with socket_acceptor.rb
+# to do remote debugging
+class SocketViewer
+  def initialize(port = DohLogger::SocketAcceptor::default_port)
+    @events = Queue.new
+    @server = TCPServer.new(port)
+  end
+
+  def shutdown
+    @server.close
+    @events.enq(:finished)
+  end
+
+  def accept_connections
+    thread_socks = []
+    begin
+      loop do
+        sock = @server.accept
+        thread_socks.push([Thread.new{handle_events_for_sock(sock)}, sock])
+      end
+    rescue IOError => e
+    ensure
+      thread_socks.each do |elem|
+        elem[1].close
+        elem[0].join
+      end
+    end
+  end
+
+  def handle_events_for_sock(sock)
+    begin
+      begin
+        obj = sock.read_object
+        @events.enq(obj)
+      end until sock.closed?
+    rescue IOError => e
+      #silently ignore IOErrors?
+    end
+    thread.join
+  end
+
+  def handle_events
+    thread = Thread.new {accept_connections}
+    loop do
+      event_str = @events.deq
+      break if event_str == :finished
+      event = Marshal::load(event_str)
+      yield event
+    end
+  end
+end
+
+end

data/lib/doh/logger/standard_interface.rb

@@ -4,44 +4,73 @@ require 'doh/logger/event'
 module DohLogger
 
 class StandardInterface
+  attr_accessor :literal
   def initialize(scheduler)
     @scheduler = scheduler
     @acceptors = []
+    @disabled_acceptors = []
   end
 
-  def fatal(msg, excpt = nil)
-    add(FATAL, msg, excpt)
+  # temporarily have this ordering of arguments, until all old style calls are switched
+  def fatal(msg, excpt = nil, location = '')
+    add(FATAL, msg, location, excpt)
   end
 
-  def error(msg, excpt = nil)
-    add(ERROR, msg, excpt)
+  # temporarily have this ordering of arguments, until all old style calls are switched
+  def error(msg, excpt = nil, location = '')
+    add(ERROR, msg, location, excpt)
   end
 
-  def notify(msg)
-    add(NOTIFY, msg)
+  def notify(msg, location = '')
+    add(NOTIFY, msg, location)
   end
 
-  def warn(msg)
-    add(WARN, msg)
+  def warn(msg, location = '')
+    add(WARN, msg, location)
   end
 
-  def info(msg)
-    add(INFO, msg)
+  def info(msg, location = '')
+    add(INFO, msg, location)
   end
 
-  def debug(msg)
-    add(DEBUG, msg)
+  def debug(msg, location = '')
+    add(DEBUG, msg, location)
   end
 
-  def add_acceptor(severity_threshold, acceptor, exact_severity_only = nil)
-    @acceptors.push([severity_threshold, acceptor, exact_severity_only])
+  def add_acceptor(severity_threshold, acceptor, exact_severity_only = nil, locations_to_exclude = nil, include_only_locations = nil)
+    @acceptors.push([severity_threshold, acceptor, exact_severity_only, locations_to_exclude, include_only_locations])
+  end
+
+  def disable_acceptors()
+    to_disable = @acceptors.inject([]) do |sum, acceptor|
+      sum.push(acceptor) if yield(acceptor)
+      sum
+    end
+    @disabled_acceptors.concat(to_disable)
+    @acceptors.reject! {|elem| to_disable.include?(elem)}
+  end
+
+  def enable_acceptors()
+    to_enable = @disabled_acceptors.inject([]) do |sum, acceptor|
+      sum.push(acceptor) if yield(acceptor)
+      sum
+    end
+    @acceptors.concat(to_enable)
+    @disabled_acceptors.reject! {|elem| to_enable.include?(elem)}
   end
 
 private
-  def add(severity, msg, excpt = nil)
+  def add(severity, msg, location, excpt = nil)
     included = []
-    @acceptors.each {|elem| included.push(elem[1]) if (severity >= elem[0] && !elem[2]) || severity == elem[0]}
-    @scheduler.push(included, Event.new(severity, msg, excpt))
+    @acceptors.each do |severity_threshold, acceptor, exact_severity_only, locations_to_exclude, include_only_locations|
+      if (((severity >= severity_threshold && !exact_severity_only) || severity == severity_threshold) &&
+          (!locations_to_exclude || !locations_to_exclude.find {|exc_loc| location.start_with?(exc_loc)}) &&
+          (!include_only_locations || include_only_locations.find {|inc_loc| location.start_with?(inc_loc)})
+          )
+        included.push(acceptor)
+      end
+    end
+    @scheduler.push(included, Event.new(severity, msg, location, excpt, @literal)) if included.size > 0
   end
 end
 

data/lib/doh/logger/util.rb

@@ -4,4 +4,22 @@ def self.default_logfile_name
   File.basename($PROGRAM_NAME, '.rb') + '.log'
 end
 
+class Proxy
+  def debug_options(shortname = 'd', longname = 'debug')
+    #note -- in order to use this right now you have to include these debug_options in your options hash, and then include a line like this:
+    #dohlog.disable_stdout unless OPTS.debug
+    {'debug' => [nil, "-#{shortname}", "--#{longname}", 'if specified, will display console debugging instead of just going to the file']}
+  end
+  def disable_stdout
+    Doh::log.disable_acceptors do |acceptor|
+      acceptor[1].ios == STDOUT
+    end
+  end
+  def enable_stdout
+    Doh::log.enable_acceptors do |acceptor|
+      acceptor[1].ios == STDOUT
+    end
+  end
+end
+
 end

data/lib/doh/logger.rb

@@ -1 +1,2 @@
-require 'doh/logger/interface'
+require 'doh/logger/proxy'
+require 'doh/logger/util'