madvertise-ext 0.4.0 → 0.5.0

data/.ruby-gemset

@@ -0,0 +1 @@
+madvertise-ext

data/.ruby-version

@@ -0,0 +1 @@
+ruby-1.9.3-p392

data/Gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source 'https://rubygems.org'
 
 gemspec
 

data/lib/madvertise/boot.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+# load a bunch of common classes here, so we don't have to track and repeat it
+# everywhere
+require 'active_support'
+require 'cgi'
+require 'date'
+require 'json'
+require 'servolux'
+require 'socket'
+
+# load all madvertise extensions
+Dir[File.join(File.dirname(__FILE__), 'ext', '*.rb')].each do |f|
+  require f
+end
+
+Dir[File.join(File.dirname(__FILE__), '*.rb')].each do |f|
+  require f
+end
+
+require 'madvertise/logging' # dedicated gem
+
+# initialize configuration and logger with hardcoded defaults
+$conf = Conf = Configuration.new
+$conf.callback do
+  ImprovedLogger::Formatter.format = $conf.log_format
+  $log = MultiLogger.new
+  $log.attach(ImprovedLogger.new($conf.log_backend.to_sym, File.basename($0)))
+  $log.level = $conf.log_level.downcase.to_sym
+  $log.log_caller = $conf.log_caller
+end
+
+# trigger log callback with defaults
+$conf.reload!

data/lib/madvertise/cli.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+
+require 'madvertise/boot'
+require 'mixlib/cli'
+
+$0 = File.basename($0)
+
+class CLI
+  include Mixlib::CLI
+
+  option :configfile,
+    short: '-c FILE',
+    long: '--config FILE',
+    description: 'Configuration File to load'
+
+  option :name,
+    :short => '-n NAME',
+    :long => '--name NAME',
+    :description => 'Process name',
+    :default => $0,
+    :proc => ->(value) { $0 = value }
+
+  option :environment,
+    :short => '-e ENVIRONMENT',
+    :long => '--environment ENVIRONMENT',
+    :description => "Set the daemon environment",
+    :default => "development",
+    :proc => ->(value) { Env.set(value) }
+
+  option :debug,
+    :short => '-D',
+    :long => '--debug',
+    :description => "Enable debug output",
+    :boolean => true,
+    :default => false,
+    :proc => ->(value) { $conf.mixin(log_level: :debug) }
+
+  option :help,
+    :short => '-h',
+    :long => '--help',
+    :description => "Show this message",
+    :on => :tail,
+    :boolean => true,
+    :show_options => true,
+    :exit => 0
+
+  def option(name, args)
+    args[:on] ||= :on
+    args[:boolean] ||= false
+    args[:required] ||= false
+    args[:proc] ||= nil
+    args[:show_options] ||= false
+    args[:exit] ||= nil
+
+    if args.has_key?(:default)
+      config[name.to_sym] = args[:default]
+    end
+
+    options[name.to_sym] = args
+  end
+
+  def self.for(cls, &block)
+    cli = new
+    cli.instance_eval(&block) if block_given?
+    cli.parse_options
+
+    $log.info("cli:initialize", cli.config)
+    $conf.reload!
+
+    opts = cli.config.merge({
+      interval: 1,
+      logger: $log,
+    })
+
+    cls.new(opts)
+  end
+
+end

data/lib/madvertise/{ext/config.rb → configuration.rb}

@@ -1,8 +1,10 @@
+# encoding: utf-8
+
 require 'yaml'
 require 'set'
 
 require 'madvertise/ext/hash'
-require 'madvertise/ext/environment'
+require 'madvertise/environment'
 
 ##
 # A {Configuration} consists of one or more Sections. A section is a hash-like
@@ -58,30 +60,14 @@ class Section < Hash
 
   end
 
-  # Build the call chain including NilSections.
-  #
   # @private
   def method_missing(name, *args)
     if name.to_s =~ /(.*)=$/
       self[$1.to_sym] = Section.from_value(args.first)
     else
       value = self[name]
-      value = value.call if value.is_a?(Proc)
-
-      if value.nil?
-        case self.class.nil_action
-        when :nil, nil
-          # do nothing
-        when :raise
-          raise "value is nil for key #{name}"
-        when :section
-          value = NilSection.new if value.nil?
-        else
-          raise "unknown nil handling: #{self.class.nil_action}"
-        end
-      end
-
-      self[name] = value
+      self[name] = value.call if value.is_a?(Proc)
+      self[name]
     end
   end
 end
@@ -92,28 +78,20 @@ end
 #
 class Configuration < Section
 
+  DEFAULTS = {
+    log_backend: :stdout,
+    log_caller: false,
+    log_format: "%{time} %{progname}(%{pid}) [%{severity}] %{msg}\n",
+    log_level: :info,
+  }
+
   # Create a new {Configuration} object.
   #
-  # @param [Symbol] mode  The mode to load from the configurtion file
-  #                       (production, development, etc)
   # @yield [config]  The new configuration object.
   def initialize
     @mixins = Set.new
     @callbacks = []
-    yield self if block_given?
-  end
-
-  # Load given mixins from +path+.
-  #
-  # @param [String] path  The path to mixin files.
-  # @param [Array] mixins_to_use  A list of mixins to load from +path+.
-  # @return [void]
-  def load_mixins(path, mixins_to_use)
-    mixins_to_use.map do |mixin_name|
-      File.join(path, "#{mixin_name}.yml")
-    end.each do |mixin_file|
-      mixin(mixin_file)
-    end
+    mixin(DEFAULTS)
   end
 
   # Mixin a configuration snippet into the current section.
@@ -124,24 +102,20 @@ class Configuration < Section
   #                              contain a YAML hash.
   # @return [void]
   def mixin(value)
+    @mixins << value
+
     if value.is_a?(String)
-      @mixins << value
       value = YAML.load(File.read(value))
     end
 
     value = Section.from_hash(value)
 
-    self.deep_merge!(value[:default]) if value.has_key?(:default)
-    self.deep_merge!(value[:generic]) if value.has_key?(:generic)
+    deep_merge!(value[:generic]) if value.has_key?(:generic)
 
     if value.has_key?(Env.to_sym)
-      self.deep_merge!(value[Env.to_sym])
+      deep_merge!(value[Env.to_sym])
     else
-      self.deep_merge!(value)
-    end
-
-    @callbacks.each do |callback|
-      callback.call
+      deep_merge!(value)
     end
   end
 
@@ -149,9 +123,14 @@ class Configuration < Section
   #
   # @return [void]
   def reload!
-    self.clear
+    clear
+
     @mixins.each do |file|
-      self.mixin(file)
+      mixin(file)
+    end
+
+    @callbacks.each do |callback|
+      callback.call
     end
   end
 
@@ -163,45 +142,3 @@ class Configuration < Section
   end
 
 end
-
-##
-# A NilSection is returned for all missing/empty values in the config file. This
-# allows for terse code when accessing values that have not been configured by
-# the user.
-#
-# Consider code like this:
-#
-#   config.server.listen.tap do |listen|
-#     open_socket(listen.host, listen.port)
-#   end
-#
-# Given that your server component is optional and does not appear in the
-# configuration file at all, +config.server.listen+ will return a NilSection
-# that does not call the block given to tap _at all_.
-#
-class NilSection
-  # @return true
-  def nil?
-    true
-  end
-
-  # @return true
-  def empty?
-    true
-  end
-
-  # @return false
-  def present?
-    false
-  end
-
-  # @return nil
-  def tap
-    nil
-  end
-
-  # @private
-  def method_missing(*args, &block)
-    self
-  end
-end

data/lib/madvertise/ext/object_space.rb

@@ -0,0 +1,49 @@
+module Memory
+  WORD_SIZE = 8
+  OBJ_SIZE = 40 # some are smaller
+  OBJ_OVERHEAD = WORD_SIZE + OBJ_SIZE
+
+  def self.size(obj)
+    return WORD_SIZE if obj.is_a?(Fixnum)
+
+    case obj
+    when String
+      obj.size
+    when Array
+      obj.size * WORD_SIZE
+    when Hash
+      obj.size * WORD_SIZE * 2
+    #when Enumerable
+    #  result = 0
+    #  obj.each do |ref|
+    #    result += WORD_SIZE
+    #  end
+    #  result
+    else
+      0
+    end + OBJ_OVERHEAD
+  rescue => e
+    puts "failed to get object size for #{obj.inspect}: #{e}"
+    return OBJ_OVERHEAD
+  end
+end
+
+module ObjectSpace
+  def self.memory_stats(*args)
+    stats = {}
+
+    self.each_object do |obj|
+      stats[obj.class] ||= []
+      stats[obj.class] << Memory.size(obj)
+    end
+
+    stats.map do |cls, sizes|
+      cnt = sizes.length
+      sum = sizes.reduce(:+)
+      avg = sum / cnt
+      [cls, [cnt, avg, sum]]
+    end.sort_by do |cls, sizes|
+      sizes[2]
+    end.reverse
+  end
+end

data/lib/madvertise/ext/signal.rb

@@ -0,0 +1,8 @@
+module Signal
+  def self.register_shutdown_handler(&block)
+    %w(QUIT INT TERM).each do |sig|
+      old = trap(sig) {}
+      trap(sig) { block.call; old.call }
+    end
+  end
+end

data/lib/madvertise/ext/version.rb

@@ -1,6 +1,6 @@
 # @private
 module Madvertise
   module Ext
-    VERSION = "0.4.0"
+    VERSION = "0.5.0"
   end
 end

data/lib/madvertise/from_file.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+module FromFile
+
+  module ClassMethods
+    def from_file(path)
+      new.tap do |obj|
+        obj.from_file(path)
+      end
+    end
+  end
+
+  def from_file(path)
+    if File.exists?(path) && File.readable?(path)
+      self.instance_eval(IO.read(path), path, 1)
+    else
+      raise IOError, "Cannot open or read #{path}!"
+    end
+  end
+
+  def self.included(receiver)
+    receiver.extend(ClassMethods)
+  end
+
+end

data/lib/madvertise/gc_stats.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require 'metriks'
+require 'singleton'
+
+class GCStats
+  include Singleton
+
+  # RUBY 1.9.X-Implementation
+  module Yarv
+    def enable
+      GC::Profiler.enable
+    end
+
+    def clear
+      GC::Profiler.clear
+    end
+
+    def gather
+      count_collections
+      count_allocations
+      count_objects
+      Metriks.timer('gc_stats.total_time').update(GC::Profiler.total_time)
+    end
+
+    def count_collections
+      Metriks.histogram('gc_stats.collections').update(GC.count)
+    end
+
+    def count_allocations
+      allocated_size = GC.respond_to?(:malloc_allocated_size) ? GC.malloc_allocated_size / 1000.0 : 0
+      Metriks.histogram('gc_stats.allocated').update(allocated_size)
+    end
+
+    def count_objects
+      objects = ObjectSpace.count_objects
+      objects = objects[:TOTAL] - objects[:FREE]
+      Metriks.histogram('gc_stats.objects').update(objects)
+    end
+  end
+
+  if RUBY_VERSION =~ /^1\.9/
+    extend Yarv
+  end
+
+  def self.available?
+    respond_to?(:gather)
+  end
+
+  def self.start!
+    if available?
+      enable
+      $log.info("gc:stats", enabled: true)
+
+      EventMachine.next_tick do
+        EventMachine::PeriodicTimer.new(60*60) do
+          GC.start
+        end
+
+        EventMachine::PeriodicTimer.new(60) do
+          gather
+          clear
+        end
+      end
+    else
+      $log.info("gc:stats", enabled: false)
+    end
+  end
+end

data/lib/madvertise/hash_helper.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+require 'digest'
+
+class HashHelper
+  def self.numeric_hash(val)
+    # Use a the first 15 bytes of a MD5 bytes - small enough to be represented as Fixnum => no garbage collection
+    Digest::MD5.hexdigest(val.to_s)[0..14].to_i(16)
+  end
+end

data/lib/madvertise/proc_stat.rb

@@ -0,0 +1,56 @@
+require 'madvertise/sysconf'
+
+ProcStat = Struct.new(
+  :ppid,
+  :pgrp,
+  :session,
+  :tty_nr,
+  :tpgid,
+  :flags,
+  :minflt,
+  :cminflt,
+  :majflt,
+  :cmajflt,
+  :utime,
+  :stime,
+  :cutime,
+  :cstime,
+  :priority,
+  :nice,
+  :num_threads,
+  :itrealvalue,
+  :starttime,
+  :vsize,
+  :rss,
+  :rsslim,
+  :startcode,
+  :endcode,
+  :startstack,
+  :kstkesp,
+  :kstkeip,
+  :signal,
+  :blocked,
+  :sigignore,
+  :sigcatch,
+  :wchan,
+  :nswap,
+  :cnswap,
+  :exit_signal,
+  :processor,
+  :rt_priority,
+  :policy,
+  :delayacct_blkio_ticks,
+  :guest_time,
+  :cguest_time
+)
+
+class ProcStat
+  def self.read
+    stat = File.read("/proc/self/stat").chomp.split
+    new(*stat[3..40].map(&:to_i))
+  end
+
+  def pagesize
+    Sysconf.sysconf(:page_size)
+  end
+end

data/lib/madvertise/sysconf.rb

@@ -0,0 +1,25 @@
+require 'ffi'
+require 'ffi/tools/const_generator'
+
+module Sysconf
+  extend FFI::Library
+  ffi_lib ["c"]
+
+  fcg = FFI::ConstGenerator.new do |gen|
+    gen.include 'unistd.h'
+
+    %w[
+    _SC_PAGE_SIZE
+    _SC_VERSION
+    ].each do |const|
+      ruby_name = const.sub(/^_SC_/, '').downcase.to_sym
+      gen.const(const, "%d", nil, ruby_name, &:to_i)
+    end
+  end
+
+  CONF = enum(*fcg.constants.map{|_, const|
+    [const.ruby_name, const.converted_value]
+  }.flatten)
+
+  attach_function :sysconf, [CONF], :long
+end

data/madvertise-ext.gemspec

@@ -10,6 +10,8 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Ruby extensions}
   gem.homepage      = "https://github.com/madvertise/ext"
 
+  gem.add_dependency "activesupport"
+  gem.add_dependency "ffi"
   gem.add_dependency "madvertise-logging"
   gem.add_dependency "mixlib-cli"
   gem.add_dependency "servolux"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: madvertise-ext
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,40 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-10 00:00:00.000000000 Z
+date: 2013-05-11 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: madvertise-logging
   requirement: !ruby/object:Gem::Requirement
@@ -67,27 +99,33 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
-- .rvmrc
+- .ruby-gemset
+- .ruby-version
 - .travis.yml
 - .yardopts
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
+- lib/madvertise/boot.rb
+- lib/madvertise/cli.rb
+- lib/madvertise/configuration.rb
+- lib/madvertise/environment.rb
 - lib/madvertise/ext/array.rb
-- lib/madvertise/ext/config.rb
 - lib/madvertise/ext/enumerable.rb
-- lib/madvertise/ext/environment.rb
 - lib/madvertise/ext/hash.rb
-- lib/madvertise/ext/logging.rb
-- lib/madvertise/ext/mask.reek
 - lib/madvertise/ext/nil.rb
+- lib/madvertise/ext/object_space.rb
+- lib/madvertise/ext/signal.rb
 - lib/madvertise/ext/string.rb
-- lib/madvertise/ext/transaction_id.rb
 - lib/madvertise/ext/version.rb
-- lib/servolux/cli.rb
+- lib/madvertise/from_file.rb
+- lib/madvertise/gc_stats.rb
+- lib/madvertise/hash_helper.rb
+- lib/madvertise/proc_stat.rb
+- lib/madvertise/sysconf.rb
+- lib/madvertise/transaction_id.rb
 - lib/servolux/eventmachine.rb
-- lib/servolux/wrapper.rb
 - madvertise-ext.gemspec
 - spec/enumerable_spec.rb
 - spec/spec.opts
@@ -109,7 +147,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1335580830932580809
+      hash: 3215958011008345483
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -118,10 +156,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1335580830932580809
+      hash: 3215958011008345483
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Ruby extensions

data/.rvmrc

@@ -1 +0,0 @@
-rvm use --create ruby-1.9.3-p327@madvertise-ext

data/lib/madvertise/ext/logging.rb

@@ -1,26 +0,0 @@
-# encoding: utf-8
-
-require 'madvertise/ext/environment'
-require 'madvertise-logging'
-
-def init_logger(progname=$0, filename=nil)
-  progname = File.basename(progname)
-  filename ||= "#{Env.mode}.log"
-
-  MultiLogger.new.tap do |logger|
-    init_multi_logger(logger, progname, filename)
-  end
-end
-
-def init_multi_logger(logger, progname=$0, filename=nil)
-  if Env.dev? or Env.test?
-    logger.attach(ImprovedLogger.new(STDERR, progname))
-  else
-    logger.attach(ImprovedLogger.new(:syslog, progname))
-  end
-
-  # default log level
-  logger.level = Logger::INFO
-end
-
-$log = init_logger

data/lib/madvertise/ext/mask.reek

@@ -1,7 +0,0 @@
----
-NestedIterators:
-  exclude:
-    - Section#from_hash
-FeatureEnvy:
-  exclude:
-    - Hash#deep_merge

data/lib/servolux/cli.rb

@@ -1,73 +0,0 @@
-# encoding: utf-8
-
-require 'madvertise/ext/environment'
-require 'mixlib/cli'
-require 'servolux'
-
-module Servolux
-  class BaseCLI
-    include Mixlib::CLI
-
-    def self.inherited(subclass)
-      subclass.option :environment,
-        :short => '-e ENVIRONMENT',
-        :long => '--environment ENVIRONMENT',
-        :description => "Set the daemon environment",
-        :default => "development",
-        :proc => ->(value) { Env.set(value) }
-
-      subclass.option :debug,
-        :short => '-D',
-        :long => '--debug',
-        :description => "Enable debug output",
-        :boolean => true,
-        :default => false
-
-      subclass.option :help,
-        :short => '-h',
-        :long => '--help',
-        :description => "Show this message",
-        :on => :tail,
-        :boolean => true,
-        :show_options => true,
-        :exit => 0
-    end
-
-    def self.parse_options
-      new.tap do |cli|
-        cli.parse_options
-      end.config
-    end
-  end
-
-  class CLI < BaseCLI
-  end
-
-  class DaemonCLI < BaseCLI
-    option :name,
-      :short => '-n NAME',
-      :long => '--name NAME',
-      :description => 'Process name',
-      :default => $0,
-      :proc => ->(value) { $0 = value }
-
-    option :pidfile,
-      :short => '-p PIDFILE',
-      :long => '--pidfile PIDFILE',
-      :description => "The daemon pidfile",
-      :default => "#{$0}.pid"
-
-    option :daemonize,
-      :short => '-d',
-      :long => '--daemonize',
-      :description => "Daemonize the server process",
-      :boolean => true,
-      :default => false
-
-    option :kill,
-      :short => '-k',
-      :long => '--kill',
-      :description => "Kill the currently running daemon instance",
-      :boolean => true
-  end
-end

data/lib/servolux/wrapper.rb

@@ -1,51 +0,0 @@
-# encoding: utf-8
-
-require 'madvertise/ext/logging'
-require 'servolux'
-require 'servolux/cli'
-
-module Servolux
-  def self.parse_opts(cli_class)
-    opts = cli_class.parse_options
-
-    # CLI.parse_options may have changed $0
-    # so we reload the logger for good measure
-    $log = init_logger
-    $log.level = opts[:debug] ? :debug : :info
-
-    return opts
-  end
-
-  def self.wrap(server_class)
-    opts = self.parse_opts(Servolux::CLI)
-    server_class.new(opts).run
-  rescue => e
-    $log.exception(e)
-    raise e
-  end
-
-  def self.wrap_daemon(server_class)
-    opts = self.parse_opts(Servolux::DaemonCLI)
-
-    server = server_class.new(opts[:name], opts.merge({
-      interval: 1,
-      logger: $log,
-      pid_file: opts[:pidfile]
-    }))
-
-    if opts[:daemonize] or opts[:kill]
-      daemon = Servolux::Daemon.new(:server => server)
-
-      if opts[:kill]
-        daemon.shutdown
-      else
-        daemon.startup
-      end
-    else
-      server.startup
-    end
-  rescue => e
-    $log.exception(e)
-    raise e
-  end
-end