alloc_track 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c2bcef216fe18bd1578ecc1b47f26e10cf6669ad
+  data.tar.gz: 2f003bcc8dc4f0f30a7123518192fb7c9a3c22da
+SHA512:
+  metadata.gz: 50d0ecf47e7a19ac2f9b59e484ab84e8379243d25c7a31487be87924d6b4f486bdecb50613874fcdc07b98a90b1c60946d264257df4597cab590233ebbd6b142
+  data.tar.gz: 86ee7fc454d489abb83df54d01d6fb226da1f0d66f56e5dacc764bf02fbc173494b3ef0bfe1abf2af7f159a978899ca05c4e3c8030c15d62d3bbb743c009a7e7

data/.gitignore

@@ -0,0 +1,4 @@
+/.bundle/
+/lib/alloc_track/*.so
+/lib/alloc_track/*.bundle
+/tmp/*

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,18 @@
+PATH
+  remote: .
+  specs:
+    alloc_track (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (10.3.1)
+    rake-compiler (0.9.2)
+      rake
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  alloc_track!
+  rake-compiler (~> 0.9)

data/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Scott Francis <scott.francis@shopify.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+## alloc_track
+
+Tracks the number of outstanding allocations on a Ruby thread using the internal tracepoint APIs.
+
+### Features
+
+- C extension for webscale
+- Allocations are counted per thread
+- Can raise an exception when allocations exceed a certain threshold
+
+### Usage
+
+It can be used to track the number of allocated objects over a period of time:
+
+```ruby
+require 'alloc_track/alloc_track'
+
+AllocTrack.start
+100.times { Object.new }
+puts AllocTrack.delta # >= 100
+GC.start
+puts AllocTrack.alloc # >= 100
+puts AllocTrack.delta # <= 100
+puts AllocTrack.free  # >= 100
+AllocTrack.stop
+```
+
+Perhaps more useful is the ability to raise when the number of allocations crosses a certain threshold:
+```ruby
+require 'alloc_track/alloc_track'
+
+AllocTrack.limit 100 do
+	200.times { Object.new } # raises AllocTrack::LimitExceeded
+end
+```
+
+### Performance
+
+In a contrived benchmark that simply allocates 10,000,000 new objects, alloc_track adds ~30% overhead:
+
+```
+[vagrant] ~/src/alloc_track (master *%) $ ruby -Ilib ./test/benchmark_alloc_track.rb
+               user     system      total        real
+none:      1.540000   0.000000   1.540000 (  1.545192)
+tracking:  2.020000   0.000000   2.020000 (  2.034162)
+```
+
+Expect real-world (operations other than just memory allocation) performance overhead to be much less severe.
+
+### Limitations
+
+- Allocation tracker can only be run on one thread per process

data/Rakefile

@@ -0,0 +1,36 @@
+task :default => :test
+
+# ==========================================================
+# Packaging
+# ==========================================================
+
+GEMSPEC = eval(File.read('alloc_track.gemspec'))
+
+require 'rubygems/package_task'
+Gem::PackageTask.new(GEMSPEC) do |pkg|
+end
+
+# ==========================================================
+# Ruby Extension
+# ==========================================================
+
+require 'rake/extensiontask'
+Rake::ExtensionTask.new('alloc_track', GEMSPEC) do |ext|
+  ext.ext_dir = 'ext/alloc_track'
+  ext.lib_dir = 'lib/alloc_track'
+end
+task :build => :compile
+
+# ==========================================================
+# Testing
+# ==========================================================
+
+require 'rake/testtask'
+Rake::TestTask.new 'test' do |t|
+  t.test_files = FileList['test/test_*.rb']
+end
+task :test => :build
+
+task :benchmark => :build do
+  ruby "-Ilib ./test/benchmark_alloc_track.rb"
+end

data/alloc_track.gemspec

@@ -0,0 +1,15 @@
+Gem::Specification.new do |s|
+  s.name = 'alloc_track'
+  s.version = '0.0.2'
+  s.summary = 'allocation tracker for ruby 2.1+'
+  s.description = 'tracks memory allocations with rgengc in ruby 2.1'
+
+  s.homepage = 'https://github.com/csfrancis/alloc_track'
+  s.authors = 'Scott Francis'
+  s.email   = 'scott.francis@shopify.com'
+  s.license = 'MIT'
+
+  s.files = `git ls-files`.split("\n")
+  s.extensions = ['ext/alloc_track/extconf.rb']
+  s.add_development_dependency 'rake-compiler', '~> 0.9'
+end

data/ext/alloc_track/alloc_track.c

@@ -0,0 +1,270 @@
+#include "ruby/ruby.h"
+#include "ruby/intern.h"
+#include "ruby/debug.h"
+
+#define ALLOCTRACK_OBJ_BIT FL_USER18
+
+typedef struct stat_collector {
+  struct stat_collector *next; /* not currently used */
+  VALUE thread;
+  int current_alloc;
+  int current_free;
+  int current_limit;
+  int limit_signal;
+} stat_collector_t;
+
+static VALUE mAllocTrack;
+static VALUE tpval, tpval_exception;
+static VALUE eAllocTrackError, eAllocTrackLimitExceeded;
+static stat_collector_t *root_collector, *current_collector;
+
+#define LOG(s) fprintf(stderr, s); fflush(stderr);
+
+static stat_collector_t *
+add_collector(VALUE thread)
+{
+  stat_collector_t *c = (stat_collector_t *) calloc(1, sizeof(*c));
+  c->thread = thread;
+  if (root_collector) {
+    c->next = root_collector;
+    root_collector = c;
+  } else {
+    root_collector = c;
+    rb_tracepoint_enable(tpval);
+  }
+  return c;
+}
+
+static void
+remove_collector(VALUE thread)
+{
+  stat_collector_t *c, *prev = NULL;
+
+  for (c = root_collector; c != NULL; prev = c, c = c->next) {
+    if (c->thread == thread) {
+      if (!prev) {
+        root_collector = c->next;
+      } else {
+        prev->next = c->next;
+      }
+      current_collector = NULL;
+      free(c);
+      break;
+    }
+  }
+
+  if (!root_collector) {
+    rb_tracepoint_disable(tpval);
+  }
+}
+
+static stat_collector_t *
+get_collector(VALUE thread)
+{
+  stat_collector_t *c;
+
+  if (current_collector && current_collector->thread == thread) {
+    return current_collector;
+  }
+
+  for (c = root_collector; c != NULL; c = c->next) {
+    if (c->thread == thread) {
+      /* cache the collector so we don't have to scan the list every time */
+      current_collector = c;
+      return c;
+    }
+  }
+
+  return NULL;
+}
+
+static VALUE
+started()
+{
+  return get_collector(rb_thread_current()) ? Qtrue : Qfalse;
+}
+
+static void
+validate_started()
+{
+  if (!started()) {
+    rb_raise(eAllocTrackError, "allocation tracker has not been started");
+  }
+}
+
+static void
+validate_stopped()
+{
+  if (started()) {
+    rb_raise(eAllocTrackError, "allocation tracker already started");
+  }
+}
+
+static VALUE
+start()
+{
+  validate_stopped();
+  /* TODO: support multiple running trackers */
+  if (root_collector) {
+    rb_raise(eAllocTrackError, "allocation tracker already running on another thread");
+  }
+  add_collector(rb_thread_current());
+  return Qnil;
+}
+
+static VALUE
+stop()
+{
+  validate_started();
+  remove_collector(rb_thread_current());
+  return Qnil;
+}
+
+static VALUE
+alloc()
+{
+  validate_started();
+  return INT2FIX(get_collector(rb_thread_current())->current_alloc);
+}
+
+static VALUE
+_free()
+{
+  validate_started();
+  return INT2FIX(get_collector(rb_thread_current())->current_free);
+}
+
+static VALUE
+delta()
+{
+  stat_collector_t *c;
+  validate_started();
+  c = get_collector(rb_thread_current());
+  return INT2FIX(c->current_alloc - c->current_free);
+}
+
+static VALUE
+do_limit(VALUE arg)
+{
+  start();
+  get_collector(rb_thread_current())->current_limit = FIX2INT(arg);
+  return rb_yield(Qnil);
+}
+
+static VALUE
+ensure_stopped(VALUE arg)
+{
+  if (started()) {
+    stop();
+  }
+  return Qnil;
+}
+
+static VALUE
+limit(VALUE self, VALUE num_allocs)
+{
+  if (!rb_block_given_p()) {
+    rb_raise(rb_eArgError, "block required");
+  }
+  if (!RB_TYPE_P(num_allocs, T_FIXNUM)) {
+    rb_raise(rb_eArgError, "limit() must be passed a number");
+  }
+  validate_stopped();
+  return rb_ensure(do_limit, num_allocs, ensure_stopped, Qnil);
+}
+
+static int
+is_collector_enabled(stat_collector_t *c)
+{
+  return c->limit_signal == 0 ? 1 : 0;
+}
+
+static int
+is_collector_limit_exceeded(stat_collector_t *c)
+{
+  return c->limit_signal;
+}
+
+static void
+tracepoint_hook(VALUE tpval, void *data)
+{
+  stat_collector_t *c;
+  rb_trace_arg_t *tparg = rb_tracearg_from_tracepoint(tpval);
+  rb_event_flag_t flag = rb_tracearg_event_flag(tparg);
+  VALUE obj = rb_tracearg_object(tparg);
+  switch(flag) {
+    case RUBY_INTERNAL_EVENT_NEWOBJ:
+      if ((c = get_collector(rb_thread_current())) != NULL && is_collector_enabled(c)) {
+        RBASIC(obj)->flags |= ALLOCTRACK_OBJ_BIT;
+        c->current_alloc++;
+
+        if (c->current_limit && (c->current_alloc - c->current_free) > c->current_limit) {
+          c->limit_signal = 1;
+          if (!rb_tracepoint_enabled_p(tpval_exception)) {
+            /*
+              it's not safe to raise an exception from an internal event handler.
+              in order to get around this, we enable a normal tracepoint on all
+              events and raise from there.
+            */
+            rb_tracepoint_enable(tpval_exception);
+          }
+        }
+      }
+      break;
+    case RUBY_INTERNAL_EVENT_FREEOBJ:
+      if ((c = get_collector(rb_thread_current())) != NULL && is_collector_enabled(c) &&
+          (RBASIC(obj)->flags & ALLOCTRACK_OBJ_BIT)) {
+        c->current_free++;
+      }
+      break;
+  }
+}
+
+static int
+any_collectors_with_exceeded_limits()
+{
+  stat_collector_t *c;
+  for (c = root_collector; c != NULL; c = c->next) {
+    if (c->limit_signal) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+static void
+exception_tracepoint_hook(VALUE tpval, void *data)
+{
+  VALUE th = rb_thread_current();
+  stat_collector_t *c;
+  if ((c = get_collector(th)) != NULL && is_collector_limit_exceeded(c)) {
+    remove_collector(th);
+    if (!any_collectors_with_exceeded_limits()) {
+      rb_tracepoint_disable(tpval_exception);
+    }
+    rb_raise(eAllocTrackLimitExceeded, "allocation limit exceeded");
+  }
+}
+
+void
+Init_alloc_track()
+{
+  mAllocTrack = rb_define_module("AllocTrack");
+
+  rb_define_singleton_method(mAllocTrack, "start", start, 0);
+  rb_define_singleton_method(mAllocTrack, "started?", started, 0);
+  rb_define_singleton_method(mAllocTrack, "stop", stop, 0);
+  rb_define_singleton_method(mAllocTrack, "alloc", alloc, 0);
+  rb_define_singleton_method(mAllocTrack, "free", _free, 0);
+  rb_define_singleton_method(mAllocTrack, "delta", delta, 0);
+  rb_define_singleton_method(mAllocTrack, "limit", limit, 1);
+
+  eAllocTrackError = rb_define_class_under(mAllocTrack, "Error", rb_eStandardError);
+  eAllocTrackLimitExceeded = rb_define_class_under(mAllocTrack, "LimitExceeded", rb_eStandardError);
+
+  tpval = rb_tracepoint_new(0, RUBY_INTERNAL_EVENT_NEWOBJ|RUBY_INTERNAL_EVENT_FREEOBJ, tracepoint_hook, NULL);
+  tpval_exception = rb_tracepoint_new(0, RUBY_EVENT_TRACEPOINT_ALL, exception_tracepoint_hook, NULL);
+
+  rb_gc_register_mark_object(tpval);
+  rb_gc_register_mark_object(tpval_exception);
+}

data/ext/alloc_track/extconf.rb

@@ -0,0 +1,15 @@
+require 'mkmf'
+
+$CFLAGS = "-O3"
+
+have_func('rb_tracepoint_enable')
+
+gc_event = have_const('RUBY_INTERNAL_EVENT_NEWOBJ')
+
+if gc_event
+  create_makefile('alloc_track/alloc_track')
+else
+  File.open('Makefile', 'w') do |f|
+    f.puts "install:\n\t\n"
+  end
+end

data/test/benchmark_alloc_track.rb

@@ -0,0 +1,28 @@
+require 'benchmark'
+require 'alloc_track/alloc_track'
+
+def alloc_obj
+  Object.new
+end
+
+n = 100
+i = 100000
+
+Benchmark.bm(8) do |x|
+  x.report("none:") do
+    GC.start
+    n.times do
+      i.times { alloc_obj }
+      GC.start
+    end
+  end
+  x.report("tracking:") do
+    GC.start
+    AllocTrack.start
+    n.times do
+      i.times { alloc_obj }
+      GC.start
+    end
+    AllocTrack.stop
+  end
+end

data/test/test_alloc_track.rb

@@ -0,0 +1,76 @@
+require 'test/unit'
+require 'alloc_track/alloc_track'
+
+class TestAllocTrack < Test::Unit::TestCase
+  def test_allocate
+    AllocTrack.start
+    100.times { Object.new }
+    assert_operator AllocTrack.delta, :>=, 100
+    AllocTrack.stop
+  end
+
+  def test_allocate_with_gc
+    AllocTrack.start
+    100.times { Object.new }
+    GC.start
+    assert_operator AllocTrack.alloc, :>=, 100
+    assert_operator AllocTrack.delta, :<, 100
+    assert_operator AllocTrack.free, :>=, 100
+    AllocTrack.stop
+  end
+
+  def test_thread_not_included
+    AllocTrack.start
+    t = Thread.new do
+      100.times { Object.new }
+    end
+    t.join
+    assert_operator AllocTrack.delta, :<, 100
+    AllocTrack.stop
+  end
+
+  def test_delta_raises_when_not_started
+    assert_raise AllocTrack::Error do
+      AllocTrack.delta
+    end
+  end
+
+  def test_limit_with_no_block
+    assert_raise ArgumentError do
+      AllocTrack.limit 100
+    end
+  end
+
+  def test_limit_with_non_number
+    assert_raise ArgumentError do
+      AllocTrack.limit "foo" do
+      end
+    end
+  end
+
+  def test_limit_raises
+    assert_raise AllocTrack::LimitExceeded do
+      AllocTrack.limit 10 do
+        200.times { Object.new }
+      end
+    end
+  end
+
+  def test_within_limit
+    assert_nothing_raised do
+      AllocTrack.limit 100 do
+        50.times { Object.new }
+      end
+    end
+  end
+
+  def test_limit_exception_stops
+    assert_raise RuntimeError do
+      AllocTrack.limit 100 do
+        raise RuntimeError
+      end
+    end
+    refute AllocTrack.started?
+  end
+
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: alloc_track
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Scott Francis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: tracks memory allocations with rgengc in ruby 2.1
+email: scott.francis@shopify.com
+executables: []
+extensions:
+- ext/alloc_track/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE.md
+- README.md
+- Rakefile
+- alloc_track.gemspec
+- ext/alloc_track/alloc_track.c
+- ext/alloc_track/extconf.rb
+- test/benchmark_alloc_track.rb
+- test/test_alloc_track.rb
+homepage: https://github.com/csfrancis/alloc_track
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: allocation tracker for ruby 2.1+
+test_files: []