counters 1.0.2 → 1.1.0

data/README.textile

@@ -1,53 +1,65 @@
 h1. Counters
 
-Easily record any metric from anywhere within your system. Metrics are recorded to Redis (using the provided backend), in a single Hash key. You can then extract the keys later and use them with Cacti to generate graphs about anything going on.
+Easily record any metrics from anywhere within your code, using a very simple interface:
 
-h2. Sample Usage
+* <code>ping</code>: When's the last time we saw this thing?
+* <code>hit</code>: Increments a counter
+* <code>magnitude</code>: Measures numerical values
+* <code>latency</code>: Measures time intervals
 
-Let's say you have a crawler. You'd like to record the number of URLs you visit, the number of URLs you skipped due to 304 Not Modified responses, and the number of bytes you consumed, and the amount of time each page takes to process. Here's how you'd do that:
+h2. Example
 
-<pre><code>require "counters"
-require "redis"
-require "rest_client"
-Counter = Counters::Redis.new(Redis.new, "counters")
+Let's say you have a web crawler. There are a ton of things you can measure about your crawler: how many pages it processed (<code>#hit</code>), how many bytes you read (<code>#magnitude</code>), how long did it take to download the page (<code>#latency</code>), how long did it take to parse the raw HTML to a useable format (<code>#latency</code>).
 
-while url = STDIN.gets
-  Counter.hit "crawler.urls"
+<pre><code>require "uri"
+require "open-uri"
+require "counters"
+require "nokogiri"
 
-  response = RestClient.get(url)
-  Counter.magnitude "crawler.bytes.read", response.length
-  next Counter.hit "crawler.urls.skipped" if response.code == 304
+Counter = Counters::Redis.new(Redis.new, :namespace => "crawler", :base_key => "counters")
 
-  Counter.latency "crawler.processing" do
-    # some long and complicated processing
+urls_to_crawl = ["http://blog.teksol.info/", "http://techcrunch.com/", "http://www.google.com/"]
+
+while url = urls_to_crawl.pop
+  Counter.ping "crawler"
+
+  begin
+    Counter.hit "urls_popped"
+
+    puts "Fetching #{url}"
+    raw_html = Counter.latency "download" do
+      URI.parse(url).read
+    end
+
+    Counter.magnitude "bytes_in", raw_html.length
+
+    parsed_html = Counter.latency "html_parsing" do
+      Nokogiri::HTML(raw_html)
+    end
+  rescue
+    Counter.hit "error"
   end
 end
 </code></pre>
 
-Redis will have a single key, named counters here, with the following keys and values in it (after 1 call with a 200 response code):
-
-* hits.crawler.urls = 1
-* magnitudes.crawler.bytes.read = 2041
-* latencies.crawler.processing.count = 1
-* latencies.crawler.processing.nanoseconds = 381000000
-
-h2. Other Implementations
+h2. Other Backends
 
 For testing purposes, there also exists a <code>Counters::Memory</code>. This would be good in test mode, for example. The counters are exposed through accessor methods returning a Hash.
 
-For file logging, you should use <code>Counters::File</code>, which accepts a <code>String</code>, <code>IO</code> or <code>Logger</code> instance. All of these will be transformed to a <code>Logger</code> instance with a very strict format. All events will be logged to the file, one event per line.
+You may log to a file, but be advised the file's size grows very quickly. Counters are stored in the file, one per line, in an easily readable format.
 
 <pre><code>$ irb -r counters
 > Counter = Counters::File.new("counters.log")
  => #<Counters::File:0x00000101a18f18 @logger=#<Logger:0x00000101a18ef0 @progname=nil, @level=0, @default_formatter=#<Logger::Formatter:0x00000101a18ea0 @datetime_format=nil>, @formatter=#<Proc:0x00000101a18bd0@/Users/francois/Projects/counters/lib/counters/file.rb:15 (lambda)>, @logdev=#<Logger::LogDevice:0x00000101a18e28 @shift_size=1048576, @shift_age=0, @filename="counters.log", @dev=#<File:counters.log>, @mutex=#<Logger::LogDevice::LogDeviceMutex:0x00000101a18e00 @mon_owner=nil, @mon_count=0, @mon_mutex=#<Mutex:0x00000101a18d88>>>>> 
 > Counter.hit "crawler.page_read"
- => true 
+ => true
 > Counter.magnitude "crawler.bytes_in", 9_921
- => true 
+ => true
 > Counter.latency "crawler.processing" do sleep 0.3 ; end
- => true 
+ => true
 > Counter.ping "crawler.alive"
- => true 
+ => true
+> exit
 
 $ cat counters.log
 2011-02-21T09:46:21.296326000 - hit: crawler.page_read
@@ -59,6 +71,10 @@ $ cat counters.log
 2011-02-21T09:46:27.989183000 - latency: crawler.processing 0.3123122982101s
 </code></pre>
 
+You may also output your counters to "StatsD":https://github.com/etsy/statsd. The only change that might be surprising is magnitudes are output as timer events. Magnitudes are used to record
+
+See the file "samples/crawler.rb":blob/master/samples/crawler.rb for a more detailed example.
+
 h2. LICENSE
 
 (The MIT License)

data/lib/counters.rb

@@ -11,4 +11,5 @@ module Counters
   autoload :Redis, "counters/redis"
   autoload :Memory, "counters/memory"
   autoload :File, "counters/file"
+  autoload :StatsD, "counters/stats_d"
 end

data/lib/counters/base.rb

@@ -1,13 +1,21 @@
+require "benchmark"
+
 module Counters
   class Base
+    attr_accessor :namespace
+
+    def initialize(options={})
+      @namespace = options[:namespace]
+    end
+
     def hit(key)
       validate(key)
-      record_hit(key)
+      record_hit(namespaced_key(key))
     end
 
     def magnitude(key, value)
       validate(key)
-      record_magnitude(key, value)
+      record_magnitude(namespaced_key(key), value)
     end
 
     def latency(key, time_in_seconds=nil)
@@ -18,13 +26,13 @@ module Counters
         time_in_seconds = Benchmark.measure { result = yield }.real
       end
 
-      record_latency(key, time_in_seconds)
+      record_latency(namespaced_key(key), time_in_seconds)
       result
     end
 
     def ping(key)
       validate(key)
-      record_ping(key)
+      record_ping(namespaced_key(key))
     end
 
     def record_hit(key)
@@ -51,5 +59,11 @@ module Counters
       key.to_s =~ /\A[.\w]+\Z/i or raise ArgumentError, "Keys can contain only letters, numbers, the underscore (_) and fullstop (.), received #{key.inspect}"
     end
     private :validate
+
+    def namespaced_key(key)
+      return key if namespace.nil?
+      "#{namespace}.#{key}"
+    end
+    private :namespaced_key
   end
 end

data/lib/counters/file.rb

@@ -1,9 +1,10 @@
-require "benchmark"
 require "logger"
 
 module Counters
   class File < Counters::Base
-    def initialize(path_or_io_or_logger)
+    def initialize(path_or_io_or_logger, options={})
+      super(options)
+
       @logger = if path_or_io_or_logger.kind_of?(Logger) then
                 path_or_io_or_logger
               elsif path_or_io_or_logger.respond_to?(:<<) then

data/lib/counters/memory.rb

@@ -4,7 +4,9 @@ module Counters
   class Memory < Counters::Base
     attr_reader :hits, :latencies, :magnitudes, :pings
 
-    def initialize
+    def initialize(options={})
+      super(options)
+
       @hits       = Hash.new {|h,k| h[k] = 0}
       @magnitudes = Hash.new {|h,k| h[k] = 0}
       @latencies  = Hash.new {|h,k| h[k] = Array.new}
@@ -16,7 +18,7 @@ module Counters
     end
 
     def record_ping(key)
-      @pings[key] = Time.now
+      @pings[key] = Time.now.utc
     end
 
     def record_latency(key, time_in_seconds)

data/lib/counters/redis.rb

@@ -1,12 +1,13 @@
-require "benchmark"
-
 # The redis gem must already be required - we don't require it.
 # This allows callers / users to use any implementation that has the right API.
 
 module Counters
   class Redis < Counters::Base
-    def initialize(redis=Redis.new, base_key="counters")
-      @redis, @base_key = redis, base_key
+    def initialize(redis=Redis.new, options={})
+      super(options)
+
+      @redis    = redis
+      @base_key = options.fetch(:base_key) { raise "Missing :base_key from #{options.inspect}" }
     end
 
     def record_hit(key)
@@ -20,8 +21,8 @@ module Counters
       end
     end
 
-    def ping(key)
-      @redis.hset(@base_key, "pings.#{key}", Time.now.to_i)
+    def record_ping(key)
+      @redis.hset(@base_key, "pings.#{key}", Time.now.utc.to_i)
     end
 
     # Redis requires integer keys, thus we scale all latencies to the nanosecond precision

data/lib/counters/stats_d.rb

@@ -0,0 +1,35 @@
+require "socket"
+
+module Counters
+  class StatsD < Counters::Base
+    attr_reader :host, :port, :socket
+
+    def initialize(host, port, options={})
+      super(options)
+
+      @host, @port = host, port
+      @socket = options.fetch(:socket) { UDPSocket.new }
+    end
+
+    def record_ping(key)
+      socket.send("pings.#{key}:1|c", 0, host, port)
+    end
+
+    def record_hit(key)
+      socket.send("hits.#{key}:1|c", 0, host, port)
+    end
+
+    # StatsD expects millisecond resolution
+    SCALING_FACTOR = 1_000
+
+    def record_latency(key, time_in_seconds)
+      value = "latencies.%s:%d|ms" % [key, time_in_seconds * SCALING_FACTOR]
+      socket.send(value, 0, host, port)
+    end
+
+    def record_magnitude(key, amount)
+      value = "magnitudes.%s:%d|ms" % [key, amount]
+      socket.send(value, 0, host, port)
+    end
+  end
+end

data/lib/counters/version.rb

@@ -1,3 +1,3 @@
 module Counters
-  VERSION = "1.0.2"
+  VERSION = "1.1.0"
 end

data/samples/crawler.rb

@@ -0,0 +1,37 @@
+require "uri"
+require "open-uri"
+require "counters"
+require "nokogiri"
+require "redis"
+require "pp"
+
+# Counter = Counters::Redis.new(Redis.new, :namespace => "crawler", :base_key => "counters")
+Counter = Counters::StatsD.new("127.0.0.1", 8125, :namespace => "crawler")
+# Counter = Counters::Memory.new(:namespace => "crawler")
+# Counter = Counters::File.new(STDOUT, :namespace => "crawler")
+
+urls_to_crawl = ["http://blog.teksol.info/", "http://techcrunch.com/", "http://www.google.com/"]
+
+while url = urls_to_crawl.pop
+  Counter.ping "crawler"
+
+  begin
+    Counter.hit "urls_popped"
+
+    puts "Fetching #{url}"
+    raw_html = Counter.latency "download" do
+      URI.parse(url).read
+    end
+
+    Counter.magnitude "bytes_in", raw_html.length
+
+    parsed_html = Counter.latency "html_parsing" do
+      Nokogiri::HTML(raw_html)
+    end
+  rescue
+    Counter.hit "error"
+  end
+end
+
+pp Counter if Counter.instance_of?(Counters::Memory)
+pp Redis.new.hgetall("counters") if Counter.instance_of?(Counters::Redis)

data/spec/file_counter_spec.rb

@@ -15,6 +15,43 @@ describe Counters::File do
 
   it_should_behave_like "all counters"
 
+  context "#initialize" do
+    it "should accept a namepsace in options" do
+      counter = Counters::File.new(tempfile, :namespace => "wine")
+      counter.namespace.should == "wine"
+    end
+  end
+
+  context "given the counter is namespaced" do
+    it "should namespace the key from #hit" do
+      counter.namespace = "juice"
+      counter.hit "foxglove"
+      tempfile.rewind
+      tempfile.read.should =~ /^#{TIMESTAMP_RE}\s-\shit:\sjuice\.foxglove$/
+    end
+
+    it "should namespace the key from #latency" do
+      counter.namespace = "cocktail"
+      counter.latency "angelica", 200
+      tempfile.rewind
+      tempfile.read.should =~ /^#{TIMESTAMP_RE}\s-\slatency:\scocktail\.angelica\s200s$/
+    end
+
+    it "should namespace the key from #magnitude" do
+      counter.namespace = "brew"
+      counter.magnitude "crocus", 100
+      tempfile.rewind
+      tempfile.read.should =~ /^#{TIMESTAMP_RE}\s-\smagnitude:\sbrew\.crocus 100$/
+    end
+
+    it "should namespace the key from #ping" do
+      counter.namespace = "beer"
+      counter.ping "tulip"
+      tempfile.rewind
+      tempfile.read.should =~ /^#{TIMESTAMP_RE}\s-\sping:\sbeer\.tulip$/
+    end
+  end
+
   it "should log a message to the logfile when a hit is recorded" do
     counter.hit "urls.visited"
     tempfile.rewind

data/spec/memory_counter_spec.rb

@@ -7,6 +7,41 @@ describe Counters::Memory do
 
   it_should_behave_like "all counters"
 
+  context "#initialize" do
+    it "should accept a namepsace in options" do
+      counter = Counters::Memory.new(:namespace => "wine")
+      counter.namespace.should == "wine"
+    end
+  end
+
+  context "given the counter is namespaced" do
+    it "should namespace the key from #hit" do
+      counter.namespace = "juice"
+      counter.hit "foxglove"
+      counter.hits["juice.foxglove"].should == 1
+    end
+
+    it "should namespace the key from #latency" do
+      counter.namespace = "cocktail"
+      counter.latency "angelica", 200
+      counter.latencies["cocktail.angelica"].should == [200]
+    end
+
+    it "should namespace the key from #magnitude" do
+      counter.namespace = "brew"
+      counter.magnitude "crocus", 100
+      counter.magnitudes["brew.crocus"].should == 100
+    end
+
+    it "should namespace the key from #ping" do
+      counter.namespace = "beer"
+      Timecop.freeze(Time.now.utc) do
+        counter.ping "tulip"
+        counter.pings["beer.tulip"].should == Time.now.utc
+      end
+    end
+  end
+
   it "should record a hit with key 'pages.read'" do
     counter.hit "pages.read"
     counter.hits.should have_key("pages.read")
@@ -20,14 +55,14 @@ describe Counters::Memory do
     Timecop.freeze do
       counter.ping "processor.alive"
       counter.pings.should have_key("processor.alive")
-      counter.pings["processor.alive"].strftime("%Y-%m-%d %H:%M:%S").should == Time.now.strftime("%Y-%m-%d %H:%M:%S")
+      counter.pings["processor.alive"].strftime("%Y-%m-%d %H:%M:%S").should == Time.now.utc.strftime("%Y-%m-%d %H:%M:%S")
     end
 
     target_time = Time.now + 9
     Timecop.travel(target_time) do
       counter.ping "processor.alive"
       counter.pings.should have_key("processor.alive")
-      counter.pings["processor.alive"].strftime("%Y-%m-%d %H:%M:%S").should == target_time.strftime("%Y-%m-%d %H:%M:%S")
+      counter.pings["processor.alive"].strftime("%Y-%m-%d %H:%M:%S").should == target_time.utc.strftime("%Y-%m-%d %H:%M:%S")
     end
   end
 

data/spec/redis_counter_spec.rb

@@ -7,7 +7,7 @@ describe Counters::Redis, "integration tests" do
   end
 
   let :counter do
-    Counters::Redis.new(redis, "counters")
+    Counters::Redis.new(redis, :base_key => "counters")
   end
 
   before(:each) do
@@ -16,6 +16,41 @@ describe Counters::Redis, "integration tests" do
 
   it_should_behave_like "all counters"
 
+  context "#initialize" do
+    it "should accept a namepsace in options" do
+      counter = Counters::Redis.new(redis, :namespace => "wine", :base_key => "counters")
+      counter.namespace.should == "wine"
+    end
+  end
+
+  context "given the counter is namespaced" do
+    it "should namespace the key from #hit" do
+      counter.namespace = "juice"
+      counter.hit "foxglove"
+      redis.hget("counters", "hits.juice.foxglove").should == "1"
+    end
+
+    it "should namespace the key from #latency" do
+      counter.namespace = "cocktail"
+      counter.latency "angelica", 200
+      redis.hget("counters", "latencies.cocktail.angelica.nanoseconds").should == (200 * Counters::Redis::SCALING_FACTOR).to_s
+    end
+
+    it "should namespace the key from #magnitude" do
+      counter.namespace = "brew"
+      counter.magnitude "crocus", 100
+      redis.hget("counters", "magnitudes.brew.crocus.value").should == "100"
+    end
+
+    it "should namespace the key from #ping" do
+      counter.namespace = "beer"
+      Timecop.freeze(Time.now.utc) do
+        counter.ping "tulip"
+        redis.hget("counters", "pings.beer.tulip").should == Time.now.utc.to_i.to_s
+      end
+    end
+  end
+
   it "should record 2 hits on 'pages.read'" do
     2.times { counter.hit "pages.read" }
     redis.hkeys("counters").should == ["hits.pages.read"]

data/spec/spec_helper.rb

@@ -9,13 +9,13 @@ end
 
 shared_examples_for "all counters" do
   it "should raise a ArgumentError when the key includes invalid chars" do
-    lambda { counter.hit "hit!"        } .should raise_error(ArgumentError)
-    lambda { counter.hit "hit counter" } .should raise_error(ArgumentError)
-    lambda { counter.hit "boy.hit?"    } .should raise_error(ArgumentError)
-    lambda { counter.hit "hit/a"       } .should raise_error(ArgumentError)
-    lambda { counter.hit "hit-a"       } .should raise_error(ArgumentError)
-    lambda { counter.hit ""            } .should raise_error(ArgumentError)
-    lambda { counter.hit nil           } .should raise_error(ArgumentError)
+    lambda { counter.hit "hit!"        }.should raise_error(ArgumentError)
+    lambda { counter.hit "hit counter" }.should raise_error(ArgumentError)
+    lambda { counter.hit "boy.hit?"    }.should raise_error(ArgumentError)
+    lambda { counter.hit "hit/a"       }.should raise_error(ArgumentError)
+    lambda { counter.hit "hit-a"       }.should raise_error(ArgumentError)
+    lambda { counter.hit ""            }.should raise_error(ArgumentError)
+    lambda { counter.hit nil           }.should raise_error(ArgumentError)
   end
 
   it "should not raise ArgumentError when the key includes a number" do

data/spec/statsd_counter_spec.rb

@@ -0,0 +1,80 @@
+require "spec_helper"
+
+describe Counters::StatsD do
+  let :socket do
+    double("socket")
+  end
+
+  let :host do
+    "127.0.0.1"
+  end
+
+  let :port do
+    8125
+  end
+
+  let :counter do
+    Counters::StatsD.new(host, port, :socket => socket)
+  end
+
+  context "" do
+    before(:each) do
+      socket.as_null_object
+    end
+
+    it_should_behave_like "all counters"
+  end
+
+  it "should record a hit as an increment of the key" do
+    socket.should_receive(:send).with("hits.tweets_received:1|c", 0, host, port)
+    counter.hit "tweets_received"
+  end
+
+  it "should record a magnitude as a timer" do
+    socket.should_receive(:send).with("magnitudes.bytes_in:381|ms", 0, host, port)
+    counter.magnitude "bytes_in", 381
+  end
+
+  it "should record a latency as a timer" do
+    socket.should_receive(:send).with("latencies.json_parsing:9|ms", 0, host, port)
+    counter.latency "json_parsing", 0.009
+  end
+
+  it "should record a ping as a counter" do
+    socket.should_receive(:send).with("pings.tweet_processor:1|c", 0, host, port)
+    counter.ping "tweet_processor"
+  end
+
+  context "#initialize" do
+    it "should accept a namepsace in options" do
+      counter = Counters::StatsD.new(host, port, :namespace => "wine")
+      counter.namespace.should == "wine"
+    end
+  end
+
+  context "given the counter is namespaced" do
+    it "should namespace the key from #hit" do
+      counter.namespace = "juice"
+      socket.should_receive(:send).with(/^hits\.juice\.foxglove\b/, anything, anything, anything)
+      counter.hit "foxglove"
+    end
+
+    it "should namespace the key from #latency" do
+      counter.namespace = "cocktail"
+      socket.should_receive(:send).with(/^latencies\.cocktail\.angelica\b/, anything, anything, anything)
+      counter.latency "angelica", 200
+    end
+
+    it "should namespace the key from #magnitude" do
+      counter.namespace = "brew"
+      socket.should_receive(:send).with(/^magnitudes\.brew\.crocus\b/, anything, anything, anything)
+      counter.magnitude "crocus", 100
+    end
+
+    it "should namespace the key from #ping" do
+      counter.namespace = "beer"
+      socket.should_receive(:send).with(/^pings\.beer\.tulip\b/, anything, anything, anything)
+      counter.ping "tulip"
+    end
+  end
+end

metadata

@@ -1,12 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: counters
 version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 0
-  - 2
-  version: 1.0.2
+  prerelease: 
+  version: 1.1.0
 platform: ruby
 authors: 
 - "Fran\xC3\xA7ois Beausoleil"
@@ -14,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-21 00:00:00 -04:00
+date: 2011-03-29 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -25,8 +21,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id001
@@ -38,8 +32,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id002
@@ -51,8 +43,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id003
@@ -64,8 +54,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id004
@@ -92,11 +80,14 @@ files:
 - lib/counters/file.rb
 - lib/counters/memory.rb
 - lib/counters/redis.rb
+- lib/counters/stats_d.rb
 - lib/counters/version.rb
+- samples/crawler.rb
 - spec/file_counter_spec.rb
 - spec/memory_counter_spec.rb
 - spec/redis_counter_spec.rb
 - spec/spec_helper.rb
+- spec/statsd_counter_spec.rb
 has_rdoc: true
 homepage: https://github.com/francois/counters
 licenses: []
@@ -111,21 +102,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Provides an API to record any kind of metrics within your system
@@ -134,3 +121,4 @@ test_files:
 - spec/memory_counter_spec.rb
 - spec/redis_counter_spec.rb
 - spec/spec_helper.rb
+- spec/statsd_counter_spec.rb