marquise 0.1.0

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/Guardfile

@@ -0,0 +1,12 @@
+guard 'spork', :rspec_env => { 'RACK_ENV' => 'test' } do
+  watch('Gemfile')
+  watch('spec/spec_helper.rb') { :rspec }
+end
+
+guard 'rspec',
+      :cmd          => "rspec --drb",
+      :all_on_start => true do
+  watch('lib/marquise/ffi.rb')
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| Dir["spec/#{m[1].gsub('/', '_')}*_spec.rb"] }
+end

data/README.md

@@ -0,0 +1,110 @@
+# Marquise for Ruby
+
+This package contains Ruby bindings for the
+[Marquise](https://github.com/anchor/libmarquise) data vault insertion
+library.  You should use it if you want to send data points to
+[Vaultaire](https://github.com/anchor/vaultaire) in Ruby code.
+
+
+# Installation
+
+You should make sure you have libmarquise installed correctly first -- this
+package will not work properly without it.
+
+Then, simply install the `marquise` gem from Rubygems:
+
+    gem install marquise
+
+And you should be good to go.
+
+
+# Usage
+
+Simply create a `Marquise` object, with the URL of the zeroMQ broker you
+wish to use:
+
+    s = Marquise.new('tcp://chateau.example.com:5560')
+
+When you're finished with your instance, you should close it, to cleanup
+file handles and such:
+
+    s.close
+
+If you're someone who doesn't like to stuff around with manual resource
+release, there's also the very Rubyish block form:
+
+    Marquise.open('tcp://chateau.example.com:5560') do |s|
+        # Do things with s
+    end
+
+Which will create a new Marquise object, yield it to your block, then close
+off the thing when you're done.
+
+
+# Sending data points
+
+You send data points through a `Marquise` instance by `tell`ing it; these
+are the simplest possible forms for each type of data Vaultaire can store:
+
+    s.tell(42)              # Send an integer
+    s.tell(Math::PI)        # Send a float
+    s.tell("Hello World")   # Send a string
+    s.tell("Hello World".encode("ASCII-8BIT")  # Send a binary blob
+    s.tell                  # Send a counter increment
+
+
+## Types and type conversion
+
+Vaultaire is able to store:
+
+ * UTF-8 Strings;
+ * Binary blobs;
+ * Integers, in the range -(2^63)+1 to (2^63)-1;
+ * Double-precision floating-point numbers;
+ * Counter increments.
+
+Marquise will automatically determine the most appropriate type to store the
+value in, given the type of the first argument passed to `#tell`, by
+applying the following rules:
+
+ 1. If the first argument `is_a? Hash`, then store a counter increment with
+    a timestamp of `Time.now`;
+
+ 1. If the first argument `is_a? Time`, then store a counter increment with
+    a timestamp of the first argument;
+
+ 1. If the first argument has a `#to_str` method, then call that and store
+    the result as a binary (if the `#encoding` method returns `ASCII-8BIT`
+    or the string does not encode cleanly to UTF-8), or as a UTF-8 string
+    otherwise;
+ 
+ 1. If the first argument has a `#integer?` method, then call that, and
+    based on whether the result of that is `true` or `false`, store either
+    an integer or float.
+
+ 1. Otherwise, raise `ArgumentError`, as we were unable to determine how to
+    convert the provided argument into something that would be understood.
+
+
+## Timestamps
+
+Vaultaire stores all data points with a timestamp.  If you want to specify
+the timestamp that should be associated with your data point, you should
+pass a `Time` object to `Marquise#send` as the first non-value argument
+(that is, the first argument if storing a counter increment, or the second
+argument otherwise).  If you do not specify a timestamp, then `Time.now`
+will be called within the `#send` method, and that time used as the
+timestamp.
+
+
+## Tagging data points
+
+To differentiate data points from different series, Vaultaire allows the
+specification of arbitrary key/value hashes, which you specify in the usual
+Rubyesque fashion:
+
+    s.tell(42, :answer => 'ultimate')
+    s.tell(Math::PI, constant: 'yes', closest_to: 3)
+
+Note that all keys and values will be converted into strings using `#to_s`,
+because Vaultaire only supports strings in its tag hashes.

data/Rakefile

@@ -0,0 +1,39 @@
+require 'rubygems'
+require 'bundler'
+
+task :default => :test
+
+begin
+	Bundler.setup
+rescue Bundler::BundlerError => e
+	$stderr.puts e.message
+	$stderr.puts "Run `bundle install` to install missing gems"
+	exit e.status_code
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rdoc/task'
+
+Rake::RDocTask.new do |rd|
+	rd.main = "README.md"
+	rd.title = 'marquise'
+	rd.rdoc_files.include("README.md", "lib/**/*.rb")
+end
+
+require 'git-version-bump/rake-tasks'
+
+desc "Run guard"
+task :guard do
+	Bundler.setup(:default, :test)
+	require 'guard'
+	::Guard.start(:clear => true)
+	while ::Guard.running do
+		sleep 0.5
+	end
+end
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new :test do |t|
+	t.pattern = "spec/**/*_spec.rb"
+end

data/lib/marquise.rb

@@ -0,0 +1,212 @@
+require 'marquise/ffi'
+require 'ffi/dry/errno'
+
+# A Vaultaire data point transport
+#
+# Instances of `Marquise` send data points that they are told about to a
+# Vaultaire data store.  It has a very simple interface that hides a lot
+# of complexity.
+class Marquise
+	include ::FFI::DRY::ErrnoHelper
+	
+	# Open a Marquise consumer and (optionally) yield it to a block
+	#
+	# With no associated block, `Marquise.open` is a synonym for
+	# `Marquise.new`.  If the optional block is given, a newly created
+	# `Marquise` object will be passed to the block, and will then be
+	# automatically closed when the block terminates, and the value of
+	# the block returned.
+	def self.open(zmq_url, batch_period = 5)
+		m = Marquise.new(zmq_url, batch_period)
+		rv = m
+		
+		if block_given?
+			begin
+				rv = yield m
+			ensure
+				m.close
+			end
+		end
+		
+		rv
+	end
+	
+	# Create a new `Marquise` transport object
+	#
+	# `zmq_url` is the URL to your ZeroMQ broker associated with the
+	# Vaultaire system you're dumping data into.  `batch_period` is optional
+	# (defaults to `5`) is the number of seconds between "flushes" of data
+	# points to ZeroMQ.  It can be a floating-point number if you wish to
+	# have sub-second flushes.  Increasing the `batch_period` increases the
+	# possibility of losing data points in the event of a spectacular
+	# failure, but also improves performance.
+	def initialize(zmq_url, batch_period = 5)
+		@consumer = Marquise::FFI.marquise_consumer_new(zmq_url, batch_period)
+		
+		if @consumer.nil?
+			raise RuntimeError,
+			      "libmarquise failed; check syslog (no, seriously)"
+		end
+		
+		@connections = {}
+
+		@janitor = Janitor.new(@consumer, @connections)
+		ObjectSpace.define_finalizer(self, @janitor)
+	end
+	
+	def tell(*args)
+		val, ts, opts = parse_tell_opts(args)
+				
+		k, v, len = if opts.length == 0
+			[nil, nil, 0]
+		else
+			[
+			 Marquise::FFI.pointer_list_from_string_array(opts.keys),
+			 Marquise::FFI.pointer_list_from_string_array(opts.values),
+			 opts.length
+			]
+		end
+			
+		rv = if val.nil?
+			Marquise::FFI.marquise_send_counter(
+			                connection,
+			                k,
+			                v,
+			                len,
+			                ts.to_f * 1_000_000_000
+			              )
+		elsif val.respond_to? :to_str and val.respond_to? :encoding
+			s = val.to_str
+			method = nil
+			
+			if s.encoding.to_s == 'ASCII-8BIT' or !s.force_encoding('UTF-8').valid_encoding?
+				method = :marquise_send_binary
+			else
+				method = :marquise_send_text
+				s = s.encode('UTF-8')
+			end
+
+			Marquise::FFI.send(
+			                method,
+			                connection,
+			                k,
+			                v,
+			                len,
+			                s,
+			                s.length,
+			                ts.to_f * 1_000_000_000
+			              )
+		elsif val.respond_to? :integer? and val.integer?
+			if val < -(2**63)+1 or val > (2**63)-1
+				raise ArgumentError,
+				      "Integer out of range for Marquise#tell"
+			end
+			
+			Marquise::FFI.marquise_send_int(
+			                connection,
+			                k,
+			                v,
+			                len,
+			                val,
+			                ts.to_f * 1_000_000_000
+			              )
+		elsif val.respond_to? :integer? and !val.integer?
+			Marquise::FFI.marquise_send_real(
+			                connection,
+			                k,
+			                v,
+			                len,
+			                val,
+			                ts.to_f * 1_000_000_000
+			              )
+		end
+		
+		if rv == -1
+			raise errno_exception
+		end
+	end
+	
+	# Close a Marquise instance
+	#
+	# This must be called when you are done with your Marquise instance, to
+	# avoid leaving memory and file descriptors.
+	def close
+		@janitor.call
+		ObjectSpace.undefine_finalizer(self)
+		@connections = {}
+		@consumer = nil
+	end
+	
+	# :stopdoc:
+	# Initialize a connection
+	#
+	# You should rarely have to call this method yourself; Marquise will do
+	# it automatically for you when required.
+	def connect
+		th = Thread.current
+		
+		return if @connections[th]
+		
+		@connections[th] = Marquise::FFI.marquise_connect(@consumer)
+		
+		if @connections[th].nil?
+			raise RuntimeError.new("marquise_connect() failed... consult syslog (no, seriously)")
+		end
+		
+		nil
+	end
+	
+	# Get the connection pointer for the current thread
+	def connection
+		self.connect
+		
+		@connections[Thread.current]
+	end
+	
+	# A helper class to cleanup Marquise consumers.  We can't just do the
+	# obvious, which would be to create a Proc inside `Marquise.initialize`,
+	# because that would leave a reference to the object laying around and
+	# we'd never get GC'd.  So we do this instead.  I stole this technique
+	# from Tempfile; blame them if this is insane.
+	class Janitor
+		def initialize(ptr, conns)
+			@ptr = ptr
+			@conns = conns
+		end
+		
+		def call(*args)
+			@conns.values.each { |c| Marquise::FFI.marquise_close(c) }
+			Marquise::FFI.marquise_consumer_shutdown(@ptr)
+		end
+	end
+	
+	private
+	def parse_tell_opts(args)
+		orig_args = args.dup
+		
+		val  = nil
+		ts   = Time.now
+		opts = {}
+		
+		if ((args[0].respond_to? :to_str and args[0].respond_to? :encoding) or
+		    args[0].respond_to? :integer?)
+			val = args.shift
+		end
+		
+		if args[0].is_a? Time
+			ts = args.shift
+		end
+
+		if args[0].is_a? Hash
+			opts = args.shift
+		end
+		
+		unless args.empty?
+			raise ArgumentError,
+			      "Invalid call to Marquise#tell (you passed '#{orig_args.map(&:inspect).join(', ')}')"
+		end
+
+		[val, ts, opts]
+	end
+	# :startdoc:
+end

data/lib/marquise/ffi.rb

@@ -0,0 +1,65 @@
+require 'ffi'
+
+class Marquise
+	module FFI
+		extend ::FFI::Library
+		ffi_lib 'libmarquise.so.1'
+		
+		attach_function :marquise_consumer_new, [ :string, :double ], :pointer
+		attach_function :marquise_consumer_shutdown, [ :pointer ], :void
+		
+		attach_function :marquise_connect, [ :pointer ], :pointer
+		attach_function :marquise_close,   [ :pointer ], :void
+		
+		attach_function :marquise_send_text, [ :pointer,
+		                                       :pointer,
+		                                       :pointer,
+		                                       :size_t,
+		                                       :string,
+		                                       :size_t,
+		                                       :uint64 ],
+		                                     :int
+		attach_function :marquise_send_int, [ :pointer,
+		                                      :pointer,
+		                                      :pointer,
+		                                      :size_t,
+		                                      :int64,
+		                                      :uint64 ],
+		                                    :int
+		attach_function :marquise_send_real, [ :pointer,
+		                                       :pointer,
+		                                       :pointer,
+		                                       :size_t,
+		                                       :double,
+		                                       :uint64 ],
+		                                     :int
+		attach_function :marquise_send_counter, [ :pointer,
+		                                          :pointer,
+		                                          :pointer,
+		                                          :size_t,
+		                                          :uint64 ],
+		                                        :int
+		attach_function :marquise_send_binary, [ :pointer,
+		                                         :pointer,
+		                                         :pointer,
+		                                         :size_t,
+		                                         :pointer,
+		                                         :size_t,
+		                                         :uint64 ],
+		                                       :int
+
+		# Common helper to take an array of strings (or things that we'll turn
+		# into strings) and turn it into an FFI::MemoryPointer containing
+		# FFI::MemoryPointers for each string.
+		def self.pointer_list_from_string_array(ary)
+			ptrs = []
+			ary.each { |e| ptrs << ::FFI::MemoryPointer.from_string(e.to_s) }
+			ptrs << nil
+			
+			ptr = ::FFI::MemoryPointer.new(:pointer, ptrs.length)
+			ptrs.each_with_index { |p, i| ptr[i].write_pointer(p) }
+			
+			ptr
+		end
+	end
+end