ruby-druid 0.1.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NjkxZmU4ODgzYmYwNzJhMTU1NDY4YjU1OGYzOTBhMmM2MzJjZDE2Ng==
+  data.tar.gz: !binary |-
+    ODQ3YWFlYThkNzQ0YWMwNjU3MTJjYjQ5Y2QzMjAxYTMyNTYwZjBiYQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZjVjOWJiZDEyNjA0ZDkwM2UzZTYyNDI5YTlkN2NkZjI5MmVjODBmNzg0Yjky
+    MGExM2JkOTM1OTliYTA3NmE3Y2VmOTk1ODhmZDI4NTUxYmMwOGQwYTZkZTRh
+    Njk1MzZhMTc5NmQ0OTQ4NmVjNjE4OWI0ZjE4M2M3Yzk4MDEwYmY=
+  data.tar.gz: !binary |-
+    MTkxZGFjYzAzNDk5NWZhY2FkM2E2ODYzNmY1MzkzZTE4M2U0MzFhMDViZWI0
+    YmNhOTZiNzlhZWFlYWI2ZDRiOWY4NGE2NmQ1Y2I3OTljNjIyNWEwNzczYzRl
+    MWZhZmQwZTA4OThiMjA4ODAyYjE0MDU0YzBjMzViY2I3ZTJjYzM=

data/.gitignore

@@ -0,0 +1,5 @@
+Gemfile.lock
+.driplrc
+ruby-druid-0.0.1.gem
+spec/test.json
+tmp/rspec_guard_result

data/Gemfile

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test, :development do
+  gem 'guard'
+  gem 'guard-bundler'
+  gem 'guard-rspec'
+  gem 'rb-fsevent'
+  gem 'rspec'
+  gem 'ruby_gntp'
+  gem 'webmock'
+  gem 'debugger'
+end
+
+group :console do
+  gem 'activesupport'
+  gem 'awesome_print'
+  gem 'ripl'
+  gem 'terminal-table'
+end

data/Guardfile

@@ -0,0 +1,10 @@
+# More info at https://github.com/guard/guard#readme
+guard :bundler do
+  watch('Gemfile')
+end
+
+guard :rspec, :cli => '--color --format nested' do
+	watch(%r{^spec/.+_spec\.rb$})
+	watch(%r{^(.+)\.rb$})  	{|m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2013 madvertise Mobile Advertising GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,286 @@
+# ruby-druid
+
+[![Code Climate](https://codeclimate.com/github/madvertise/ruby-druid.png)](https://codeclimate.com/github/madvertise/ruby-druid)
+
+A ruby client for [druid](https://github.com/madvertise/druid).
+
+ruby-druid generates complete JSON queries by chaining methods.
+The resulting JSON can be send directly to a druid server or handled seperatly.
+
+## bin/dripl
+
+ruby-druid now includes a repl:
+
+```ruby
+$ bin/dripl
+>> metrics
+[
+    [0] "actions"
+]
+
+>> dimensions
+[
+    [0] "actions"
+]
+
+>> long_sum(:actions)
++---------+
+| actions |
++---------+
+|   98575 |
++---------+
+
+>> long_sum(:actions)[-7.days].granularity(:day)
++-------------------------------+----------+
+| timestamp                     | actions  |
++-------------------------------+----------+
+| 2013-03-28T00:00:00.000+01:00 |   93371  |
+| 2013-03-29T00:00:00.000+01:00 |   448200 |
+| 2013-03-30T00:00:00.000+01:00 |   117167 |
+| 2013-03-31T00:00:00.000+01:00 |   828321 |
+| 2013-04-01T00:00:00.000+02:00 |   261578 |
+| 2013-04-02T00:00:00.000+02:00 |   05149  |
+| 2013-04-03T00:00:00.000+02:00 |   27512  |
+| 2013-04-04T00:00:00.000+02:00 |   18897  |
++-------------------------------+----------+
+
+>> long_sum(:actions)[-7.days].granularity(:day).properties
+{
+      :dataSource => "events",
+     :granularity => {
+            :type => "period",
+          :period => "P1D",
+        :timeZone => "Europe/Berlin"
+    },
+       :intervals => [
+        [0] "2013-03-28T00:00:00+01:00/2013-04-04T11:57:20+02:00"
+    ],
+       :queryType => :groupBy,
+    :aggregations => [
+        [0] {
+                 :type => "longSum",
+                 :name => :actions,
+            :fieldName => :actions
+        }
+    ]
+}
+```
+
+## Getting started
+
+In your Gemfile:
+
+```ruby
+  gem 'ruby-druid'
+```
+
+In your code:
+
+```ruby
+  require 'druid'
+```
+
+## Usage
+
+```ruby
+Druid::Client.new('zk1:2181,zk2:2181/druid').query('service/source')
+```
+
+returns a query object on which all other methods can be called to create a full and valid druid query.
+
+A query object can be sent like this:
+
+```ruby
+Druid::Client.new('zk1:2181,zk2:2181/druid').query('service/source').send
+#or
+client = Druid::Client.new('zk1:2181,zk2:2181/druid')
+query = Druid::Query.new('service/source')
+client.send(query)
+```
+
+The `send` method returns the parsed response from the druid server as an array.
+If the response is not empty it contains one `ResponseRow` object for each row.
+The timestamp by can be received by a method with the same name (i.e. `row.timestamp`),
+all row values by hashlike syntax (i.e. `row['dimension'])
+
+### group_by
+
+Sets the dimensions to group the data.
+
+`queryType` is set automatically to `groupBy`.
+
+
+```ruby
+Druid::Query.new('service/source').group_by([:dimension1, :dimension2])
+```
+
+### long_sum
+
+```ruby
+Druid::Query.new('service/source').long_sum([:aggregate1, :aggregate2])
+```
+
+### postagg
+
+A simple syntax for post aggregations with +,-,/,* can be used like:
+
+```ruby
+query = Druid::Query.new('service/source').long_sum([:aggregate1, :aggregate2])
+
+query.postagg{(aggregate2 + aggregate2).as output_field_name}
+```
+
+Required fields for the postaggregation are fetched automatically by the library.
+
+### interval
+
+The interval for the query takes a string with date and time or objects that provide a `iso8601` method
+
+```ruby
+query = Druid::Query.new('service/source').long_sum(:aggregate1)
+
+query.interval("2013-01-01T00", Time.now)
+```
+
+### granularity
+
+granularity can be `:all`, `:none`, `:minute`, `:fifteen_minute`, `:thirthy_minute`, `:hour` or `:day`.
+
+It can also be a period granularity as described in https://github.com/metamx/druid/wiki/Granularities.
+
+The period `'day'` or `:day` will be interpreted as `'P1D'`.
+
+If a period granularity is specifed, the (optional) second parameter is a time zone. It defaults
+to the machines local time zone.
+
+I.E:
+```ruby
+query = Druid::Query.new('service/source').long_sum(:aggregate1)
+
+query.granularity(:day)
+```
+
+is (on my box) the same as
+
+```ruby
+query = Druid::Query.new('service/source').long_sum(:aggregate1)
+
+query.granularity('P1D', 'Europe/Berlin')
+```
+
+## having (for metrics)
+
+### having >
+
+```ruby
+Druid::Query.new('service/source').having{metric > 10}
+```
+
+### having <
+
+```ruby
+Druid::Query.new('service/source').having{metric < 10}
+```
+
+## filter (for dimensions)
+
+Filters are set by the `filter` method. It takes a block or a hash as parameter.
+
+Filters can be chained `filter{...}.filter{...}`
+
+### filter == , eq
+
+```ruby
+Druid::Query.new('service/source').filter{dimension.eq 1}
+
+#this is the same as
+
+Druid::Query.new('service/source').filter{dimension == 1}
+```
+
+### filter != , neq
+
+```ruby
+Druid::Query.new('service/source').filter{dimension.neq 1}
+
+#this is the same as
+
+Druid::Query.new('service/source').filter{dimension != 1}
+```
+
+### filter and
+
+a logical or than can combine all other filters
+
+```ruby
+Druid::Query.new('service/source').filter{dimension.neq 1 & dimension2.neq 2}
+```
+
+### filter or
+
+a logical or than can combine all other filters
+
+```ruby
+Druid::Query.new('service/source').filter{dimension.neq 1 | dimension2.neq 2}
+```
+
+### filter not
+
+a logical not than can negate all other filter
+
+```ruby
+Druid::Query.new('service/source').filter{!dimension.eq(1)}
+```
+
+### filter in
+
+This filter creates a set of equals filters in an and filter.
+
+```ruby
+Druid::Query.new('service/source').filter{dimension.in(1,2,3)}
+```
+
+### filter with hash syntax
+
+sometimes it can be useful to use a hash syntax for filtering
+for example if you already get them from a list or parameterhash
+
+```ruby
+Druid::Query.new('service/source').filter{dimension => 1, dimension1 =>2, dimension2 => 3}
+
+#this is the same as
+
+Druid::Query.new('service/source').filter{dimension.eq(1) & dimension1.eq(2) & dimension2.eq(3)}
+```
+
+### filter >, <, >=, <=
+
+```ruby
+Druid::Query.new('service/source').filter{dimension >= 1}
+```
+
+### filter javascript
+
+```ruby
+Druid::Query.new('service/source').filter{a.javascript('dimension >= 1 && dimension < 5')}
+
+#this also the same as
+
+Druid::Query.new('service/source').filter{(dimension >= 1) & (dimension < 5)}
+```
+
+## Acknowledgements
+
+Post aggregation expression parsing built with the help of [Squeel](https://github.com/ernie/squeel).
+
+## Contributions
+
+ruby-druid is developed by madvertise Mobile Advertising GmbH
+
+You can support us on different ways:
+
+* Use ruby-druid, and let us know if you encounter anything that's broken or missing.
+  A failing spec is great. A pull request with your fix is even better!
+* Spread the word about ruby-druid on Twitter, Facebook, and elsewhere.
+* Work with us at madvertise on awesome stuff like this.
+  [Read the job description](http://madvertise.com/software-developer-ruby-fm-berlin) and send a mail to careers@madvertise.com.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/dripl

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$:.unshift(File.join(File.expand_path("../..", __FILE__), 'lib'))
+
+$0 = "dripl"
+
+def zookeeper(value)
+  @zk_uri = value
+end
+
+def uri(value)
+  puts "using 'uri' in the config is deprecated, use 'zookeeper' instead"
+  zookeeper value
+end
+
+def source(value)
+  @source = value
+end
+
+def options(value)
+  @options = value
+end
+
+begin
+  driplrc = File.read(File.join(File.expand_path("../..", __FILE__), '.driplrc'))
+rescue
+  puts "You need to create a .driplrc, take a look at dot_driplrc_example"
+  exit 1
+end
+
+instance_eval(driplrc)
+
+unless @zk_uri || (@options && @options[:static_setup])
+  puts "Your .driplrc is incomplete, please fix"
+  exit 1
+end
+
+require 'druid/console'
+Druid::Console.new(@zk_uri, @source, @options)

data/dot_driplrc_example

@@ -0,0 +1,12 @@
+## your zookeeper config. For static scenarios (i.e. ssh tunnels) see options
+##
+# zookeeper "localhost:2181/druid"
+
+## using options, you can disable zookeeper lookup
+## options[:static_setup], the key is the source name, the value is the brokers post uri
+##
+# options :static_setup => { 'example/events' => 'http://localhost:8080/druid/v2/' }
+
+## dripl will default to use the first available data source. use this to override
+##
+# source "example/events"

data/lib/druid.rb

@@ -0,0 +1,8 @@
+require 'druid/client'
+require 'druid/query'
+require 'druid/response_row'
+require 'druid/zoo_handler'
+
+module Druid
+
+end

data/lib/druid/client.rb

@@ -0,0 +1,95 @@
+module Druid
+  class Client
+    TIMEOUT = 2 * 60 * 1000
+
+    def initialize(zookeeper_uri, opts = nil)
+      opts ||= {}
+
+      if opts[:static_setup] && !opts[:fallback]
+        @static = opts[:static_setup]
+      else
+        @backup = opts[:static_setup] if opts[:fallback]
+        zookeeper_caching_management!(zookeeper_uri, opts)
+      end
+    end
+
+    def send(query)
+      uri = data_source_uri(query.source)
+      raise "data source #{query.source} (currently) not available" unless uri
+
+      req = Net::HTTP::Post.new(uri.path, initheader = {'Content-Type' =>'application/json'})
+      req.body = query.to_json
+      puts req.body
+
+      response = Net::HTTP.new(uri.host, uri.port).start do |http|
+        http.read_timeout = TIMEOUT
+        http.request(req)
+      end
+
+      if response.code == "200"
+        JSON.parse(response.body).map{ |row| ResponseRow.new(row) }
+      else
+        raise "Request failed: #{response.code}: #{response.body}"
+      end
+    end
+
+    def query(id, &block)
+      uri = data_source_uri(id)
+      raise "data source #{id} (currently) not available" unless uri
+      query = Query.new(id, self)
+      return query unless block
+
+      send query
+    end
+
+    def zookeeper_caching_management!(zookeeper_uri, opts)
+      @zk = ZooHandler.new(zookeeper_uri, opts)
+
+      unless opts[:zk_keepalive]
+        @cached_data_sources = @zk.data_sources unless @zk.nil?
+
+        @zk.close!
+      end
+    end
+
+    def ds
+      @cached_data_sources || (@zk.data_sources unless @zk.nil?)
+    end
+
+    def data_sources
+      (ds.nil? ? @static : ds).keys
+    end
+
+    def data_source_uri(source)
+      uri = (ds.nil? ? @static : ds)[source]
+      begin
+        return URI(uri) if uri
+      rescue
+         return URI(@backup) if @backup
+      end
+    end
+
+    def data_source(source)
+      uri = data_source_uri(source)
+      raise "data source #{source} (currently) not available" unless uri
+
+      meta_path = "#{uri.path}datasources/#{source.split('/').last}"
+
+      req = Net::HTTP::Get.new(meta_path)
+
+      response = Net::HTTP.new(uri.host, uri.port).start do |http|
+        http.read_timeout = TIMEOUT
+        http.request(req)
+      end
+
+      if response.code == "200"
+        meta = JSON.parse(response.body)
+        meta.define_singleton_method(:dimensions) { self['dimensions'] }
+        meta.define_singleton_method(:metrics) { self['metrics'] }
+        meta
+      else
+        raise "Request failed: #{response.code}: #{response.body}"
+      end
+    end
+  end
+end