td-querier 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af5159b1036b3b9b0e353d5c9cc41a087d86a453
-  data.tar.gz: 222cb3b62bd1a6b3e13e262f49090bd0ba3642ab
+  metadata.gz: 478227b02106d0610c2662f2507c894bf32bd393
+  data.tar.gz: ead6a5f69c35e4c6d0e167ea7a93c90889e12aca
 SHA512:
-  metadata.gz: d608f9780b1880a545e23e67c25f1ab4cc545ab3524d4a31c9a974b41940cb1c950b44e05f5f1d97ddb6940f7a402c72fd0bdd13076f1c644f465bd52620a8cf
-  data.tar.gz: 9c0516e64a687014e86102c149912e901f484c2af607c4cfd4bc10c782e3c24be300291abf7c816c33f2a5ff3de83d66658a2bcfbe406daa946f57ab404a5ccf
+  metadata.gz: 013197d771d0710df0c51823d2975b4b2e262ff039e926120c0e6cb40d559b0818060f862b5837914143e13738189f44ae83e1c57448f3c7430deddfaca5bf45
+  data.tar.gz: 735e9f9635ec2cb22651d4effe909cdbd5ce40e817a4ad4bbe6da34781012d761fef867cf0a8703b2faccf9cee12615ba923180b64604af4e5554650e059553b

data/Gemfile.lock

@@ -1,24 +1,28 @@
 PATH
   remote: .
   specs:
-    td-querier (0.0.2)
-      sidekiq (>= 2.7.2)
-      td (>= 0.10.77)
+    td-querier (0.0.5)
+      sidekiq (~> 2.7.2)
+      td (~> 0.10.73)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    celluloid (0.14.0)
+    celluloid (0.12.4)
+      facter (>= 1.6.12)
       timers (>= 1.0.0)
     connection_pool (1.0.0)
     diff-lcs (1.2.4)
+    facter (1.6.17)
     fluent-logger (0.4.5)
       msgpack (= 0.4.7)
       yajl-ruby (~> 1.0)
     hirb (0.7.1)
     json (1.7.7)
     msgpack (0.4.7)
+    multi_json (1.7.3)
     parallel (0.5.21)
+    rake (10.0.4)
     redis (3.0.4)
     redis-namespace (1.3.0)
       redis (~> 3.0.0)
@@ -31,11 +35,11 @@ GEM
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.13.1)
     rubyzip (0.9.9)
-    sidekiq (2.12.0)
-      celluloid (>= 0.14.0)
-      connection_pool (>= 1.0.0)
-      json
-      redis (>= 3.0)
+    sidekiq (2.7.2)
+      celluloid (~> 0.12.0)
+      connection_pool (~> 1.0)
+      multi_json (~> 1)
+      redis (~> 3)
       redis-namespace
     td (0.10.77)
       hirb (>= 0.4.5)
@@ -59,5 +63,6 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  rake
   rspec
   td-querier!

data/README.md

@@ -1,7 +1,51 @@
-# Treasure Data and Sidekiq awesomeness
+## Treasure Data and Sidekiq awesomeness
 
-##Installation
+###Concept
+Treasure data jobs take sometime to finish, and in most scenarios waiting is not really an option. 
 
+Td-querier will create a Sidekiq job with the job_id of your Treasure Data Queries and will check if the job has finished.
+
+If it is finished, it will send a callback to continue your data process.
+
+If not it will reschedule itself until the job is done.
+
+###Installation
     $ gem install td-querier
 
-##Configuration
+###Usage
+```
+querier = Querier.new("TREASURE_DATA_API_KEY")
+database_name = 'my_td_database_name'
+query_text = 'select count(*) from my_table'
+options = {:klass=>"MyClass", :method=>"my_method", :results => "true"} #See Options section for this one
+
+#Optional
+on_demand_path = 'mysql://user:password@host/database/table' #will insert the result of your query into another table
+priority = 1 #default 1
+reschedule_time #Time interval for checking if the job is finished
+
+querier.query(database_name, query_text, on_demand_path, options, priority, reschedule_time)
+```
+
+###Options
+Once the job has finished sidekiq will stop retriying and will send a callback to a class method specified on the options.
+
+* klass: The name of the class you want to use, i.e. "MyClass"
+* method: The name of the class method you want to use, i.e. "my_method"
+* results: if is "true" will fetch the results from treasure data and it will pass those results to your method as a parameter. Be aware that exceptionally large results might impact your performance.
+
+###Internals
+Querier objects are designed to query treasure data api asynchronously. 
+
+This gem uses [Sidekiq] (https://github.com/mperham/sidekiq) so make sure your app plays nice with that.
+
+Also it uses [td gem](https://rubygems.org/gems/td)
+
+### Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/VERSION

@@ -1 +1 @@
-0.0.5
+0.0.6

data/lib/td/querier.rb

@@ -10,24 +10,24 @@ class Querier
     @api_key = api_key
   end
 
-  def query(database, query, redirect_to, opts, priority=1, reschedule_time=5.minutes)
+  def query(database, query, opts, on_demand_path='', priority=1, reschedule_time=300)
     client = TreasureData::Client.new(@api_key)
-    job = client.query(database, query, redirect_to, priority)
+    job = client.query(database, query, on_demand_path, priority)
     Querier.perform_async(@api_key, job.job_id, opts, reschedule_time)
   end
 
-  def perform(api_key, job_id, opts, reschedule_time=5.minutes)
+  def perform(api_key, job_id, opts, reschedule_time=300)
     client = TreasureData::Client.new(api_key)
     job = client.job(job_id)
     #reschedule if the job is not finished
     return Querier.perform_in(reschedule_time, api_key, job_id, opts, reschedule_time) unless job.finished?
 
     if opts
-      klass = opts['klass']
-      meth = opts['method']
-      send_results = opts['results']
+      klass = opts[:klass]
+      meth = opts[:method]
+      send_results = opts[:results]
       results = (send_results.to_s == "true" ? job.results : nil)
-      class_eval(klass).send(meth.to_s, results)
+      eval(klass).send(meth.to_s, results)
     end
   end
 end

data/spec/querier_spec.rb

@@ -1,6 +1,91 @@
 require 'spec_helper'
-require 'debugger'
 
 describe Querier do
-  
+  describe "Send the query to treasaure data" do
+    context "Wrong parameters" do
+
+      it "will raise an error if the api key is not valid" do
+        fake_client = Object.new
+        fake_client.should_receive(:query).and_raise(TreasureData::APIError.new("apikey authentication failed"))
+        TreasureData::Client.should_receive(:new).and_return(fake_client)
+        Querier.should_not_receive(:perform_async)
+
+        querier = Querier.new("bad_key")
+        lambda{querier.query("database", "my_query", {})}.should raise_error(TreasureData::APIError)
+      end
+
+      it "will raise an error if the database name is not valid" do
+        fake_client = Object.new
+        fake_client.should_receive(:query).and_raise(TreasureData::NotFoundError.new("Couldn't find UserDatabase with name = bad_name"))
+        TreasureData::Client.should_receive(:new).and_return(fake_client)
+        Querier.should_not_receive(:perform_async)
+
+        querier = Querier.new("good_key")
+        lambda{querier.query("bad_name", "my_query", {})}.should raise_error(TreasureData::NotFoundError)
+      end
+    end
+
+    context "Right parameters" do
+      it "will send a query to treasure data and instance a sidekiq job with the job id of the new job" do
+        fake_job = Object.new
+        fake_job.should_receive(:job_id).and_return(1)
+
+        fake_client = Object.new
+        fake_client.should_receive(:query).and_return(fake_job)
+
+        TreasureData::Client.should_receive(:new).and_return(fake_client)
+
+        Querier.should_receive(:perform_async).and_return(true)
+
+        querier = Querier.new("good_key")
+        querier.query("database", "my_query", {})
+      end
+    end
+  end
+
+  describe "Sidekiq processing" do
+    context "Standar processing" do
+      before(:each) do
+        @opts = {:klass => "Object", :method => "fake_method", :results => "false"}
+        @fake_job = Object.new
+        @fake_client = Object.new
+        @fake_client.should_receive(:job).and_return(@fake_job)
+        TreasureData::Client.should_receive(:new).and_return(@fake_client)
+      end
+
+      it "will reschedule the sidekiq job if the treasure data job is not yet finished" do
+        @fake_job.should_receive(:finished?).and_return(false)
+        Querier.should_receive(:perform_in).with(300, "apikey", 1, @opts, 300).and_return(true)
+        Querier.new.perform("apikey", 1, @opts)
+      end
+
+      it "will make a call to a given class method if que job is finished" do
+        @fake_job.should_receive(:finished?).and_return(true)
+        Object.should_receive(:fake_method).and_return(true)
+        Querier.new.perform("apikey", 1, @opts)
+      end
+
+      it "will request the query results if the options for result are set to true" do
+        new_opts = {:klass => "Object", :method => "fake_method", :results => "true"}
+        results = [[1]]
+        @fake_job.should_receive(:finished?).and_return(true)
+        @fake_job.should_receive(:results).and_return(results)
+        Object.should_receive(:fake_method).and_return(true)
+        Querier.new.perform("apikey", 1, new_opts)
+      end
+    end
+
+    context "Error case" do
+      it "will raise an exception if something explodes within sidekiq" do
+        @opts = {:klass => "Object", :method => "fake_method", :results => "false"}
+        @fake_job = Object.new
+        @fake_client = Object.new
+        @fake_client.should_receive(:job).and_return(@fake_job)
+        TreasureData::Client.should_receive(:new).and_return(@fake_client)
+
+        @fake_job.should_receive(:finished?).and_raise(RuntimeError.new("unexpected error"))
+        lambda{Querier.new.perform("apikey", 1, @opts)}.should raise_error(RuntimeError)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: td-querier
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Carlos Donderis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-21 00:00:00.000000000 Z
+date: 2013-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq