td-querier 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +14 -9
- data/VERSION +1 -1
- data/lib/td/querier.rb +16 -8
- data/spec/querier_spec.rb +5 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6906902fe3b7a95a8dca37812c5161f61378b4db
|
4
|
+
data.tar.gz: 6a19001f042c206bbfc27e7f2a29ba54a4fc551d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 362797d97acc68b83bb9d07194a108920f98fe5926cb3e62f1023ca56bdfb5d43a389357a363e3a0527b7b774d4c815f5d82536031bdf4314e87ba009f360087
|
7
|
+
data.tar.gz: 2ea3bcebecb8c958857f49862a3fcf122580fcf1fb828bf2d808f1b8835aeea95e80aab236c1ba259d028eedc8eead90c470aa3de7c1056b9a8d3015dda844ee
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -17,14 +17,15 @@ If not it will reschedule itself until the job is done.
|
|
17
17
|
querier = Querier.new("TREASURE_DATA_API_KEY")
|
18
18
|
database_name = 'my_td_database_name'
|
19
19
|
query_text = 'select count(*) from my_table'
|
20
|
-
options = {:klass=>
|
21
|
-
|
22
|
-
|
23
|
-
on_demand_path
|
24
|
-
priority
|
25
|
-
reschedule_time
|
26
|
-
|
27
|
-
|
20
|
+
options = {:klass => 'MyClass',
|
21
|
+
:method => 'my_method',
|
22
|
+
:results => 'true',
|
23
|
+
:on_demand_path => 'mysql://user:password@host/database/table',
|
24
|
+
:priority => 1,
|
25
|
+
:reschedule_time => 300
|
26
|
+
} #See Options section for this one
|
27
|
+
|
28
|
+
querier.query(database_name, query_text, options)
|
28
29
|
```
|
29
30
|
|
30
31
|
###Options
|
@@ -32,7 +33,11 @@ Once the job has finished sidekiq will stop retriying and will send a callback t
|
|
32
33
|
|
33
34
|
* klass: The name of the class you want to use, i.e. "MyClass"
|
34
35
|
* method: The name of the class method you want to use, i.e. "my_method"
|
35
|
-
* results: if is "true" will fetch the results from treasure data and it will pass those results to your method as a
|
36
|
+
* results: if is "true" will fetch the results from treasure data and it will pass those results to your method as a
|
37
|
+
parameter. Be aware that exceptionally large results might impact your performance.
|
38
|
+
* on_demand_path: will insert the result of your query into another table, i.e: 'mysql://user:password@host/database/table'
|
39
|
+
* priority: Treasure data desired priority. By default 1
|
40
|
+
* reschedule_time: Time interval for checking if the job is finished and rescheudle sidekiq job. By default 300 seconds
|
36
41
|
|
37
42
|
###Internals
|
38
43
|
Querier objects are designed to query treasure data api asynchronously.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.7
|
data/lib/td/querier.rb
CHANGED
@@ -10,23 +10,31 @@ class Querier
|
|
10
10
|
@api_key = api_key
|
11
11
|
end
|
12
12
|
|
13
|
-
def query(database, query, opts
|
13
|
+
def query(database, query, opts)
|
14
|
+
on_demand_path = (opts && opts['on_demand_path'] != nil && opts['on_demand_path'] != '') ? opts['on_demand_path'] : ''
|
15
|
+
priority = (opts && opts['priority'] != nil && opts['priority'] != '') ? opts['priority'] : 1
|
16
|
+
reschedule_time = (opts && opts['reschedule_time'] != nil && opts['reschedule_time'] != '') ? opts['reschedule_time'] : 300
|
17
|
+
|
14
18
|
client = TreasureData::Client.new(@api_key)
|
15
19
|
job = client.query(database, query, on_demand_path, priority)
|
16
|
-
Querier.perform_async(@api_key, job.job_id, opts
|
20
|
+
Querier.perform_async(@api_key, job.job_id, opts)
|
17
21
|
end
|
18
22
|
|
19
|
-
def perform(api_key, job_id, opts
|
23
|
+
def perform(api_key, job_id, opts)
|
20
24
|
client = TreasureData::Client.new(api_key)
|
21
25
|
job = client.job(job_id)
|
22
26
|
#reschedule if the job is not finished
|
23
|
-
|
27
|
+
unless job.finished?
|
28
|
+
reschedule_time = (opts && opts['reschedule_time'] != nil && opts['reschedule_time'] != '') ? opts['reschedule_time'].to_i : 300
|
29
|
+
return Querier.perform_in(reschedule_time, api_key, job_id, opts)
|
30
|
+
end
|
24
31
|
|
25
32
|
if opts
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
33
|
+
stringified_opts = Hash[opts.map{ |k, v| [k.to_s, v] }]
|
34
|
+
klass = stringified_opts['klass']
|
35
|
+
meth = stringified_opts['method']
|
36
|
+
send_results = stringified_opts['results']
|
37
|
+
results = (send_results.to_s == "true" ? job.result : nil)
|
30
38
|
eval(klass).send(meth.to_s, results)
|
31
39
|
end
|
32
40
|
end
|
data/spec/querier_spec.rb
CHANGED
@@ -46,7 +46,7 @@ describe Querier do
|
|
46
46
|
describe "Sidekiq processing" do
|
47
47
|
context "Standar processing" do
|
48
48
|
before(:each) do
|
49
|
-
@opts = {
|
49
|
+
@opts = {'klass' => "Object", 'method' => "fake_method", 'results' => "false"}
|
50
50
|
@fake_job = Object.new
|
51
51
|
@fake_client = Object.new
|
52
52
|
@fake_client.should_receive(:job).and_return(@fake_job)
|
@@ -55,7 +55,7 @@ describe Querier do
|
|
55
55
|
|
56
56
|
it "will reschedule the sidekiq job if the treasure data job is not yet finished" do
|
57
57
|
@fake_job.should_receive(:finished?).and_return(false)
|
58
|
-
Querier.should_receive(:perform_in).with(300, "apikey", 1, @opts
|
58
|
+
Querier.should_receive(:perform_in).with(300, "apikey", 1, @opts).and_return(true)
|
59
59
|
Querier.new.perform("apikey", 1, @opts)
|
60
60
|
end
|
61
61
|
|
@@ -69,7 +69,7 @@ describe Querier do
|
|
69
69
|
new_opts = {:klass => "Object", :method => "fake_method", :results => "true"}
|
70
70
|
results = [[1]]
|
71
71
|
@fake_job.should_receive(:finished?).and_return(true)
|
72
|
-
@fake_job.should_receive(:
|
72
|
+
@fake_job.should_receive(:result).and_return(results)
|
73
73
|
Object.should_receive(:fake_method).and_return(true)
|
74
74
|
Querier.new.perform("apikey", 1, new_opts)
|
75
75
|
end
|
@@ -77,14 +77,14 @@ describe Querier do
|
|
77
77
|
|
78
78
|
context "Error case" do
|
79
79
|
it "will raise an exception if something explodes within sidekiq" do
|
80
|
-
|
80
|
+
opts = {:klass => "Object", :method => "fake_method", :results => "false"}
|
81
81
|
@fake_job = Object.new
|
82
82
|
@fake_client = Object.new
|
83
83
|
@fake_client.should_receive(:job).and_return(@fake_job)
|
84
84
|
TreasureData::Client.should_receive(:new).and_return(@fake_client)
|
85
85
|
|
86
86
|
@fake_job.should_receive(:finished?).and_raise(RuntimeError.new("unexpected error"))
|
87
|
-
lambda{Querier.new.perform("apikey", 1,
|
87
|
+
lambda{Querier.new.perform("apikey", 1, opts)}.should raise_error(RuntimeError)
|
88
88
|
end
|
89
89
|
end
|
90
90
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: td-querier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Carlos Donderis
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-05-
|
11
|
+
date: 2013-05-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|