queuery_client 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/Gemfile +4 -0
- data/README.md +51 -0
- data/Rakefile +2 -0
- data/bin/console +11 -0
- data/bin/setup +8 -0
- data/lib/queuery_client/basic_auth_garage_client.rb +26 -0
- data/lib/queuery_client/client.rb +61 -0
- data/lib/queuery_client/configuration.rb +37 -0
- data/lib/queuery_client/query_error.rb +3 -0
- data/lib/queuery_client/queuery_data_file_bundle.rb +19 -0
- data/lib/queuery_client/version.rb +3 -0
- data/lib/queuery_client.rb +22 -0
- data/queuery_client.gemspec +36 -0
- data/queuery_client.rb +64 -0
- metadata +130 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d9bdfe61f21a506cef4c09057e13510305826234
|
4
|
+
data.tar.gz: '068d8a19aea9e7a13dd1c2b997848363e5a66217'
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6cdc5d84c7ce94b3ed1c835b00f20d7893b0f08776eac198bb58951bf2def275692a3fd87f4324bbb033150c26984c50f9918bce201f4aa9c7137a33926b66ea
|
7
|
+
data.tar.gz: 50bc3ebad07bd41bed5c1a4c554155478d554a93c2a2887eb7693c319d0ab9d7cb7d52266121285d81139f4a30be9079144aed9d47debb92e2805439065fc086
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
# QueueryClient
|
2
|
+
|
3
|
+
Queuery client for Ruby.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'queuery_client'
|
11
|
+
```
|
12
|
+
|
13
|
+
## Configuration
|
14
|
+
|
15
|
+
### If you don't use Rails
|
16
|
+
|
17
|
+
```ruby
|
18
|
+
# configuration
|
19
|
+
RedshiftConnector.logger = Logger.new($stdout)
|
20
|
+
GarageClient.configure do |config|
|
21
|
+
config.name = "queuery-example"
|
22
|
+
end
|
23
|
+
QueueryClient.configure do |config|
|
24
|
+
config.endpoint = 'http://localhost:3000'
|
25
|
+
config.token = 'XXXXXXXXXXXXXXXXXXXXX'
|
26
|
+
config.token_secret = '*******************'
|
27
|
+
end
|
28
|
+
```
|
29
|
+
|
30
|
+
### If you are on Rails
|
31
|
+
|
32
|
+
In `config/initializers/queuery.rb`:
|
33
|
+
|
34
|
+
```ruby
|
35
|
+
QueueryClient.configure do |config|
|
36
|
+
config.endpoint = 'http://localhost:3000'
|
37
|
+
config.token = 'XXXXXXXXXXXXXXXXXXXXX'
|
38
|
+
config.token_secret = '*******************'
|
39
|
+
end
|
40
|
+
```
|
41
|
+
|
42
|
+
## Usage
|
43
|
+
|
44
|
+
```ruby
|
45
|
+
select_stmt = 'select column_a, column_b from the_great_table; -- an awesome query shows amazing fact up'
|
46
|
+
bundle = QueueryClient.query(select_stmt)
|
47
|
+
bundle.each do |row|
|
48
|
+
# do some useful works
|
49
|
+
p row
|
50
|
+
end
|
51
|
+
```
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "queuery_client"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
require "pry"
|
11
|
+
Pry.start
|
data/bin/setup
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'garage_client'
|
2
|
+
|
3
|
+
module QueueryClient
|
4
|
+
class BasicAuthGarageClient < GarageClient::Client
|
5
|
+
# Override
|
6
|
+
def apply_auth_middleware(faraday_builder)
|
7
|
+
faraday_builder.use Faraday::Request::BasicAuthentication, login, password
|
8
|
+
end
|
9
|
+
|
10
|
+
def login
|
11
|
+
options[:login]
|
12
|
+
end
|
13
|
+
|
14
|
+
def login=(login)
|
15
|
+
options[:login] = login
|
16
|
+
end
|
17
|
+
|
18
|
+
def password
|
19
|
+
options[:password]
|
20
|
+
end
|
21
|
+
|
22
|
+
def password=(password)
|
23
|
+
options[:password] = password
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module QueueryClient
|
2
|
+
class Client
|
3
|
+
def initialize(options = {})
|
4
|
+
@options = options
|
5
|
+
end
|
6
|
+
|
7
|
+
def execute_query(select_stmt, values)
|
8
|
+
garage_client.post("/v1/queries", q: select_stmt, values: values)
|
9
|
+
end
|
10
|
+
|
11
|
+
def get_query(id)
|
12
|
+
garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix')
|
13
|
+
end
|
14
|
+
|
15
|
+
def wait_for(id)
|
16
|
+
loop do
|
17
|
+
query = get_query(id)
|
18
|
+
case query.status
|
19
|
+
when 'success', 'failed'
|
20
|
+
return query
|
21
|
+
end
|
22
|
+
sleep 3
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def query_and_wait(select_stmt, values)
|
27
|
+
query = execute_query(select_stmt, values)
|
28
|
+
wait_for(query.id)
|
29
|
+
end
|
30
|
+
|
31
|
+
def query(select_stmt, values)
|
32
|
+
query = query_and_wait(select_stmt, values)
|
33
|
+
case query.status
|
34
|
+
when 'success'
|
35
|
+
QueueryDataFileBundle.new(
|
36
|
+
query.data_file_urls,
|
37
|
+
s3_prefix: query.s3_prefix,
|
38
|
+
)
|
39
|
+
when 'failed'
|
40
|
+
raise QueryError.new(query.error)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def garage_client
|
45
|
+
@garage_client ||= BasicAuthGarageClient.new(
|
46
|
+
endpoint: options.endpoint,
|
47
|
+
path_prefix: '/',
|
48
|
+
login: options.token,
|
49
|
+
password: options.token_secret
|
50
|
+
)
|
51
|
+
end
|
52
|
+
|
53
|
+
def options
|
54
|
+
default_options.merge(@options)
|
55
|
+
end
|
56
|
+
|
57
|
+
def default_options
|
58
|
+
QueueryClient.configuration
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module QueueryClient
|
2
|
+
class Configuration
|
3
|
+
def initialize(options = {})
|
4
|
+
@options = options
|
5
|
+
end
|
6
|
+
|
7
|
+
def options
|
8
|
+
@options ||= {}
|
9
|
+
end
|
10
|
+
|
11
|
+
def reset
|
12
|
+
@options = nil
|
13
|
+
end
|
14
|
+
|
15
|
+
[
|
16
|
+
:endpoint,
|
17
|
+
:token,
|
18
|
+
:token_secret,
|
19
|
+
].each do |key|
|
20
|
+
define_method(key) do
|
21
|
+
options.fetch(key)
|
22
|
+
end
|
23
|
+
|
24
|
+
define_method("#{key}=") do |value|
|
25
|
+
options[key] = value
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def merge(other)
|
30
|
+
Configuration.new(to_h.merge(other.to_h))
|
31
|
+
end
|
32
|
+
|
33
|
+
def to_h
|
34
|
+
options
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'redshift_connector/url_data_file_bundle'
|
2
|
+
|
3
|
+
module QueueryClient
|
4
|
+
class QueueryDataFileBundle < RedshiftConnector::UrlDataFileBundle
|
5
|
+
def initialize(url, s3_prefix:, **args)
|
6
|
+
super(url, **args)
|
7
|
+
@s3_prefix = s3_prefix
|
8
|
+
end
|
9
|
+
|
10
|
+
attr_reader :s3_prefix
|
11
|
+
|
12
|
+
def url
|
13
|
+
uri = data_files.first.url.dup
|
14
|
+
uri.query = nil
|
15
|
+
uri.path = File.dirname(uri.path)
|
16
|
+
uri.to_s
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require "queuery_client/version"
|
2
|
+
require "queuery_client/configuration"
|
3
|
+
require "queuery_client/basic_auth_garage_client"
|
4
|
+
require "queuery_client/query_error"
|
5
|
+
require "queuery_client/client"
|
6
|
+
require "queuery_client/queuery_data_file_bundle"
|
7
|
+
|
8
|
+
module QueueryClient
|
9
|
+
class << self
|
10
|
+
def configuration
|
11
|
+
@configuration ||= Configuration.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def configure(&block)
|
15
|
+
configuration.instance_eval(&block)
|
16
|
+
end
|
17
|
+
|
18
|
+
def query(select_stmt, values = [])
|
19
|
+
Client.new.query(select_stmt, values)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'queuery_client/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "queuery_client"
|
8
|
+
spec.version = QueueryClient::VERSION
|
9
|
+
spec.authors = ["Hidekazu Kobayashi"]
|
10
|
+
spec.email = ["hidekazu-kobayashi@cookpad.com"]
|
11
|
+
spec.license = "MIT"
|
12
|
+
|
13
|
+
spec.summary = "Client library for Queuery Redshift HTTP API"
|
14
|
+
spec.homepage = "https://github.com/bricolages/queuery_client"
|
15
|
+
|
16
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
17
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
18
|
+
if spec.respond_to?(:metadata)
|
19
|
+
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
20
|
+
else
|
21
|
+
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
22
|
+
end
|
23
|
+
|
24
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
25
|
+
f.match(%r{^(test|spec|features)/})
|
26
|
+
end
|
27
|
+
spec.bindir = "exe"
|
28
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
29
|
+
spec.require_paths = ["lib"]
|
30
|
+
|
31
|
+
spec.add_dependency 'garage_client'
|
32
|
+
spec.add_dependency "redshift-connector-data_file", ">= 7.1"
|
33
|
+
spec.add_development_dependency "bundler", "~> 1.13"
|
34
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
35
|
+
spec.add_development_dependency "pry"
|
36
|
+
end
|
data/queuery_client.rb
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
class QueueryClient
|
5
|
+
class << self
|
6
|
+
attr_accessor :host
|
7
|
+
attr_accessor :port
|
8
|
+
|
9
|
+
def query(select_stmt)
|
10
|
+
client = new(
|
11
|
+
host: host,
|
12
|
+
port: port,
|
13
|
+
)
|
14
|
+
|
15
|
+
res = client.query_and_wait(select_stmt)
|
16
|
+
RedshiftConnector::UrlDataFileBundle.new(res['data_objects'])
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def initialize(host:, port: 80)
|
21
|
+
@host = host
|
22
|
+
@port = port
|
23
|
+
end
|
24
|
+
|
25
|
+
def request(req)
|
26
|
+
res = Net::HTTP.new(@host, @port).request(req)
|
27
|
+
JSON.parse(res.body)
|
28
|
+
end
|
29
|
+
|
30
|
+
def request_post(path, params = {})
|
31
|
+
post_req = Net::HTTP::Post.new(path)
|
32
|
+
post_req.form_data = params
|
33
|
+
request(post_req)
|
34
|
+
end
|
35
|
+
|
36
|
+
def request_get(path)
|
37
|
+
get_req = Net::HTTP::Get.new(path)
|
38
|
+
request(get_req)
|
39
|
+
end
|
40
|
+
|
41
|
+
def query(select_stmt)
|
42
|
+
request_post('/queries', { q: select_stmt })
|
43
|
+
end
|
44
|
+
|
45
|
+
def status(job_id)
|
46
|
+
request_get("/queries/#{job_id}")
|
47
|
+
end
|
48
|
+
|
49
|
+
def wait_for(job_id)
|
50
|
+
loop do
|
51
|
+
res = status(job_id)
|
52
|
+
case res['status']
|
53
|
+
when 'success', 'failed'
|
54
|
+
return res
|
55
|
+
end
|
56
|
+
sleep 1
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def query_and_wait(select_stmt)
|
61
|
+
res = query(select_stmt)
|
62
|
+
wait_for(res['job_id'])
|
63
|
+
end
|
64
|
+
end
|
metadata
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: queuery_client
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.8.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Hidekazu Kobayashi
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-03-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: garage_client
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: redshift-connector-data_file
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '7.1'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '7.1'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.13'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.13'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '10.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '10.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description:
|
84
|
+
email:
|
85
|
+
- hidekazu-kobayashi@cookpad.com
|
86
|
+
executables: []
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- ".gitignore"
|
91
|
+
- Gemfile
|
92
|
+
- README.md
|
93
|
+
- Rakefile
|
94
|
+
- bin/console
|
95
|
+
- bin/setup
|
96
|
+
- lib/queuery_client.rb
|
97
|
+
- lib/queuery_client/basic_auth_garage_client.rb
|
98
|
+
- lib/queuery_client/client.rb
|
99
|
+
- lib/queuery_client/configuration.rb
|
100
|
+
- lib/queuery_client/query_error.rb
|
101
|
+
- lib/queuery_client/queuery_data_file_bundle.rb
|
102
|
+
- lib/queuery_client/version.rb
|
103
|
+
- queuery_client.gemspec
|
104
|
+
- queuery_client.rb
|
105
|
+
homepage: https://github.com/bricolages/queuery_client
|
106
|
+
licenses:
|
107
|
+
- MIT
|
108
|
+
metadata:
|
109
|
+
allowed_push_host: https://rubygems.org
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
requirements: []
|
125
|
+
rubyforge_project:
|
126
|
+
rubygems_version: 2.6.14
|
127
|
+
signing_key:
|
128
|
+
specification_version: 4
|
129
|
+
summary: Client library for Queuery Redshift HTTP API
|
130
|
+
test_files: []
|