queuery_client 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: d9bdfe61f21a506cef4c09057e13510305826234
4
+ data.tar.gz: '068d8a19aea9e7a13dd1c2b997848363e5a66217'
5
+ SHA512:
6
+ metadata.gz: 6cdc5d84c7ce94b3ed1c835b00f20d7893b0f08776eac198bb58951bf2def275692a3fd87f4324bbb033150c26984c50f9918bce201f4aa9c7137a33926b66ea
7
+ data.tar.gz: 50bc3ebad07bd41bed5c1a4c554155478d554a93c2a2887eb7693c319d0ab9d7cb7d52266121285d81139f4a30be9079144aed9d47debb92e2805439065fc086
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in queuery_client.gemspec
4
+ gemspec
data/README.md ADDED
@@ -0,0 +1,51 @@
1
+ # QueueryClient
2
+
3
+ Queuery client for Ruby.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'queuery_client'
11
+ ```
12
+
13
+ ## Configuration
14
+
15
+ ### If you don't use Rails
16
+
17
+ ```ruby
18
+ # configuration
19
+ RedshiftConnector.logger = Logger.new($stdout)
20
+ GarageClient.configure do |config|
21
+ config.name = "queuery-example"
22
+ end
23
+ QueueryClient.configure do |config|
24
+ config.endpoint = 'http://localhost:3000'
25
+ config.token = 'XXXXXXXXXXXXXXXXXXXXX'
26
+ config.token_secret = '*******************'
27
+ end
28
+ ```
29
+
30
+ ### If you are on Rails
31
+
32
+ In `config/initializers/queuery.rb`:
33
+
34
+ ```ruby
35
+ QueueryClient.configure do |config|
36
+ config.endpoint = 'http://localhost:3000'
37
+ config.token = 'XXXXXXXXXXXXXXXXXXXXX'
38
+ config.token_secret = '*******************'
39
+ end
40
+ ```
41
+
42
+ ## Usage
43
+
44
+ ```ruby
45
+ select_stmt = 'select column_a, column_b from the_great_table; -- an awesome query shows amazing fact up'
46
+ bundle = QueueryClient.query(select_stmt)
47
+ bundle.each do |row|
48
+ # do some useful works
49
+ p row
50
+ end
51
+ ```
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "queuery_client"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ require "pry"
11
+ Pry.start
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,26 @@
1
+ require 'garage_client'
2
+
3
+ module QueueryClient
4
+ class BasicAuthGarageClient < GarageClient::Client
5
+ # Override
6
+ def apply_auth_middleware(faraday_builder)
7
+ faraday_builder.use Faraday::Request::BasicAuthentication, login, password
8
+ end
9
+
10
+ def login
11
+ options[:login]
12
+ end
13
+
14
+ def login=(login)
15
+ options[:login] = login
16
+ end
17
+
18
+ def password
19
+ options[:password]
20
+ end
21
+
22
+ def password=(password)
23
+ options[:password] = password
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,61 @@
1
+ module QueueryClient
2
+ class Client
3
+ def initialize(options = {})
4
+ @options = options
5
+ end
6
+
7
+ def execute_query(select_stmt, values)
8
+ garage_client.post("/v1/queries", q: select_stmt, values: values)
9
+ end
10
+
11
+ def get_query(id)
12
+ garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix')
13
+ end
14
+
15
+ def wait_for(id)
16
+ loop do
17
+ query = get_query(id)
18
+ case query.status
19
+ when 'success', 'failed'
20
+ return query
21
+ end
22
+ sleep 3
23
+ end
24
+ end
25
+
26
+ def query_and_wait(select_stmt, values)
27
+ query = execute_query(select_stmt, values)
28
+ wait_for(query.id)
29
+ end
30
+
31
+ def query(select_stmt, values)
32
+ query = query_and_wait(select_stmt, values)
33
+ case query.status
34
+ when 'success'
35
+ QueueryDataFileBundle.new(
36
+ query.data_file_urls,
37
+ s3_prefix: query.s3_prefix,
38
+ )
39
+ when 'failed'
40
+ raise QueryError.new(query.error)
41
+ end
42
+ end
43
+
44
+ def garage_client
45
+ @garage_client ||= BasicAuthGarageClient.new(
46
+ endpoint: options.endpoint,
47
+ path_prefix: '/',
48
+ login: options.token,
49
+ password: options.token_secret
50
+ )
51
+ end
52
+
53
+ def options
54
+ default_options.merge(@options)
55
+ end
56
+
57
+ def default_options
58
+ QueueryClient.configuration
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,37 @@
1
+ module QueueryClient
2
+ class Configuration
3
+ def initialize(options = {})
4
+ @options = options
5
+ end
6
+
7
+ def options
8
+ @options ||= {}
9
+ end
10
+
11
+ def reset
12
+ @options = nil
13
+ end
14
+
15
+ [
16
+ :endpoint,
17
+ :token,
18
+ :token_secret,
19
+ ].each do |key|
20
+ define_method(key) do
21
+ options.fetch(key)
22
+ end
23
+
24
+ define_method("#{key}=") do |value|
25
+ options[key] = value
26
+ end
27
+ end
28
+
29
+ def merge(other)
30
+ Configuration.new(to_h.merge(other.to_h))
31
+ end
32
+
33
+ def to_h
34
+ options
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,3 @@
1
+ module QueueryClient
2
+ class QueryError < StandardError; end
3
+ end
@@ -0,0 +1,19 @@
1
+ require 'redshift_connector/url_data_file_bundle'
2
+
3
+ module QueueryClient
4
+ class QueueryDataFileBundle < RedshiftConnector::UrlDataFileBundle
5
+ def initialize(url, s3_prefix:, **args)
6
+ super(url, **args)
7
+ @s3_prefix = s3_prefix
8
+ end
9
+
10
+ attr_reader :s3_prefix
11
+
12
+ def url
13
+ uri = data_files.first.url.dup
14
+ uri.query = nil
15
+ uri.path = File.dirname(uri.path)
16
+ uri.to_s
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,3 @@
1
+ module QueueryClient
2
+ VERSION = "0.8.0"
3
+ end
@@ -0,0 +1,22 @@
1
+ require "queuery_client/version"
2
+ require "queuery_client/configuration"
3
+ require "queuery_client/basic_auth_garage_client"
4
+ require "queuery_client/query_error"
5
+ require "queuery_client/client"
6
+ require "queuery_client/queuery_data_file_bundle"
7
+
8
+ module QueueryClient
9
+ class << self
10
+ def configuration
11
+ @configuration ||= Configuration.new
12
+ end
13
+
14
+ def configure(&block)
15
+ configuration.instance_eval(&block)
16
+ end
17
+
18
+ def query(select_stmt, values = [])
19
+ Client.new.query(select_stmt, values)
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,36 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'queuery_client/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "queuery_client"
8
+ spec.version = QueueryClient::VERSION
9
+ spec.authors = ["Hidekazu Kobayashi"]
10
+ spec.email = ["hidekazu-kobayashi@cookpad.com"]
11
+ spec.license = "MIT"
12
+
13
+ spec.summary = "Client library for Queuery Redshift HTTP API"
14
+ spec.homepage = "https://github.com/bricolages/queuery_client"
15
+
16
+ # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
17
+ # to allow pushing to a single host or delete this section to allow pushing to any host.
18
+ if spec.respond_to?(:metadata)
19
+ spec.metadata['allowed_push_host'] = "https://rubygems.org"
20
+ else
21
+ raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
22
+ end
23
+
24
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
25
+ f.match(%r{^(test|spec|features)/})
26
+ end
27
+ spec.bindir = "exe"
28
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
29
+ spec.require_paths = ["lib"]
30
+
31
+ spec.add_dependency 'garage_client'
32
+ spec.add_dependency "redshift-connector-data_file", ">= 7.1"
33
+ spec.add_development_dependency "bundler", "~> 1.13"
34
+ spec.add_development_dependency "rake", "~> 10.0"
35
+ spec.add_development_dependency "pry"
36
+ end
data/queuery_client.rb ADDED
@@ -0,0 +1,64 @@
1
+ require 'uri'
2
+ require 'net/http'
3
+
4
+ class QueueryClient
5
+ class << self
6
+ attr_accessor :host
7
+ attr_accessor :port
8
+
9
+ def query(select_stmt)
10
+ client = new(
11
+ host: host,
12
+ port: port,
13
+ )
14
+
15
+ res = client.query_and_wait(select_stmt)
16
+ RedshiftConnector::UrlDataFileBundle.new(res['data_objects'])
17
+ end
18
+ end
19
+
20
+ def initialize(host:, port: 80)
21
+ @host = host
22
+ @port = port
23
+ end
24
+
25
+ def request(req)
26
+ res = Net::HTTP.new(@host, @port).request(req)
27
+ JSON.parse(res.body)
28
+ end
29
+
30
+ def request_post(path, params = {})
31
+ post_req = Net::HTTP::Post.new(path)
32
+ post_req.form_data = params
33
+ request(post_req)
34
+ end
35
+
36
+ def request_get(path)
37
+ get_req = Net::HTTP::Get.new(path)
38
+ request(get_req)
39
+ end
40
+
41
+ def query(select_stmt)
42
+ request_post('/queries', { q: select_stmt })
43
+ end
44
+
45
+ def status(job_id)
46
+ request_get("/queries/#{job_id}")
47
+ end
48
+
49
+ def wait_for(job_id)
50
+ loop do
51
+ res = status(job_id)
52
+ case res['status']
53
+ when 'success', 'failed'
54
+ return res
55
+ end
56
+ sleep 1
57
+ end
58
+ end
59
+
60
+ def query_and_wait(select_stmt)
61
+ res = query(select_stmt)
62
+ wait_for(res['job_id'])
63
+ end
64
+ end
metadata ADDED
@@ -0,0 +1,130 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: queuery_client
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.8.0
5
+ platform: ruby
6
+ authors:
7
+ - Hidekazu Kobayashi
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2018-03-22 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: garage_client
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: redshift-connector-data_file
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '7.1'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '7.1'
41
+ - !ruby/object:Gem::Dependency
42
+ name: bundler
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.13'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.13'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '10.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: pry
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ description:
84
+ email:
85
+ - hidekazu-kobayashi@cookpad.com
86
+ executables: []
87
+ extensions: []
88
+ extra_rdoc_files: []
89
+ files:
90
+ - ".gitignore"
91
+ - Gemfile
92
+ - README.md
93
+ - Rakefile
94
+ - bin/console
95
+ - bin/setup
96
+ - lib/queuery_client.rb
97
+ - lib/queuery_client/basic_auth_garage_client.rb
98
+ - lib/queuery_client/client.rb
99
+ - lib/queuery_client/configuration.rb
100
+ - lib/queuery_client/query_error.rb
101
+ - lib/queuery_client/queuery_data_file_bundle.rb
102
+ - lib/queuery_client/version.rb
103
+ - queuery_client.gemspec
104
+ - queuery_client.rb
105
+ homepage: https://github.com/bricolages/queuery_client
106
+ licenses:
107
+ - MIT
108
+ metadata:
109
+ allowed_push_host: https://rubygems.org
110
+ post_install_message:
111
+ rdoc_options: []
112
+ require_paths:
113
+ - lib
114
+ required_ruby_version: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - ">="
117
+ - !ruby/object:Gem::Version
118
+ version: '0'
119
+ required_rubygems_version: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - ">="
122
+ - !ruby/object:Gem::Version
123
+ version: '0'
124
+ requirements: []
125
+ rubyforge_project:
126
+ rubygems_version: 2.6.14
127
+ signing_key:
128
+ specification_version: 4
129
+ summary: Client library for Queuery Redshift HTTP API
130
+ test_files: []