queuery_client 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 893d3e23c931f4f67621e03306cb3523e2bf86affe3a4c7bd6958e5cd4a7aa12
4
+ data.tar.gz: 1fbcebdefffabad096aee653ab255329696b8e5b6e2056f4742a619ec15e3810
5
+ SHA512:
6
+ metadata.gz: 4e7ac1fc3d9b850fc0d7d9261c1db44664511997ec17789dd153d9faf9296af7caa689561d30cfde7e07918d43331fbd11b92a1704c9781b489bb3b17704a4e3
7
+ data.tar.gz: 59ca2f54c33e271ffa93d59c224fa98ae11b729152407b5b7691413bce1ca2b4a2ec6b14f9a6c6253cade2fe1bee290aa12f6496daa821441e9d7e9092b3cf91
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in queuery_client.gemspec
4
+ gemspec
@@ -0,0 +1,51 @@
1
+ # QueueryClient
2
+
3
+ Queuery client for Ruby.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'queuery_client'
11
+ ```
12
+
13
+ ## Configuration
14
+
15
+ ### If you don't use Rails
16
+
17
+ ```ruby
18
+ # configuration
19
+ RedshiftConnector.logger = Logger.new($stdout)
20
+ GarageClient.configure do |config|
21
+ config.name = "queuery-example"
22
+ end
23
+ QueueryClient.configure do |config|
24
+ config.endpoint = 'http://localhost:3000'
25
+ config.token = 'XXXXXXXXXXXXXXXXXXXXX'
26
+ config.token_secret = '*******************'
27
+ end
28
+ ```
29
+
30
+ ### If you are on Rails
31
+
32
+ In `config/initializers/queuery.rb`:
33
+
34
+ ```ruby
35
+ QueueryClient.configure do |config|
36
+ config.endpoint = 'http://localhost:3000'
37
+ config.token = 'XXXXXXXXXXXXXXXXXXXXX'
38
+ config.token_secret = '*******************'
39
+ end
40
+ ```
41
+
42
+ ## Usage
43
+
44
+ ```ruby
45
+ select_stmt = 'select column_a, column_b from the_great_table; -- an awesome query shows amazing fact up'
46
+ bundle = QueueryClient.query(select_stmt)
47
+ bundle.each do |row|
48
+ # do some useful works
49
+ p row
50
+ end
51
+ ```
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+ task :default => :spec
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "queuery_client"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ require "pry"
11
+ Pry.start
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,23 @@
1
+ require "queuery_client/version"
2
+ require "queuery_client/configuration"
3
+ require "queuery_client/basic_auth_garage_client"
4
+ require "queuery_client/query_error"
5
+ require "queuery_client/client"
6
+ require "queuery_client/url_data_file_bundle"
7
+ require "queuery_client/s3_data_file_bundle"
8
+
9
+ module QueueryClient
10
+ class << self
11
+ def configuration
12
+ @configuration ||= Configuration.new
13
+ end
14
+
15
+ def configure(&block)
16
+ configuration.instance_eval(&block)
17
+ end
18
+
19
+ def query(select_stmt, values = [])
20
+ Client.new.query(select_stmt, values)
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,26 @@
1
+ require 'garage_client'
2
+
3
+ module QueueryClient
4
+ class BasicAuthGarageClient < GarageClient::Client
5
+ # Override
6
+ def apply_auth_middleware(faraday_builder)
7
+ faraday_builder.use Faraday::Request::BasicAuthentication, login, password
8
+ end
9
+
10
+ def login
11
+ options[:login]
12
+ end
13
+
14
+ def login=(login)
15
+ options[:login] = login
16
+ end
17
+
18
+ def password
19
+ options[:password]
20
+ end
21
+
22
+ def password=(password)
23
+ options[:password] = password
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,84 @@
1
+ module QueueryClient
2
+ class Client
3
+ def initialize(options = {})
4
+ @options = options
5
+ end
6
+
7
+ def execute_query(select_stmt, values)
8
+ garage_client.post("/v1/queries", q: select_stmt, values: values)
9
+ end
10
+ alias start_query execute_query
11
+
12
+ def get_query(id)
13
+ garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix')
14
+ end
15
+
16
+ def wait_for(id)
17
+ loop do
18
+ query = get_query(id)
19
+ case query.status
20
+ when 'success', 'failed'
21
+ return query
22
+ end
23
+ sleep 3
24
+ end
25
+ end
26
+
27
+ def query_and_wait(select_stmt, values)
28
+ query = execute_query(select_stmt, values)
29
+ wait_for(query.id)
30
+ end
31
+
32
+ def query(select_stmt, values)
33
+ query = query_and_wait(select_stmt, values)
34
+ case query.status
35
+ when 'success'
36
+ UrlDataFileBundle.new(
37
+ query.data_file_urls,
38
+ s3_prefix: query.s3_prefix,
39
+ )
40
+ when 'failed'
41
+ raise QueryError.new(query.error)
42
+ end
43
+ end
44
+
45
+ # poll_result returns the results only if the query has already successed.
46
+ def poll_result(id)
47
+ query = get_query(id)
48
+ get_query_result(query)
49
+ end
50
+
51
+ def garage_client
52
+ @garage_client ||= BasicAuthGarageClient.new(
53
+ endpoint: options.endpoint,
54
+ path_prefix: '/',
55
+ login: options.token,
56
+ password: options.token_secret
57
+ )
58
+ end
59
+
60
+ def options
61
+ default_options.merge(@options)
62
+ end
63
+
64
+ def default_options
65
+ QueueryClient.configuration
66
+ end
67
+
68
+ private
69
+
70
+ def get_query_result(query)
71
+ case query.status
72
+ when 'pending', 'running'
73
+ nil
74
+ when 'success'
75
+ UrlDataFileBundle.new(
76
+ query.data_file_urls,
77
+ s3_prefix: query.s3_prefix,
78
+ )
79
+ when 'failure'
80
+ raise QueryError.new(query.error)
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,37 @@
1
+ module QueueryClient
2
+ class Configuration
3
+ def initialize(options = {})
4
+ @options = options
5
+ end
6
+
7
+ def options
8
+ @options ||= {}
9
+ end
10
+
11
+ def reset
12
+ @options = nil
13
+ end
14
+
15
+ [
16
+ :endpoint,
17
+ :token,
18
+ :token_secret,
19
+ ].each do |key|
20
+ define_method(key) do
21
+ options.fetch(key)
22
+ end
23
+
24
+ define_method("#{key}=") do |value|
25
+ options[key] = value
26
+ end
27
+ end
28
+
29
+ def merge(other)
30
+ Configuration.new(to_h.merge(other.to_h))
31
+ end
32
+
33
+ def to_h
34
+ options
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,26 @@
1
+ require 'redshift_csv_file'
2
+ require 'zlib'
3
+
4
+ module QueueryClient
5
+ class DataFile
6
+ def data_object?
7
+ /\.csv(?:\.|\z)/ =~ File.basename(key)
8
+ end
9
+
10
+ def gzipped_object?
11
+ File.extname(key) == '.gz'
12
+ end
13
+
14
+ def each_row(&block)
15
+ f = open
16
+ begin
17
+ if gzipped_object?
18
+ f = Zlib::GzipReader.new(f)
19
+ end
20
+ RedshiftCsvFile.new(f).each(&block)
21
+ ensure
22
+ f.close
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,15 @@
1
+ module QueueryClient
2
+ class DataFileBundle
3
+ # abstract data_files :: [DataFile]
4
+
5
+ def each_row(&block)
6
+ data_files.each do |file|
7
+ if file.data_object?
8
+ file.each_row(&block)
9
+ end
10
+ end
11
+ end
12
+
13
+ alias each each_row
14
+ end
15
+ end
@@ -0,0 +1,3 @@
1
+ module QueueryClient
2
+ class QueryError < StandardError; end
3
+ end
@@ -0,0 +1,18 @@
1
+ require 'queuery_client/data_file'
2
+ require 'forwardable'
3
+
4
+ module QueueryClient
5
+ class S3DataFile < DataFile
6
+ extend Forwardable
7
+
8
+ def initialize(object)
9
+ @object = object
10
+ end
11
+
12
+ def_delegators '@object', :url, :key, :presigned_url
13
+
14
+ def open
15
+ @object.get.body
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,28 @@
1
+ require 'queuery_client/data_file_bundle'
2
+ require 'queuery_client/s3_data_file'
3
+ require 'aws-sdk-s3'
4
+ require 'logger'
5
+
6
+ module QueueryClient
7
+ class S3DataFileBundle < DataFileBundle
8
+ def initialize(bucket, prefix, s3_client: nil, logger: Logger.new($stderr))
9
+ @s3_client = s3_client || Aws::S3::Client.new # Use env to inject credentials
10
+ @bucket = bucket
11
+ @prefix = prefix
12
+ @logger = logger
13
+ end
14
+
15
+ attr_reader :bucket
16
+ attr_reader :prefix
17
+ attr_reader :logger
18
+
19
+ def url
20
+ "s3://#{@bucket}/#{@prefix}"
21
+ end
22
+
23
+ def data_files
24
+ b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
25
+ b.objects(prefix: @prefix).map {|obj| S3DataFile.new(obj) }
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,27 @@
1
+ require 'queuery_client/data_file'
2
+ require 'net/http'
3
+ require 'stringio'
4
+
5
+ module QueueryClient
6
+ class UrlDataFile < DataFile
7
+ def initialize(url)
8
+ @url = url
9
+ end
10
+
11
+ attr_reader :url
12
+
13
+ def key
14
+ @url.path
15
+ end
16
+
17
+ def open
18
+ http = Net::HTTP.new(@url.host, @url.port)
19
+ http.use_ssl = (@url.scheme.downcase == 'https')
20
+ content = http.start {
21
+ res = http.get(@url.request_uri)
22
+ res.body
23
+ }
24
+ StringIO.new(content)
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,33 @@
1
+ require 'queuery_client/data_file_bundle'
2
+ require 'queuery_client/url_data_file'
3
+ require 'uri'
4
+ require 'logger'
5
+
6
+ module QueueryClient
7
+ class UrlDataFileBundle < DataFileBundle
8
+ def initialize(urls, s3_prefix:, logger: Logger.new($stderr))
9
+ raise ArgumentError, 'no URL given' if urls.empty?
10
+ @data_files = urls.map {|url| UrlDataFile.new(URI.parse(url)) }
11
+ @s3_prefix = s3_prefix
12
+ @logger = logger
13
+ end
14
+
15
+ attr_reader :data_files
16
+ attr_reader :s3_prefix
17
+ attr_reader :logger
18
+
19
+ def url
20
+ uri = data_files.first.url.dup
21
+ uri.query = nil
22
+ uri.path = File.dirname(uri.path)
23
+ uri.to_s
24
+ end
25
+
26
+ def direct(bucket_opts = {}, bundle_opts = {})
27
+ s3_uri = URI.parse(s3_prefix)
28
+ bucket = s3_uri.host
29
+ prefix = s3_uri.path[1..-1] # trim heading slash
30
+ S3DataFileBundle.new(bucket, prefix)
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,3 @@
1
+ module QueueryClient
2
+ VERSION = "1.0.2"
3
+ end
@@ -0,0 +1,37 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'queuery_client/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "queuery_client"
8
+ spec.version = QueueryClient::VERSION
9
+ spec.authors = ["Hidekazu Kobayashi"]
10
+ spec.email = ["hidekazu-kobayashi@cookpad.com"]
11
+ spec.license = "MIT"
12
+
13
+ spec.summary = "Client library for Queuery Redshift HTTP API"
14
+ spec.homepage = "https://github.com/bricolages/queuery_client"
15
+
16
+ # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
17
+ # to allow pushing to a single host or delete this section to allow pushing to any host.
18
+ if spec.respond_to?(:metadata)
19
+ spec.metadata['allowed_push_host'] = "https://rubygems.org"
20
+ else
21
+ raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
22
+ end
23
+
24
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
25
+ f.match(%r{^(test|spec|features)/})
26
+ end
27
+ spec.bindir = "exe"
28
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
29
+ spec.require_paths = ["lib"]
30
+
31
+ spec.add_dependency 'garage_client'
32
+ spec.add_dependency 'redshift_csv_file'
33
+ spec.add_dependency 'aws-sdk-s3'
34
+ spec.add_development_dependency "bundler", "~> 1.13"
35
+ spec.add_development_dependency "rake", "~> 12.0"
36
+ spec.add_development_dependency "pry"
37
+ end
metadata ADDED
@@ -0,0 +1,147 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: queuery_client
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.2
5
+ platform: ruby
6
+ authors:
7
+ - Hidekazu Kobayashi
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-07-22 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: garage_client
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: redshift_csv_file
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: aws-sdk-s3
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.13'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.13'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rake
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '12.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '12.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: pry
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description:
98
+ email:
99
+ - hidekazu-kobayashi@cookpad.com
100
+ executables: []
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - ".gitignore"
105
+ - Gemfile
106
+ - README.md
107
+ - Rakefile
108
+ - bin/console
109
+ - bin/setup
110
+ - lib/queuery_client.rb
111
+ - lib/queuery_client/basic_auth_garage_client.rb
112
+ - lib/queuery_client/client.rb
113
+ - lib/queuery_client/configuration.rb
114
+ - lib/queuery_client/data_file.rb
115
+ - lib/queuery_client/data_file_bundle.rb
116
+ - lib/queuery_client/query_error.rb
117
+ - lib/queuery_client/s3_data_file.rb
118
+ - lib/queuery_client/s3_data_file_bundle.rb
119
+ - lib/queuery_client/url_data_file.rb
120
+ - lib/queuery_client/url_data_file_bundle.rb
121
+ - lib/queuery_client/version.rb
122
+ - queuery_client.gemspec
123
+ homepage: https://github.com/bricolages/queuery_client
124
+ licenses:
125
+ - MIT
126
+ metadata:
127
+ allowed_push_host: https://rubygems.org
128
+ post_install_message:
129
+ rdoc_options: []
130
+ require_paths:
131
+ - lib
132
+ required_ruby_version: !ruby/object:Gem::Requirement
133
+ requirements:
134
+ - - ">="
135
+ - !ruby/object:Gem::Version
136
+ version: '0'
137
+ required_rubygems_version: !ruby/object:Gem::Requirement
138
+ requirements:
139
+ - - ">="
140
+ - !ruby/object:Gem::Version
141
+ version: '0'
142
+ requirements: []
143
+ rubygems_version: 3.1.2
144
+ signing_key:
145
+ specification_version: 4
146
+ summary: Client library for Queuery Redshift HTTP API
147
+ test_files: []