queuery_client 0.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d9bdfe61f21a506cef4c09057e13510305826234
+  data.tar.gz: '068d8a19aea9e7a13dd1c2b997848363e5a66217'
+SHA512:
+  metadata.gz: 6cdc5d84c7ce94b3ed1c835b00f20d7893b0f08776eac198bb58951bf2def275692a3fd87f4324bbb033150c26984c50f9918bce201f4aa9c7137a33926b66ea
+  data.tar.gz: 50bc3ebad07bd41bed5c1a4c554155478d554a93c2a2887eb7693c319d0ab9d7cb7d52266121285d81139f4a30be9079144aed9d47debb92e2805439065fc086

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in queuery_client.gemspec
+gemspec

data/README.md

@@ -0,0 +1,51 @@
+# QueueryClient
+
+Queuery client for Ruby.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'queuery_client'
+```
+
+## Configuration
+
+### If you don't use Rails
+
+```ruby
+# configuration
+RedshiftConnector.logger = Logger.new($stdout)
+GarageClient.configure do |config|
+  config.name = "queuery-example"
+end
+QueueryClient.configure do |config|
+  config.endpoint = 'http://localhost:3000'
+  config.token = 'XXXXXXXXXXXXXXXXXXXXX'
+  config.token_secret = '*******************'
+end
+```
+
+### If you are on Rails
+
+In `config/initializers/queuery.rb`:
+
+```ruby
+QueueryClient.configure do |config|
+  config.endpoint = 'http://localhost:3000'
+  config.token = 'XXXXXXXXXXXXXXXXXXXXX'
+  config.token_secret = '*******************'
+end
+```
+
+## Usage
+
+```ruby
+select_stmt = 'select column_a, column_b from the_great_table; -- an awesome query shows amazing fact up'
+bundle = QueueryClient.query(select_stmt)
+bundle.each do |row|
+  # do some useful works
+  p row
+end
+```

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "queuery_client"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/queuery_client/basic_auth_garage_client.rb

@@ -0,0 +1,26 @@
+require 'garage_client'
+
+module QueueryClient
+  class BasicAuthGarageClient < GarageClient::Client
+    # Override
+    def apply_auth_middleware(faraday_builder)
+      faraday_builder.use Faraday::Request::BasicAuthentication, login, password
+    end
+
+    def login
+      options[:login]
+    end
+
+    def login=(login)
+      options[:login] = login
+    end
+
+    def password
+      options[:password]
+    end
+
+    def password=(password)
+      options[:password] = password
+    end
+  end
+end

data/lib/queuery_client/client.rb

@@ -0,0 +1,61 @@
+module QueueryClient
+  class Client
+    def initialize(options = {})
+      @options = options
+    end
+
+    def execute_query(select_stmt, values)
+      garage_client.post("/v1/queries", q: select_stmt, values: values)
+    end
+
+    def get_query(id)
+      garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix')
+    end
+
+    def wait_for(id)
+      loop do
+        query = get_query(id)
+        case query.status
+        when 'success', 'failed'
+          return query
+        end
+        sleep 3
+      end
+    end
+
+    def query_and_wait(select_stmt, values)
+      query = execute_query(select_stmt, values)
+      wait_for(query.id)
+    end
+
+    def query(select_stmt, values)
+      query = query_and_wait(select_stmt, values)
+      case query.status
+      when 'success'
+        QueueryDataFileBundle.new(
+          query.data_file_urls,
+          s3_prefix: query.s3_prefix,
+        )
+      when 'failed'
+        raise QueryError.new(query.error)
+      end
+    end
+
+    def garage_client
+      @garage_client ||= BasicAuthGarageClient.new(
+        endpoint: options.endpoint,
+        path_prefix: '/',
+        login: options.token,
+        password: options.token_secret
+      )
+    end
+
+    def options
+      default_options.merge(@options)
+    end
+
+    def default_options
+      QueueryClient.configuration
+    end
+  end
+end

data/lib/queuery_client/configuration.rb

@@ -0,0 +1,37 @@
+module QueueryClient
+  class Configuration
+    def initialize(options = {})
+      @options = options
+    end
+
+    def options
+      @options ||= {}
+    end
+
+    def reset
+      @options = nil
+    end
+
+    [
+      :endpoint,
+      :token,
+      :token_secret,
+    ].each do |key|
+      define_method(key) do
+        options.fetch(key)
+      end
+
+      define_method("#{key}=") do |value|
+        options[key] = value
+      end
+    end
+
+    def merge(other)
+      Configuration.new(to_h.merge(other.to_h))
+    end
+
+    def to_h
+      options
+    end
+  end
+end

data/lib/queuery_client/query_error.rb

@@ -0,0 +1,3 @@
+module QueueryClient
+  class QueryError < StandardError; end
+end

data/lib/queuery_client/queuery_data_file_bundle.rb

@@ -0,0 +1,19 @@
+require 'redshift_connector/url_data_file_bundle'
+
+module QueueryClient
+  class QueueryDataFileBundle < RedshiftConnector::UrlDataFileBundle
+    def initialize(url, s3_prefix:, **args)
+      super(url, **args)
+      @s3_prefix = s3_prefix
+    end
+
+    attr_reader :s3_prefix
+
+    def url
+      uri = data_files.first.url.dup
+      uri.query = nil
+      uri.path = File.dirname(uri.path)
+      uri.to_s
+    end
+  end
+end

data/lib/queuery_client/version.rb

@@ -0,0 +1,3 @@
+module QueueryClient
+  VERSION = "0.8.0"
+end

data/lib/queuery_client.rb

@@ -0,0 +1,22 @@
+require "queuery_client/version"
+require "queuery_client/configuration"
+require "queuery_client/basic_auth_garage_client"
+require "queuery_client/query_error"
+require "queuery_client/client"
+require "queuery_client/queuery_data_file_bundle"
+
+module QueueryClient
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure(&block)
+      configuration.instance_eval(&block)
+    end
+
+    def query(select_stmt, values = [])
+      Client.new.query(select_stmt, values)
+    end
+  end
+end

data/queuery_client.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'queuery_client/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "queuery_client"
+  spec.version       = QueueryClient::VERSION
+  spec.authors       = ["Hidekazu Kobayashi"]
+  spec.email         = ["hidekazu-kobayashi@cookpad.com"]
+  spec.license       = "MIT"
+
+  spec.summary       = "Client library for Queuery Redshift HTTP API"
+  spec.homepage      = "https://github.com/bricolages/queuery_client"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'garage_client'
+  spec.add_dependency "redshift-connector-data_file", ">= 7.1"
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "pry"
+end

data/queuery_client.rb

@@ -0,0 +1,64 @@
+require 'uri'
+require 'net/http'
+
+class QueueryClient
+  class << self
+    attr_accessor :host
+    attr_accessor :port
+
+    def query(select_stmt)
+      client = new(
+        host: host,
+        port: port,
+      )
+
+      res = client.query_and_wait(select_stmt)
+      RedshiftConnector::UrlDataFileBundle.new(res['data_objects'])
+    end
+  end
+
+  def initialize(host:, port: 80)
+    @host = host
+    @port = port
+  end
+
+  def request(req)
+    res = Net::HTTP.new(@host, @port).request(req)
+    JSON.parse(res.body)
+  end
+
+  def request_post(path, params = {})
+    post_req = Net::HTTP::Post.new(path)
+    post_req.form_data = params
+    request(post_req)
+  end
+
+  def request_get(path)
+    get_req = Net::HTTP::Get.new(path)
+    request(get_req)
+  end
+
+  def query(select_stmt)
+    request_post('/queries', { q: select_stmt })
+  end
+
+  def status(job_id)
+    request_get("/queries/#{job_id}")
+  end
+
+  def wait_for(job_id)
+    loop do
+      res = status(job_id)
+      case res['status']
+      when 'success', 'failed'
+        return res
+      end
+      sleep 1
+    end
+  end
+
+  def query_and_wait(select_stmt)
+    res = query(select_stmt)
+    wait_for(res['job_id'])
+  end
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: queuery_client
+version: !ruby/object:Gem::Version
+  version: 0.8.0
+platform: ruby
+authors:
+- Hidekazu Kobayashi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-03-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: garage_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redshift-connector-data_file
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- hidekazu-kobayashi@cookpad.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/queuery_client.rb
+- lib/queuery_client/basic_auth_garage_client.rb
+- lib/queuery_client/client.rb
+- lib/queuery_client/configuration.rb
+- lib/queuery_client/query_error.rb
+- lib/queuery_client/queuery_data_file_bundle.rb
+- lib/queuery_client/version.rb
+- queuery_client.gemspec
+- queuery_client.rb
+homepage: https://github.com/bricolages/queuery_client
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Client library for Queuery Redshift HTTP API
+test_files: []