queuery_client 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 893d3e23c931f4f67621e03306cb3523e2bf86affe3a4c7bd6958e5cd4a7aa12
+  data.tar.gz: 1fbcebdefffabad096aee653ab255329696b8e5b6e2056f4742a619ec15e3810
+SHA512:
+  metadata.gz: 4e7ac1fc3d9b850fc0d7d9261c1db44664511997ec17789dd153d9faf9296af7caa689561d30cfde7e07918d43331fbd11b92a1704c9781b489bb3b17704a4e3
+  data.tar.gz: 59ca2f54c33e271ffa93d59c224fa98ae11b729152407b5b7691413bce1ca2b4a2ec6b14f9a6c6253cade2fe1bee290aa12f6496daa821441e9d7e9092b3cf91

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in queuery_client.gemspec
+gemspec

data/README.md

@@ -0,0 +1,51 @@
+# QueueryClient
+
+Queuery client for Ruby.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'queuery_client'
+```
+
+## Configuration
+
+### If you don't use Rails
+
+```ruby
+# configuration
+RedshiftConnector.logger = Logger.new($stdout)
+GarageClient.configure do |config|
+  config.name = "queuery-example"
+end
+QueueryClient.configure do |config|
+  config.endpoint = 'http://localhost:3000'
+  config.token = 'XXXXXXXXXXXXXXXXXXXXX'
+  config.token_secret = '*******************'
+end
+```
+
+### If you are on Rails
+
+In `config/initializers/queuery.rb`:
+
+```ruby
+QueueryClient.configure do |config|
+  config.endpoint = 'http://localhost:3000'
+  config.token = 'XXXXXXXXXXXXXXXXXXXXX'
+  config.token_secret = '*******************'
+end
+```
+
+## Usage
+
+```ruby
+select_stmt = 'select column_a, column_b from the_great_table; -- an awesome query shows amazing fact up'
+bundle = QueueryClient.query(select_stmt)
+bundle.each do |row|
+  # do some useful works
+  p row
+end
+```

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "queuery_client"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/queuery_client.rb

@@ -0,0 +1,23 @@
+require "queuery_client/version"
+require "queuery_client/configuration"
+require "queuery_client/basic_auth_garage_client"
+require "queuery_client/query_error"
+require "queuery_client/client"
+require "queuery_client/url_data_file_bundle"
+require "queuery_client/s3_data_file_bundle"
+
+module QueueryClient
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure(&block)
+      configuration.instance_eval(&block)
+    end
+
+    def query(select_stmt, values = [])
+      Client.new.query(select_stmt, values)
+    end
+  end
+end

data/lib/queuery_client/basic_auth_garage_client.rb

@@ -0,0 +1,26 @@
+require 'garage_client'
+
+module QueueryClient
+  class BasicAuthGarageClient < GarageClient::Client
+    # Override
+    def apply_auth_middleware(faraday_builder)
+      faraday_builder.use Faraday::Request::BasicAuthentication, login, password
+    end
+
+    def login
+      options[:login]
+    end
+
+    def login=(login)
+      options[:login] = login
+    end
+
+    def password
+      options[:password]
+    end
+
+    def password=(password)
+      options[:password] = password
+    end
+  end
+end

data/lib/queuery_client/client.rb

@@ -0,0 +1,84 @@
+module QueueryClient
+  class Client
+    def initialize(options = {})
+      @options = options
+    end
+
+    def execute_query(select_stmt, values)
+      garage_client.post("/v1/queries", q: select_stmt, values: values)
+    end
+    alias start_query execute_query
+
+    def get_query(id)
+      garage_client.get("/v1/queries/#{id}", fields: '__default__,s3_prefix')
+    end
+
+    def wait_for(id)
+      loop do
+        query = get_query(id)
+        case query.status
+        when 'success', 'failed'
+          return query
+        end
+        sleep 3
+      end
+    end
+
+    def query_and_wait(select_stmt, values)
+      query = execute_query(select_stmt, values)
+      wait_for(query.id)
+    end
+
+    def query(select_stmt, values)
+      query = query_and_wait(select_stmt, values)
+      case query.status
+      when 'success'
+        UrlDataFileBundle.new(
+          query.data_file_urls,
+          s3_prefix: query.s3_prefix,
+        )
+      when 'failed'
+        raise QueryError.new(query.error)
+      end
+    end
+
+    # poll_result returns the results only if the query has already successed.
+    def poll_result(id)
+      query = get_query(id)
+      get_query_result(query)
+    end
+
+    def garage_client
+      @garage_client ||= BasicAuthGarageClient.new(
+        endpoint: options.endpoint,
+        path_prefix: '/',
+        login: options.token,
+        password: options.token_secret
+      )
+    end
+
+    def options
+      default_options.merge(@options)
+    end
+
+    def default_options
+      QueueryClient.configuration
+    end
+
+    private
+
+    def get_query_result(query)
+      case query.status
+      when 'pending', 'running'
+        nil
+      when 'success'
+        UrlDataFileBundle.new(
+          query.data_file_urls,
+          s3_prefix: query.s3_prefix,
+        )
+      when 'failure'
+        raise QueryError.new(query.error)
+      end
+    end
+  end
+end

data/lib/queuery_client/configuration.rb

@@ -0,0 +1,37 @@
+module QueueryClient
+  class Configuration
+    def initialize(options = {})
+      @options = options
+    end
+
+    def options
+      @options ||= {}
+    end
+
+    def reset
+      @options = nil
+    end
+
+    [
+      :endpoint,
+      :token,
+      :token_secret,
+    ].each do |key|
+      define_method(key) do
+        options.fetch(key)
+      end
+
+      define_method("#{key}=") do |value|
+        options[key] = value
+      end
+    end
+
+    def merge(other)
+      Configuration.new(to_h.merge(other.to_h))
+    end
+
+    def to_h
+      options
+    end
+  end
+end

data/lib/queuery_client/data_file.rb

@@ -0,0 +1,26 @@
+require 'redshift_csv_file'
+require 'zlib'
+
+module QueueryClient
+  class DataFile
+    def data_object?
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
+    end
+
+    def gzipped_object?
+      File.extname(key) == '.gz'
+    end
+
+    def each_row(&block)
+      f = open
+      begin
+        if gzipped_object?
+          f = Zlib::GzipReader.new(f)
+        end
+        RedshiftCsvFile.new(f).each(&block)
+      ensure
+        f.close
+      end
+    end
+  end
+end

data/lib/queuery_client/data_file_bundle.rb

@@ -0,0 +1,15 @@
+module QueueryClient
+  class DataFileBundle
+    # abstract data_files :: [DataFile]
+
+    def each_row(&block)
+      data_files.each do |file|
+        if file.data_object?
+          file.each_row(&block)
+        end
+      end
+    end
+
+    alias each each_row
+  end
+end

data/lib/queuery_client/query_error.rb

@@ -0,0 +1,3 @@
+module QueueryClient
+  class QueryError < StandardError; end
+end

data/lib/queuery_client/s3_data_file.rb

@@ -0,0 +1,18 @@
+require 'queuery_client/data_file'
+require 'forwardable'
+
+module QueueryClient
+  class S3DataFile < DataFile
+    extend Forwardable
+
+    def initialize(object)
+      @object = object
+    end
+
+    def_delegators '@object', :url, :key, :presigned_url
+
+    def open
+      @object.get.body
+    end
+  end
+end

data/lib/queuery_client/s3_data_file_bundle.rb

@@ -0,0 +1,28 @@
+require 'queuery_client/data_file_bundle'
+require 'queuery_client/s3_data_file'
+require 'aws-sdk-s3'
+require 'logger'
+
+module QueueryClient
+  class S3DataFileBundle < DataFileBundle
+    def initialize(bucket, prefix, s3_client: nil, logger: Logger.new($stderr))
+      @s3_client = s3_client || Aws::S3::Client.new   # Use env to inject credentials
+      @bucket = bucket
+      @prefix = prefix
+      @logger = logger
+    end
+
+    attr_reader :bucket
+    attr_reader :prefix
+    attr_reader :logger
+
+    def url
+      "s3://#{@bucket}/#{@prefix}"
+    end
+
+    def data_files
+      b = Aws::S3::Resource.new(client: @s3_client).bucket(@bucket)
+      b.objects(prefix: @prefix).map {|obj| S3DataFile.new(obj) }
+    end
+  end
+end

data/lib/queuery_client/url_data_file.rb

@@ -0,0 +1,27 @@
+require 'queuery_client/data_file'
+require 'net/http'
+require 'stringio'
+
+module QueueryClient
+  class UrlDataFile < DataFile
+    def initialize(url)
+      @url = url
+    end
+
+    attr_reader :url
+
+    def key
+      @url.path
+    end
+
+    def open
+      http = Net::HTTP.new(@url.host, @url.port)
+      http.use_ssl = (@url.scheme.downcase == 'https')
+      content = http.start {
+        res = http.get(@url.request_uri)
+        res.body
+      }
+      StringIO.new(content)
+    end
+  end
+end

data/lib/queuery_client/url_data_file_bundle.rb

@@ -0,0 +1,33 @@
+require 'queuery_client/data_file_bundle'
+require 'queuery_client/url_data_file'
+require 'uri'
+require 'logger'
+
+module QueueryClient
+  class UrlDataFileBundle < DataFileBundle
+    def initialize(urls, s3_prefix:, logger: Logger.new($stderr))
+      raise ArgumentError, 'no URL given' if urls.empty?
+      @data_files = urls.map {|url| UrlDataFile.new(URI.parse(url)) }
+      @s3_prefix = s3_prefix
+      @logger = logger
+    end
+
+    attr_reader :data_files
+    attr_reader :s3_prefix
+    attr_reader :logger
+
+    def url
+      uri = data_files.first.url.dup
+      uri.query = nil
+      uri.path = File.dirname(uri.path)
+      uri.to_s
+    end
+
+    def direct(bucket_opts = {}, bundle_opts = {})
+      s3_uri = URI.parse(s3_prefix)
+      bucket = s3_uri.host
+      prefix = s3_uri.path[1..-1] # trim heading slash
+      S3DataFileBundle.new(bucket, prefix)
+    end
+  end
+end

data/lib/queuery_client/version.rb

@@ -0,0 +1,3 @@
+module QueueryClient
+  VERSION = "1.0.2"
+end

data/queuery_client.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'queuery_client/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "queuery_client"
+  spec.version       = QueueryClient::VERSION
+  spec.authors       = ["Hidekazu Kobayashi"]
+  spec.email         = ["hidekazu-kobayashi@cookpad.com"]
+  spec.license       = "MIT"
+
+  spec.summary       = "Client library for Queuery Redshift HTTP API"
+  spec.homepage      = "https://github.com/bricolages/queuery_client"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'garage_client'
+  spec.add_dependency 'redshift_csv_file'
+  spec.add_dependency 'aws-sdk-s3'
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "pry"
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: queuery_client
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+platform: ruby
+authors:
+- Hidekazu Kobayashi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-07-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: garage_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redshift_csv_file
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- hidekazu-kobayashi@cookpad.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/queuery_client.rb
+- lib/queuery_client/basic_auth_garage_client.rb
+- lib/queuery_client/client.rb
+- lib/queuery_client/configuration.rb
+- lib/queuery_client/data_file.rb
+- lib/queuery_client/data_file_bundle.rb
+- lib/queuery_client/query_error.rb
+- lib/queuery_client/s3_data_file.rb
+- lib/queuery_client/s3_data_file_bundle.rb
+- lib/queuery_client/url_data_file.rb
+- lib/queuery_client/url_data_file_bundle.rb
+- lib/queuery_client/version.rb
+- queuery_client.gemspec
+homepage: https://github.com/bricolages/queuery_client
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Client library for Queuery Redshift HTTP API
+test_files: []