redshift-connector-queuery 8.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d767e23fc7ef3ab2978e28744903fba36b7bb84f
+  data.tar.gz: ae3986f41bc6dde1bd7e3b3ca18ab0da426111d8
+SHA512:
+  metadata.gz: '09863e2679827b01eeaf9bd604401a78af0069e2c5e7d65f6e1138c57c69d7554df52df2b0fff9416502a4dcf73ae93faabdd69fdb622604ad11ce423a7bb6f9'
+  data.tar.gz: 65fd7ad93fbe99a87bffa6c372e3080d46bfbfc74812309489601ddf9d090a5122434ea06ee19b4965c8042be0d54281d762292a4634be1bd83dad6be440f0ca

data/README.md

@@ -0,0 +1,15 @@
+# redshift-connector-queuery
+
+Queuery Redshift HTTP API adapter for redshift-connector.
+
+## Setup
+
+Add `config/initializer/redshift-connector.rb` as following:
+```
+QueueryClient.configure do |config|
+  config.endpoint = "https://queuery.example.com"
+  config.token = "xxxxxxxxxx"
+  config.token_secret = "yyyyyyyyyy"
+end
+RedshiftConnector::Exporter.default_data_source = RedshiftConnector::QueueryDataSource.new
+```

data/lib/redshift-connector-queuery.rb

@@ -0,0 +1,2 @@
+require 'redshift_connector/queuery_data_source'
+require 'redshift_connector/queuery_exporter'

data/lib/redshift_connector/queuery_data_source.rb

@@ -0,0 +1,22 @@
+require 'redshift_connector/queuery_exporter'
+require 'redshift_connector/exporter_builder'
+require 'redshift_connector/exception'
+require 'queuery_client/client'
+
+module RedshiftConnector
+  class QueueryDataSource
+    def initialize(client = nil)
+      @client = client || QueueryClient::Client.new
+    end
+
+    def exporter_builder
+      ExporterBuilder.new(ds: self, exporter_class: QueueryExporter)
+    end
+
+    def execute_query(stmt, params = [])
+      @client.query(stmt, params)
+    rescue QueueryClient::QueryError => ex
+      raise ExportError, ex.message
+    end
+  end
+end

data/lib/redshift_connector/queuery_exporter.rb

@@ -0,0 +1,28 @@
+require 'redshift_connector/logger'
+
+module RedshiftConnector
+  class QueueryExporter
+    def initialize(ds:, query:, bundle_params: nil, enable_sort: false, logger: RedshiftConnector.logger)
+      @ds = ds
+      @query = query
+      @bundle_params = bundle_params
+      @bundle = nil
+      @enable_sort = enable_sort
+      @logger = logger
+    end
+
+    attr_reader :query
+    attr_reader :bundle_params
+    attr_reader :bundle
+    attr_reader :logger
+
+    def execute
+      @logger.info "EXPORT #{@query.description} -> (Queuery S3 tmp)"
+      stmt = @query.to_sql
+      @logger.info "[SQL/Queuery] #{stmt.strip}"
+      # FIXME: support enable_sort
+      @bundle = @ds.execute_query(stmt)
+      @bundle
+    end
+  end
+end

data/test/config.rb

@@ -0,0 +1,16 @@
+GarageClient.configure do |config|
+  config.name = "redshift-connector-queuery"
+end
+
+QueueryClient.configure do |config|
+  config.endpoint = 'https://queuery.ckpd.co'
+  # app: aamine-test
+  config.token = "606C6UDNdYGz7RY9Y0Cd2ZqTcmTHqvMlxY5KxB2md84"
+  config.token_secret = "kgLOgJgbi8_XMfK6bEUbMFWJT6-LZBYx_om4ZtBvGzIuHuC7YCeWNLUHtqUIgWxMQ0rtlaVMzLk9gOCb-J50yw"
+end
+
+$TEST_SCHEMA = 'aamine'
+
+module RedshiftConnector
+  Exporter.default_data_source = QueueryDataSource.new
+end

data/test/database.yml

@@ -0,0 +1,6 @@
+mysql:
+  adapter: mysql2
+  host: localhost
+  username: minero-aoki
+  database: test
+  encoding: utf8

data/test/helper.rb

@@ -0,0 +1,26 @@
+require 'active_record'
+require 'activerecord-import'
+require 'activerecord-import/base'
+ActiveRecord::Import.require_adapter 'mysql2'
+require 'redshift-connector'
+require 'redshift-connector-queuery'
+require 'garage_client'
+require 'queuery_client'
+require 'yaml'
+require 'logger'
+
+YAML.load_file("#{__dir__}/database.yml").each do |name, ent|
+  ActiveRecord::Base.configurations[name] = ent
+end
+
+class BaseConn < ActiveRecord::Base
+  establish_connection :mysql
+  self.abstract_class = true
+end
+class ItemPv < BaseConn
+  connection
+end
+
+require_relative 'config'
+
+RedshiftConnector.logger = Logger.new($stderr)

data/test/test_connector.rb

@@ -0,0 +1,22 @@
+require_relative 'helper'
+require 'test/unit'
+
+class TestConnector < Test::Unit::TestCase
+  def test_transport
+    data_date = '2016-11-03'
+    job = RedshiftConnector.transport_delta(
+      schema: $TEST_SCHEMA,
+      table: 'item_pvs',
+
+      txn_id: data_date,
+      condition: %Q(data_date = date '#{data_date}'),
+
+      columns: %w[id data_date item_id pv uu],
+      upsert_columns: %w[pv uu],
+      filter: -> (id, data_date, item_id, pv, uu) {
+        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+      }
+    )
+    job.execute
+  end
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: redshift-connector-queuery
+version: !ruby/object:Gem::Version
+  version: 8.0.0
+platform: ruby
+authors:
+- Minero Aoki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-03-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redshift-connector
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.1
+- !ruby/object:Gem::Dependency
+  name: redshift-connector-data_file
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.1.1
+- !ruby/object:Gem::Dependency
+  name: queuery_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- aamine@loveruby.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/redshift-connector-queuery.rb
+- lib/redshift_connector/queuery_data_source.rb
+- lib/redshift_connector/queuery_exporter.rb
+- test/config.rb
+- test/database.yml
+- test/helper.rb
+- test/test_connector.rb
+homepage: https://github.com/bricolages/redshift-connector-queuery
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Queuery adapter for redshift-connector
+test_files: []