turboquery 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e52ed1a1ab83ae67ea386d648de86f4cb8852fa9
-  data.tar.gz: c8964efb1208aa21975ea6a6bd07680f2c7124e5
+  metadata.gz: 9dfdef1436e235cb0347f34b212ca4138c96ace1
+  data.tar.gz: ca35a1c264d0b6b5a01847af7833c0f97d1438e8
 SHA512:
-  metadata.gz: 0ca6e4c6bdbb3118f6e1e948e06910c61abb5494564279d1f52fb76a0a309f565528bad532335a4ea2b2d9dd2be337f4ad75264e25d153e39918ee14366a40c5
-  data.tar.gz: a894f45341fd2f2b2c260b8df329703075cac03c4da53d73e12ec7d1f218e1fea184fd467d6d9dfa9765490a98195be846a8c23a366d1224ac6990789061184e
+  metadata.gz: 8c23d04a206e1ac6d233a3ca3e9134d6cc1c5802b580d68c98a5eceb0bc8c0b5ebe4b9dfdbc6df8a60e88f15f5f9e7ab298bd3ad7b60838b89792e98d96a9e6d
+  data.tar.gz: 7db02852560b18969e574eeca25f4ba6366270b3d07247f898ba9f27ecef135e3f598f7fa83648836ab0c888a90f3f00ed80b6c148031a8a0467d8ca9fb28c37

data/README.md

@@ -1,3 +1,3 @@
-# turboquery [![Build Status](https://travis-ci.org/playlist-media/turboquery.svg?branch=master)](https://travis-ci.org/playlist-media/turboquery) [![Code Climate](https://codeclimate.com/repos/54e54f5e69568049a90046d9/badges/2c9a4a68230e9e594789/gpa.svg)](https://codeclimate.com/repos/54e54f5e69568049a90046d9/feed)
+# turboquery [![Gem](https://img.shields.io/gem/v/formatador.svg)](https://rubygems.org/gems/turboquery) [![Build Status](https://travis-ci.org/playlist-media/turboquery.svg?branch=master)](https://travis-ci.org/playlist-media/turboquery) [![Code Climate](https://codeclimate.com/repos/54e54f5e69568049a90046d9/badges/2c9a4a68230e9e594789/gpa.svg)](https://codeclimate.com/repos/54e54f5e69568049a90046d9/feed)
 
 :gem: Turboquery executes PostgreSQL queries against your PostgreSQL database utilizing Redshift to massively speed up complex queries.

data/lib/turboquery/connection.rb

@@ -98,10 +98,6 @@ class Turboquery::Connection
     object.upload_file(filename)
   end
 
-  class AR < ActiveRecord::Base
-    self.abstract_class = true
-  end
-
   module PsqlEnv
     def self.set(config)
       host(config)     if config[:host]

data/lib/turboquery/database_url.rb

@@ -0,0 +1,43 @@
+require 'uri'
+require 'cgi'
+
+class Turboquery::DatabaseUrl
+  def initialize(url = nil)
+    @uri = URI.parse(url || ENV['DATABASE_URL'])
+    @query = CGI.parse @uri.query.to_s
+  end
+
+  def to_hash
+    build_hash
+    @hash
+  end
+
+  private
+
+  def build_hash
+    @hash = {
+      adapter: @uri.scheme,
+      host: @uri.host,
+      database: File.basename(@uri.path)
+    }
+
+    set_optional :port, @uri.port
+    set_optional :username, @uri.user
+    set_optional :password, @uri.password
+    set_optional :encoding, encoding
+    set_optional :pool, pool
+  end
+
+  def set_optional(key, val)
+    @hash[key] = val if val
+  end
+
+  def encoding
+    @query.key?('encoding') ? @query['encoding'][0] : nil
+  end
+
+  def pool
+    pool = @query['pool'] || @query['max_connections']
+    pool.length > 0 ? pool[0].to_i : nil
+  end
+end

data/lib/turboquery/olap.rb

@@ -5,12 +5,12 @@ class Turboquery::OLAP < Turboquery::Connection
 
   def copy_result_to_s3(query)
     key = random_key
-    execute("UNLOAD ('#{query}') TO 's3://#{Turboquery.s3_config['bucket']}/#{key}' #{copy_options};")
+    execute("UNLOAD ('#{query}') TO 's3://#{Turboquery.aws_bucket}/#{key}' #{copy_options};")
     key
   end
 
   def copy_s3_to_table(key, table)
-    execute("COPY #{table} FROM 's3://#{Turboquery.s3_config['bucket']}/#{key}manifest' #{copy_options}
+    execute("COPY #{table} FROM 's3://#{Turboquery.aws_bucket}/#{key}manifest' #{copy_options}
     DATEFORMAT 'auto' TIMEFORMAT 'auto';")
   end
 
@@ -27,13 +27,13 @@ class Turboquery::OLAP < Turboquery::Connection
   end
 
   def self.after_fork
-    ARRedshift.reconnect
+    AROLAP.connect
   end
 
   protected
 
   def connection
-    ARRedshift.connection
+    AROLAP.connection
   end
 
   def excape_single_quotes(str)
@@ -41,15 +41,13 @@ class Turboquery::OLAP < Turboquery::Connection
   end
 
   def copy_options
-    "CREDENTIALS 'aws_access_key_id=#{Turboquery.s3_config['access_key_id']};aws_secret_access_key=#{Turboquery.s3_config['secret_access_key']}'
+    "CREDENTIALS 'aws_access_key_id=#{Turboquery.aws_key};aws_secret_access_key=#{Turboquery.aws_secret}'
      MANIFEST DELIMITER '\\t' NULL AS '\\\\N'"
   end
 
-  class ARRedshift < ActiveRecord::Base
-    establish_connection DatabaseUrl.new(ENV['REDSHIFT_DATABASE_URL']).to_hash
-
-    def self.reconnect
-      establish_connection DatabaseUrl.new(ENV['REDSHIFT_DATABASE_URL']).to_hash
+  class AROLAP < ActiveRecord::Base
+    def self.connect
+      establish_connection Turboquery::DatabaseUrl.new(Turboquery.olap_database_url).to_hash
     end
   end
 end

data/lib/turboquery/oltp.rb

@@ -21,4 +21,20 @@ class Turboquery::OLTP < Turboquery::Connection
     command = "cat #{Shellwords.escape(path)} | psql -c #{Shellwords.escape(sql)} #{Shellwords.escape(config[:database])}"
     Kernel.system(command)
   end
+
+  def self.after_fork
+    AROLTP.connect
+  end
+
+  protected
+
+  def connection
+    AROLTP.connection
+  end
+
+  class AROLTP < ActiveRecord::Base
+    def self.connect
+      establish_connection Turboquery::DatabaseUrl.new(Turboquery.oltp_database_url).to_hash
+    end
+  end
 end

data/lib/turboquery/version.rb

@@ -1,3 +1,3 @@
 module Turboquery
-  VERSION = '0.1.0'
+  VERSION = '1.0.0'
 end

data/lib/turboquery.rb

@@ -1,5 +1,27 @@
+require 'active_record'
+require 'pg_query'
+
+module Turboquery
+end
+
+require 'turboquery/database_url'
+require 'turboquery/connection'
+require 'turboquery/oltp'
+require 'turboquery/olap'
+require 'turboquery/remote_query'
+require 'turboquery/table_mover'
+require 'turboquery/version'
+
 module Turboquery
   class << self
+    attr_accessor :oltp_database_url
+    attr_accessor :olap_database_url
+    attr_accessor :aws_key
+    attr_accessor :aws_secret
+    attr_accessor :aws_bucket
+    attr_accessor :aws_region
+    attr_accessor :tmp_path
+
     def query(sql, opts = {})
       Turboquery::RemoteQuery.new(sql, opts).execute
     end
@@ -19,24 +41,32 @@ module Turboquery
     def s3_bucket
       @s3_bucket ||= begin
         Aws::S3::Resource.new(
-          region: 'us-east-1',
-          access_key_id: s3_config['access_key_id'],
-          secret_access_key: s3_config['secret_access_key']
-        ).bucket(s3_config['bucket'])
+          region: aws_region,
+          access_key_id: aws_key,
+          secret_access_key: aws_secret
+        ).bucket(aws_bucket)
       end
     end
 
-    def s3_config
-      Rails.application.secrets.turboquery
-    end
-
-    def tmp_path
-      ENV['TURBOQUERY_TMP_PATH'] || Rails.root.join('tmp')
-    end
-
     def after_fork
       olap.after_fork
       oltp.after_fork
     end
   end
+
+  self.oltp_database_url = ENV['TURBOQUERY_OLTP_DATABASE_URL']
+  self.olap_database_url = ENV['TURBOQUERY_OLAP_DATABASE_URL']
+  self.aws_key = ENV['TURBOQUERY_AWS_KEY']
+  self.aws_secret = ENV['TURBOQUERY_AWS_SECRET']
+  self.aws_bucket = ENV['TURBOQUERY_AWS_BUCKET']
+  self.aws_region = ENV['TURBOQUERY_AWS_REGION'] || 'us-east-1'
+  self.tmp_path = ENV['TURBOQUERY_TMP_PATH']
+
+  oltp.after_fork if oltp_database_url
+  olap.after_fork if olap_database_url
+
+  def self.config
+    yield self
+    after_fork
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: turboquery
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Jacob Gillespie
@@ -97,6 +97,7 @@ files:
 - Rakefile
 - lib/turboquery.rb
 - lib/turboquery/connection.rb
+- lib/turboquery/database_url.rb
 - lib/turboquery/olap.rb
 - lib/turboquery/oltp.rb
 - lib/turboquery/remote_query.rb