turboquery 0.1.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/turboquery/connection.rb +0 -4
- data/lib/turboquery/database_url.rb +43 -0
- data/lib/turboquery/olap.rb +8 -10
- data/lib/turboquery/oltp.rb +16 -0
- data/lib/turboquery/version.rb +1 -1
- data/lib/turboquery.rb +42 -12
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9dfdef1436e235cb0347f34b212ca4138c96ace1
|
4
|
+
data.tar.gz: ca35a1c264d0b6b5a01847af7833c0f97d1438e8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8c23d04a206e1ac6d233a3ca3e9134d6cc1c5802b580d68c98a5eceb0bc8c0b5ebe4b9dfdbc6df8a60e88f15f5f9e7ab298bd3ad7b60838b89792e98d96a9e6d
|
7
|
+
data.tar.gz: 7db02852560b18969e574eeca25f4ba6366270b3d07247f898ba9f27ecef135e3f598f7fa83648836ab0c888a90f3f00ed80b6c148031a8a0467d8ca9fb28c37
|
data/README.md
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
# turboquery [![Build Status](https://travis-ci.org/playlist-media/turboquery.svg?branch=master)](https://travis-ci.org/playlist-media/turboquery) [![Code Climate](https://codeclimate.com/repos/54e54f5e69568049a90046d9/badges/2c9a4a68230e9e594789/gpa.svg)](https://codeclimate.com/repos/54e54f5e69568049a90046d9/feed)
|
1
|
+
# turboquery [![Gem](https://img.shields.io/gem/v/formatador.svg)](https://rubygems.org/gems/turboquery) [![Build Status](https://travis-ci.org/playlist-media/turboquery.svg?branch=master)](https://travis-ci.org/playlist-media/turboquery) [![Code Climate](https://codeclimate.com/repos/54e54f5e69568049a90046d9/badges/2c9a4a68230e9e594789/gpa.svg)](https://codeclimate.com/repos/54e54f5e69568049a90046d9/feed)
|
2
2
|
|
3
3
|
:gem: Turboquery executes PostgreSQL queries against your PostgreSQL database utilizing Redshift to massively speed up complex queries.
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'cgi'
|
3
|
+
|
4
|
+
class Turboquery::DatabaseUrl
|
5
|
+
def initialize(url = nil)
|
6
|
+
@uri = URI.parse(url || ENV['DATABASE_URL'])
|
7
|
+
@query = CGI.parse @uri.query.to_s
|
8
|
+
end
|
9
|
+
|
10
|
+
def to_hash
|
11
|
+
build_hash
|
12
|
+
@hash
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def build_hash
|
18
|
+
@hash = {
|
19
|
+
adapter: @uri.scheme,
|
20
|
+
host: @uri.host,
|
21
|
+
database: File.basename(@uri.path)
|
22
|
+
}
|
23
|
+
|
24
|
+
set_optional :port, @uri.port
|
25
|
+
set_optional :username, @uri.user
|
26
|
+
set_optional :password, @uri.password
|
27
|
+
set_optional :encoding, encoding
|
28
|
+
set_optional :pool, pool
|
29
|
+
end
|
30
|
+
|
31
|
+
def set_optional(key, val)
|
32
|
+
@hash[key] = val if val
|
33
|
+
end
|
34
|
+
|
35
|
+
def encoding
|
36
|
+
@query.key?('encoding') ? @query['encoding'][0] : nil
|
37
|
+
end
|
38
|
+
|
39
|
+
def pool
|
40
|
+
pool = @query['pool'] || @query['max_connections']
|
41
|
+
pool.length > 0 ? pool[0].to_i : nil
|
42
|
+
end
|
43
|
+
end
|
data/lib/turboquery/olap.rb
CHANGED
@@ -5,12 +5,12 @@ class Turboquery::OLAP < Turboquery::Connection
|
|
5
5
|
|
6
6
|
def copy_result_to_s3(query)
|
7
7
|
key = random_key
|
8
|
-
execute("UNLOAD ('#{query}') TO 's3://#{Turboquery.
|
8
|
+
execute("UNLOAD ('#{query}') TO 's3://#{Turboquery.aws_bucket}/#{key}' #{copy_options};")
|
9
9
|
key
|
10
10
|
end
|
11
11
|
|
12
12
|
def copy_s3_to_table(key, table)
|
13
|
-
execute("COPY #{table} FROM 's3://#{Turboquery.
|
13
|
+
execute("COPY #{table} FROM 's3://#{Turboquery.aws_bucket}/#{key}manifest' #{copy_options}
|
14
14
|
DATEFORMAT 'auto' TIMEFORMAT 'auto';")
|
15
15
|
end
|
16
16
|
|
@@ -27,13 +27,13 @@ class Turboquery::OLAP < Turboquery::Connection
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def self.after_fork
|
30
|
-
|
30
|
+
AROLAP.connect
|
31
31
|
end
|
32
32
|
|
33
33
|
protected
|
34
34
|
|
35
35
|
def connection
|
36
|
-
|
36
|
+
AROLAP.connection
|
37
37
|
end
|
38
38
|
|
39
39
|
def excape_single_quotes(str)
|
@@ -41,15 +41,13 @@ class Turboquery::OLAP < Turboquery::Connection
|
|
41
41
|
end
|
42
42
|
|
43
43
|
def copy_options
|
44
|
-
"CREDENTIALS 'aws_access_key_id=#{Turboquery.
|
44
|
+
"CREDENTIALS 'aws_access_key_id=#{Turboquery.aws_key};aws_secret_access_key=#{Turboquery.aws_secret}'
|
45
45
|
MANIFEST DELIMITER '\\t' NULL AS '\\\\N'"
|
46
46
|
end
|
47
47
|
|
48
|
-
class
|
49
|
-
|
50
|
-
|
51
|
-
def self.reconnect
|
52
|
-
establish_connection DatabaseUrl.new(ENV['REDSHIFT_DATABASE_URL']).to_hash
|
48
|
+
class AROLAP < ActiveRecord::Base
|
49
|
+
def self.connect
|
50
|
+
establish_connection Turboquery::DatabaseUrl.new(Turboquery.olap_database_url).to_hash
|
53
51
|
end
|
54
52
|
end
|
55
53
|
end
|
data/lib/turboquery/oltp.rb
CHANGED
@@ -21,4 +21,20 @@ class Turboquery::OLTP < Turboquery::Connection
|
|
21
21
|
command = "cat #{Shellwords.escape(path)} | psql -c #{Shellwords.escape(sql)} #{Shellwords.escape(config[:database])}"
|
22
22
|
Kernel.system(command)
|
23
23
|
end
|
24
|
+
|
25
|
+
def self.after_fork
|
26
|
+
AROLTP.connect
|
27
|
+
end
|
28
|
+
|
29
|
+
protected
|
30
|
+
|
31
|
+
def connection
|
32
|
+
AROLTP.connection
|
33
|
+
end
|
34
|
+
|
35
|
+
class AROLTP < ActiveRecord::Base
|
36
|
+
def self.connect
|
37
|
+
establish_connection Turboquery::DatabaseUrl.new(Turboquery.oltp_database_url).to_hash
|
38
|
+
end
|
39
|
+
end
|
24
40
|
end
|
data/lib/turboquery/version.rb
CHANGED
data/lib/turboquery.rb
CHANGED
@@ -1,5 +1,27 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'pg_query'
|
3
|
+
|
4
|
+
module Turboquery
|
5
|
+
end
|
6
|
+
|
7
|
+
require 'turboquery/database_url'
|
8
|
+
require 'turboquery/connection'
|
9
|
+
require 'turboquery/oltp'
|
10
|
+
require 'turboquery/olap'
|
11
|
+
require 'turboquery/remote_query'
|
12
|
+
require 'turboquery/table_mover'
|
13
|
+
require 'turboquery/version'
|
14
|
+
|
1
15
|
module Turboquery
|
2
16
|
class << self
|
17
|
+
attr_accessor :oltp_database_url
|
18
|
+
attr_accessor :olap_database_url
|
19
|
+
attr_accessor :aws_key
|
20
|
+
attr_accessor :aws_secret
|
21
|
+
attr_accessor :aws_bucket
|
22
|
+
attr_accessor :aws_region
|
23
|
+
attr_accessor :tmp_path
|
24
|
+
|
3
25
|
def query(sql, opts = {})
|
4
26
|
Turboquery::RemoteQuery.new(sql, opts).execute
|
5
27
|
end
|
@@ -19,24 +41,32 @@ module Turboquery
|
|
19
41
|
def s3_bucket
|
20
42
|
@s3_bucket ||= begin
|
21
43
|
Aws::S3::Resource.new(
|
22
|
-
region:
|
23
|
-
access_key_id:
|
24
|
-
secret_access_key:
|
25
|
-
).bucket(
|
44
|
+
region: aws_region,
|
45
|
+
access_key_id: aws_key,
|
46
|
+
secret_access_key: aws_secret
|
47
|
+
).bucket(aws_bucket)
|
26
48
|
end
|
27
49
|
end
|
28
50
|
|
29
|
-
def s3_config
|
30
|
-
Rails.application.secrets.turboquery
|
31
|
-
end
|
32
|
-
|
33
|
-
def tmp_path
|
34
|
-
ENV['TURBOQUERY_TMP_PATH'] || Rails.root.join('tmp')
|
35
|
-
end
|
36
|
-
|
37
51
|
def after_fork
|
38
52
|
olap.after_fork
|
39
53
|
oltp.after_fork
|
40
54
|
end
|
41
55
|
end
|
56
|
+
|
57
|
+
self.oltp_database_url = ENV['TURBOQUERY_OLTP_DATABASE_URL']
|
58
|
+
self.olap_database_url = ENV['TURBOQUERY_OLAP_DATABASE_URL']
|
59
|
+
self.aws_key = ENV['TURBOQUERY_AWS_KEY']
|
60
|
+
self.aws_secret = ENV['TURBOQUERY_AWS_SECRET']
|
61
|
+
self.aws_bucket = ENV['TURBOQUERY_AWS_BUCKET']
|
62
|
+
self.aws_region = ENV['TURBOQUERY_AWS_REGION'] || 'us-east-1'
|
63
|
+
self.tmp_path = ENV['TURBOQUERY_TMP_PATH']
|
64
|
+
|
65
|
+
oltp.after_fork if oltp_database_url
|
66
|
+
olap.after_fork if olap_database_url
|
67
|
+
|
68
|
+
def self.config
|
69
|
+
yield self
|
70
|
+
after_fork
|
71
|
+
end
|
42
72
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: turboquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jacob Gillespie
|
@@ -97,6 +97,7 @@ files:
|
|
97
97
|
- Rakefile
|
98
98
|
- lib/turboquery.rb
|
99
99
|
- lib/turboquery/connection.rb
|
100
|
+
- lib/turboquery/database_url.rb
|
100
101
|
- lib/turboquery/olap.rb
|
101
102
|
- lib/turboquery/oltp.rb
|
102
103
|
- lib/turboquery/remote_query.rb
|