turboquery 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e52ed1a1ab83ae67ea386d648de86f4cb8852fa9
+  data.tar.gz: c8964efb1208aa21975ea6a6bd07680f2c7124e5
+SHA512:
+  metadata.gz: 0ca6e4c6bdbb3118f6e1e948e06910c61abb5494564279d1f52fb76a0a309f565528bad532335a4ea2b2d9dd2be337f4ad75264e25d153e39918ee14366a40c5
+  data.tar.gz: a894f45341fd2f2b2c260b8df329703075cac03c4da53d73e12ec7d1f218e1fea184fd467d6d9dfa9765490a98195be846a8c23a366d1224ac6990789061184e

data/.editorconfig

@@ -0,0 +1,21 @@
+# editorconfig.org
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.go]
+indent_style = tab
+indent_size = 4
+
+[Makefile]
+indent_style = tab
+indent_size = 4

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+*.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.log
+.DS_Store
+.ruby-*

data/.rubocop.yml

@@ -0,0 +1,35 @@
+# Disable for db/schema.rb
+AllCops:
+  Exclude:
+    - db/**/*
+
+# Disable requirement for documentation
+Documentation:
+  Enabled: false
+
+# Allow extra lines around class body
+EmptyLinesAroundClassBody:
+  Enabled: false
+
+# Allow extra lines around module body
+EmptyLinesAroundModuleBody:
+  Enabled: false
+
+# Allow extra lines inside block body
+EmptyLinesAroundBlockBody:
+  Enabled: false
+
+# Use single quotes unless necessary
+StringLiterals:
+  EnforcedStyle: single_quotes
+
+# Only treat is_ as a predicate name
+PredicateName:
+  NamePrefixBlacklist:
+    - is_
+
+LineLength:
+  Enabled: false
+
+ClassAndModuleChildren:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in turboquery.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2015 Jacob Gillespie <jacobwgillespie@gmail.com>
+
+Permission  is  hereby granted, free of charge, to any person ob-
+taining a copy of  this  software  and  associated  documentation
+files  (the "Software"), to deal in the Software without restric-
+tion, including without limitation the rights to use, copy, modi-
+fy, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is  fur-
+nished to do so, subject to the following conditions:
+
+The  above  copyright  notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF  ANY  KIND,
+EXPRESS  OR  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE  AND  NONIN-
+FRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER  IN  AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN  THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# turboquery [![Build Status](https://travis-ci.org/playlist-media/turboquery.svg?branch=master)](https://travis-ci.org/playlist-media/turboquery) [![Code Climate](https://codeclimate.com/repos/54e54f5e69568049a90046d9/badges/2c9a4a68230e9e594789/gpa.svg)](https://codeclimate.com/repos/54e54f5e69568049a90046d9/feed)
+
+:gem: Turboquery executes PostgreSQL queries against your PostgreSQL database utilizing Redshift to massively speed up complex queries.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+task default: :test
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+end

data/lib/turboquery/connection.rb

@@ -0,0 +1,130 @@
+require 'shellwords'
+
+class Turboquery::Connection
+
+  def table_exists?(table = nil)
+    return false if table.nil?
+    connection.execute("
+       SELECT 1 AS exists
+       FROM   information_schema.tables
+       WHERE  table_schema = 'public'
+       AND    table_name = '#{table}';
+    ").count == 1
+  end
+
+  def dump_table_ddl(table)
+    set_env
+    schema = `pg_dump -i -s -x -o -O --no-tablespaces --no-security-labels -t #{Shellwords.escape(table)} #{Shellwords.escape(config[:database])}`
+    schema.empty? ? '' : schema.match(/CREATE TABLE[^;]+;/)[0]
+  end
+
+  def copy_table_to_s3(_table)
+    fail 'Not implemented'
+  end
+
+  def copy_s3_to_table(_key, _table)
+    fail 'Not implemented'
+  end
+
+  def execute(*args, &block)
+    connection.execute(*args, &block)
+  end
+
+  def config
+    connection.instance_variable_get(:@config)
+  end
+
+  def set_env
+    PsqlEnv.set(config)
+  end
+
+  def after_fork
+    self.class.after_fork
+  end
+
+  def self.after_fork
+    AR.establish_connection
+  end
+
+  protected
+
+  def connection
+    AR.connection
+  end
+
+  def random_key
+    SecureRandom.hex(10)
+  end
+
+  def valid_objects(key)
+    Turboquery.s3_bucket.objects.to_a.select do |obj|
+      obj.key =~ /^#{key}/ && !(obj.key =~ /manifest$/)
+    end
+  end
+
+  def build_manifest(objects)
+    {
+      entries: objects.map do |obj|
+        {
+          url: "s3://#{obj.bucket_name}/#{obj.key}",
+          mandatory: true
+        }
+      end
+    }.to_json
+  end
+
+  def upload_manifest(key)
+    objects = valid_objects(key)
+    file = Tempfile.open('turboquery', Turboquery.tmp_path) do |f|
+      f.write(build_manifest(objects))
+      f
+    end
+    copy_file_to_s3(file.path, "#{key}manifest")
+  end
+
+  def copy_s3_to_tmp(key)
+    objects = valid_objects(key)
+    file = Tempfile.open('turboquery', Turboquery.tmp_path) do |f|
+      objects.each do |object|
+        f.write object.get.body.read
+      end
+      f
+    end
+    file.path
+  end
+
+  def copy_file_to_s3(filename, key)
+    object = Turboquery.s3_bucket.object(key)
+    object.upload_file(filename)
+  end
+
+  class AR < ActiveRecord::Base
+    self.abstract_class = true
+  end
+
+  module PsqlEnv
+    def self.set(config)
+      host(config)     if config[:host]
+      port(config)     if config[:port]
+      password(config) if config[:password]
+      username(config) if config[:username]
+    end
+
+    def self.host(config)
+      ENV['PGHOST']     = config[:host]
+    end
+
+    def self.port(config)
+      ENV['PGPORT']     = config[:port].to_s
+    end
+
+    def self.password(config)
+      ENV['PGPASSWORD'] = config[:password].to_s
+    end
+
+    def self.username(config)
+      ENV['PGUSER']     = config[:username].to_s
+    end
+  end
+
+end

data/lib/turboquery/olap.rb

@@ -0,0 +1,55 @@
+class Turboquery::OLAP < Turboquery::Connection
+  def copy_table_to_s3(table)
+    copy_result_to_s3("SELECT * FROM #{table}")
+  end
+
+  def copy_result_to_s3(query)
+    key = random_key
+    execute("UNLOAD ('#{query}') TO 's3://#{Turboquery.s3_config['bucket']}/#{key}' #{copy_options};")
+    key
+  end
+
+  def copy_s3_to_table(key, table)
+    execute("COPY #{table} FROM 's3://#{Turboquery.s3_config['bucket']}/#{key}manifest' #{copy_options}
+    DATEFORMAT 'auto' TIMEFORMAT 'auto';")
+  end
+
+  def execute_to_temporary_table(query)
+    key = random_key
+    sql = "CREATE TEMPORARY TABLE #{key} AS #{query}"
+    execute(sql)
+    key
+  end
+
+  def drop_table(key)
+    sql = "DROP TABLE IF EXISTS #{key}"
+    execute(sql)
+  end
+
+  def self.after_fork
+    ARRedshift.reconnect
+  end
+
+  protected
+
+  def connection
+    ARRedshift.connection
+  end
+
+  def excape_single_quotes(str)
+    str.gsub(/'/, %q(\\\'))
+  end
+
+  def copy_options
+    "CREDENTIALS 'aws_access_key_id=#{Turboquery.s3_config['access_key_id']};aws_secret_access_key=#{Turboquery.s3_config['secret_access_key']}'
+     MANIFEST DELIMITER '\\t' NULL AS '\\\\N'"
+  end
+
+  class ARRedshift < ActiveRecord::Base
+    establish_connection DatabaseUrl.new(ENV['REDSHIFT_DATABASE_URL']).to_hash
+
+    def self.reconnect
+      establish_connection DatabaseUrl.new(ENV['REDSHIFT_DATABASE_URL']).to_hash
+    end
+  end
+end

data/lib/turboquery/oltp.rb

@@ -0,0 +1,24 @@
+require 'tempfile'
+
+class Turboquery::OLTP < Turboquery::Connection
+  def copy_table_to_s3(table)
+    set_env
+
+    temp = Tempfile.new('turboquery', Turboquery.tmp_path)
+    command = "echo 'COPY #{table} TO STDOUT' | psql #{Shellwords.escape(config[:database])} > #{temp.path}"
+    Kernel.system(command)
+
+    key = random_key
+    copy_file_to_s3(temp.path, key)
+    upload_manifest(key)
+    key
+  end
+
+  def copy_s3_to_table(key, table)
+    set_env
+    path = copy_s3_to_tmp(key)
+    sql = "COPY #{table} FROM STDIN"
+    command = "cat #{Shellwords.escape(path)} | psql -c #{Shellwords.escape(sql)} #{Shellwords.escape(config[:database])}"
+    Kernel.system(command)
+  end
+end

data/lib/turboquery/remote_query.rb

@@ -0,0 +1,144 @@
+class Turboquery::RemoteQuery
+
+  INSERT_INTO_REGEX = /^INSERT INTO (.+) SELECT/i
+
+  attr_reader :query, :opts, :destination
+
+  def initialize(query, opts = {})
+    @query = query.dup
+    @table_mapping = opts[:table_mapping] || {}
+    @reverse_table_mapping = {}
+    @copy_tables = opts[:copy_tables] || []
+    @opts = opts
+    format_query
+  end
+
+  def to_sql
+    query
+  end
+
+  def execute
+    # Copy any missing tables or requested tables
+    (missing_tables + @copy_tables).uniq.each { |t| copy_local_table_to_remote(t) }
+
+    # Run query and save to temporary table
+    execute_query
+
+    # Export results if no destination specified
+    return Turboquery.olap.execute("SELECT * FROM #{temp_table_key}") unless @destination
+
+    # Copy data from OLAP to OLTP
+    copy_remote_table_to_local(temp_table_key, @destination)
+
+    # Clean up temporary table
+    Turboquery.olap.execute("DROP TABLE #{temp_table_key}")
+
+    # Return success
+    true
+  end
+
+  private
+
+  def temp_table_key
+    @temp_table_key ||= "turboquery_#{SecureRandom.hex(10)}"
+  end
+
+  def insert_into
+    @insert_into ||= begin
+      match = query.match(INSERT_INTO_REGEX)
+      match.nil? ? nil : match[1]
+    end
+  end
+
+  def insert_into?
+    !insert_into.nil?
+  end
+
+  def format_query
+    @destination = insert_into || opts[:destination]
+    strip_insert_into if insert_into?
+    build_table_tokens
+    build_table_mapping
+    build_reverse_table_mapping
+    prefix_tables
+  end
+
+  def build_table_tokens
+    @table_tokens = locate_tables(PgQuery.parse(@query).parsetree[0])
+  end
+
+  def locate_tables(parsetree)
+    tables = []
+
+    return [] unless parsetree.is_a? Hash
+
+    parsetree.each do |key, val|
+      tables << { table: val, location: parsetree['location'] } if key == 'relname'
+      tables << locate_tables(val) if val.is_a? Hash
+      tables << val.map { |v| locate_tables(v) } if val.is_a? Array
+    end
+
+    tables.flatten
+  end
+
+  def build_table_mapping
+    @table_tokens.each do |table|
+      @table_mapping[table[:table]] ||= "turboquery_#{table[:table]}"
+    end
+  end
+
+  def build_reverse_table_mapping
+    @table_mapping.each do |local, remote|
+      @reverse_table_mapping[remote] = local
+    end
+  end
+
+  def prefix_tables
+    @table_tokens.reverse.each do |table|
+      @query[table[:location], table[:table].length] = @table_mapping[table[:table]]
+    end
+  end
+
+  def strip_insert_into
+    @query.gsub! INSERT_INTO_REGEX, 'SELECT'
+  end
+
+  def local_tables
+    @table_mapping.map { |k, _v| k }
+  end
+
+  def remote_tables
+    @table_mapping.map { |_k, v| v }
+  end
+
+  def missing_tables
+    remote_tables.select { |t| !Turboquery.olap.table_exists?(t) }
+  end
+
+  def copy_local_table_to_remote(table)
+    Turboquery::TableMover.new(
+      source: Turboquery.oltp,
+      destination: Turboquery.olap,
+      from_table: @reverse_table_mapping[table],
+      to_table: table
+    ).move
+  end
+
+  def copy_remote_table_to_local(remote, local)
+    Turboquery::TableMover.new(
+      source: Turboquery.olap,
+      destination: Turboquery.oltp,
+      from_table: remote,
+      to_table: local
+    ).move
+  end
+
+  def execute_query
+    if @destination
+      Turboquery.olap.execute("CREATE TABLE #{temp_table_key} AS #{@query}")
+    else
+      Turboquery.olap.execute("CREATE TEMPORARY TABLE #{temp_table_key} AS #{@query}")
+    end
+  end
+
+end

data/lib/turboquery/table_mover.rb

@@ -0,0 +1,43 @@
+class Turboquery::TableMover
+
+  attr_accessor :source, :destination, :from_table, :to_table
+
+  def initialize(source:, destination:, from_table:, to_table:)
+    self.source = source
+    self.destination = destination
+    self.from_table = from_table
+    self.to_table = to_table
+  end
+
+  def move
+    create_destination unless destination_exists?
+    key = copy_source_to_s3
+    copy_s3_to_destination(key)
+  end
+
+  private
+
+  def source_exists?
+    source.table_exists?(from_table)
+  end
+
+  def destination_exists?
+    destination.table_exists?(to_table)
+  end
+
+  def create_destination
+    structure = source.dump_table_ddl(from_table)
+    structure.gsub!(/^CREATE TABLE [^\(]+\(/, "CREATE TABLE #{to_table} (")
+    destination.execute(structure)
+    fail 'Unable to create destination table' unless destination_exists?
+  end
+
+  def copy_source_to_s3
+    source.copy_table_to_s3(from_table)
+  end
+
+  def copy_s3_to_destination(key)
+    destination.copy_s3_to_table(key, to_table)
+  end
+
+end

data/lib/turboquery/version.rb

@@ -0,0 +1,3 @@
+module Turboquery
+  VERSION = '0.1.0'
+end

data/lib/turboquery.rb

@@ -0,0 +1,42 @@
+module Turboquery
+  class << self
+    def query(sql, opts = {})
+      Turboquery::RemoteQuery.new(sql, opts).execute
+    end
+
+    def update_table(table)
+      Turboquery::TableMover.new(source: oltp, destination: olap, from_table: table, to_table: "turboquery_#{table}")
+    end
+
+    def oltp
+      @oltp ||= Turboquery::OLTP.new
+    end
+
+    def olap
+      @olap ||= Turboquery::OLAP.new
+    end
+
+    def s3_bucket
+      @s3_bucket ||= begin
+        Aws::S3::Resource.new(
+          region: 'us-east-1',
+          access_key_id: s3_config['access_key_id'],
+          secret_access_key: s3_config['secret_access_key']
+        ).bucket(s3_config['bucket'])
+      end
+    end
+
+    def s3_config
+      Rails.application.secrets.turboquery
+    end
+
+    def tmp_path
+      ENV['TURBOQUERY_TMP_PATH'] || Rails.root.join('tmp')
+    end
+
+    def after_fork
+      olap.after_fork
+      oltp.after_fork
+    end
+  end
+end

data/turboquery.gemspec

@@ -0,0 +1,24 @@
+require File.expand_path('../lib/turboquery/version', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'turboquery'
+  spec.version       = Turboquery::VERSION
+  spec.authors       = ['Jacob Gillespie']
+  spec.email         = ['jacobwgillespie@gmail.com']
+  spec.description   = 'Execute with the power of Redshift'
+  spec.summary       = 'Turboquery allows you to execute your Postgres queries on your Postgres data with Redshift.'
+  spec.homepage      = 'https://github.com/jacobwgillespie/turboquery'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split("\n")
+  spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(/^(test|spec|features)\//)
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activerecord', '~> 4.0'
+  spec.add_dependency 'pg_query'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'minitest'
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: turboquery
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jacob Gillespie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Execute with the power of Redshift
+email:
+- jacobwgillespie@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".editorconfig"
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/turboquery.rb
+- lib/turboquery/connection.rb
+- lib/turboquery/olap.rb
+- lib/turboquery/oltp.rb
+- lib/turboquery/remote_query.rb
+- lib/turboquery/table_mover.rb
+- lib/turboquery/version.rb
+- turboquery.gemspec
+homepage: https://github.com/jacobwgillespie/turboquery
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Turboquery allows you to execute your Postgres queries on your Postgres data
+  with Redshift.
+test_files: []
+has_rdoc: