jredshift 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8bd2e7cbd469a48f42d1182e83af7391620ad338
+  data.tar.gz: 411d575cae9571cd92df1d3bc015372b0cdee43e
+SHA512:
+  metadata.gz: 71e88500c05c1517ed3f6cee631daae94c28f1e7001ff4d1bf6a29308acaf2d632a1ad0246f21da0e8b33a37c8761ff5d9a4a3581deb6a0baeab2dc9f1b91a5f
+  data.tar.gz: ceeab03d691c5d93453aa0b484a0b5638a4cc50def92c3df780532896b883fad37db34501b155cf026b0f47dbebbb8fb24a6cab3fab2cb688c0a6545fd7f9f66

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in jredshift.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

data/README.md

@@ -0,0 +1,29 @@
+# Jredshift
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'jredshift'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install jredshift
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/jredshift/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/jredshift.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'jredshift/version'
+
+
+Gem::Specification.new do |spec|
+  spec.name          = 'jredshift'
+  spec.version       = Jredshift::VERSION
+  spec.authors       = ['Dean Morin']
+  spec.email         = ['morin.dean@gmail.com']
+  spec.summary       = 'Redshift wrapper for JRuby.'
+  spec.description = <<-EOS
+    Convience wrapper for using Redshift with JRuby.
+  EOS
+  spec.homepage      = 'http://github.com/deanmorin/jredshift'
+  spec.license       = 'The Unlicense'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'jdbc-postgres', '~> 9.3'
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake', '~> 0'
+end

data/lib/jredshift/array_of_hashes_result.rb

@@ -0,0 +1,85 @@
+require 'bigdecimal'
+
+
+module Jredshift
+  class ArrayOfHashesResult
+    attr_reader :result
+
+    # Defined at http://docs.oracle.com/javase/6/docs/api/java/sql/Types.html
+    COL_TYPE_BIGINT = -5
+    COL_TYPE_BIT = -7
+    COL_TYPE_CHAR = 1
+    COL_TYPE_DATE = 41
+    COL_TYPE_DECIMAL = 3
+    COL_TYPE_DOUBLE = 8
+    COL_TYPE_FLOAT = 6
+    COL_TYPE_INTEGER = 4
+    COL_TYPE_NCHAR = -15
+    COL_TYPE_NUMERIC = 2
+    COL_TYPE_NVARCHAR = -9
+    COL_TYPE_SMALLINT = 5
+    COL_TYPE_TIME = 92
+    COL_TYPE_TIMESTAMP = 93
+    COL_TYPE_TINYINT = -6
+    COL_TYPE_VARCHAR = 12
+
+
+    def initialize(java_result_set)
+      @java_result_set = java_result_set
+      @result = result_set_to_array_of_hashes
+    end
+
+    private
+
+    def result_set_to_array_of_hashes
+      meta = @java_result_set.meta_data
+      rows = []
+
+      while @java_result_set.next
+        row = {}
+
+        (1..meta.column_count).each do |i|
+          name = meta.column_name(i)
+          row[name] = get_field_by_type(meta, i)
+        end
+
+        rows << row
+      end
+      rows
+    end
+
+    def get_field_by_type(meta, i)
+      return nil if @java_result_set.get_string(i).nil?
+
+      case meta.column_type(i)
+
+      when COL_TYPE_TINYINT, COL_TYPE_SMALLINT, COL_TYPE_INTEGER
+        @java_result_set.get_int(i).to_i
+      when COL_TYPE_BIGINT
+        @java_result_set.get_long(i).to_i
+      when COL_TYPE_DATE
+        @java_result_set.get_date(i)
+      when COL_TYPE_TIME
+        @java_result_set.get_time(i).to_i
+      when COL_TYPE_TIMESTAMP
+        datetime_str = @java_result_set.get_timestamp(i).to_s
+        DateTime.strptime(datetime_str, '%Y-%m-%d %H:%M:%S')
+      when COL_TYPE_NUMERIC, COL_TYPE_DECIMAL, COL_TYPE_FLOAT, COL_TYPE_DOUBLE
+        BigDecimal.new(@java_result_set.get_string(i).to_s)
+      when COL_TYPE_CHAR, COL_TYPE_NCHAR, COL_TYPE_NVARCHAR, COL_TYPE_VARCHAR
+        @java_result_set.get_string(i).to_s
+      when COL_TYPE_BIT
+        str = @java_result_set.get_string(i)
+        if str == 't'
+          true
+        elsif str == 'f'
+          false
+        else
+          str
+        end
+      else
+        @java_result_set.get_string(i).to_s
+      end
+    end
+  end
+end

data/lib/jredshift/jdbc_db.rb

@@ -0,0 +1,192 @@
+require 'java'
+require 'jdbc/postgres'
+require 'jredshift/array_of_hashes_result'
+require 'jredshift/logger'
+
+
+module Jredshift
+  class JdbcDb
+    attr_reader :conn
+
+    # Java uses an int for the updateCount field in ResultHandler. If the
+    # update count overflows an int, you'll get this error message (but the
+    # records will be updated/inserted correctly).
+    ERROR_UPDATE_COUNT_OVERFLOW_REGEX =
+      /Unable to interpret the update count in command completion tag: /i
+
+
+    def initialize(jdbc_url, user, password, options={})
+      @abort_on_error = options.fetch(:abort_on_error, true)
+      @quiet = options.fetch(:quiet, false)
+      @jdbc_url = jdbc_url
+      @user = user
+      @password = password
+      @error_occurred = false
+      @retry_count = 0
+      init_connection
+
+      secrets = options.fetch(:secrets, [])
+      @logger = Logger.new(:secrets => secrets)
+    end
+
+    def execute(sql, options={})
+      errors_to_ignore = options.fetch(:errors_to_ignore, [])
+      quiet = options.fetch(:quiet, @quiet)
+
+      log "\n#{sql}\n" unless quiet
+
+      affected_rows = with_error_handling(:errors_to_ignore => errors_to_ignore) do
+        @statement.execute_update(sql)
+      end
+
+      unless transaction_statement?(sql) || quiet || affected_rows.nil?
+        log "Affected #{affected_rows} row(s)."
+      end
+
+      affected_rows
+    end
+
+    # Returns an empty array for no results, or nil if an error occurred and
+    # @abort_on_error = false
+    def query(sql, options={})
+      quiet = options.fetch(:quiet, @quiet)
+
+      log "\n#{sql}\n" unless quiet
+      result_set = with_error_handling { @statement.execute_query(sql) }
+      result_set ? ArrayOfHashesResult.new(result_set).result : nil
+    end
+
+    # Returns a ResultSet (instead of an Array of Hashes). Requires caller to
+    # close ResultSet and set autocommit back to true
+    def big_query(sql, options={})
+      quiet = options.fetch(:quiet, @quiet)
+      fetch_size = options.fetch(:fetch_size, 100000)
+
+      @conn.set_auto_commit(false);
+      stmt = @conn.create_statement
+      stmt.set_fetch_size(fetch_size)
+      stmt.execute_query(sql)
+
+      log "\n#{sql}\n" unless quiet
+      with_error_handling { stmt.execute_query(sql) }
+    end
+
+    def execute_script(filename, options={})
+      File.open(filename) do |fh|
+        sql = fh.read
+        sql = remove_comments(sql)
+        sql = substitute_variables(sql)
+        execute_each_statement(sql, options)
+      end
+    end
+
+    def error_occurred?
+      @error_occurred
+    end
+
+    def clear_error_state
+      @error_occurred = false
+    end
+
+    private
+
+    def init_connection
+      Jdbc::Postgres.load_driver
+      with_error_handling do
+        @conn = java.sql::DriverManager.get_connection(@jdbc_url, @user, @password)
+      end
+      @statement = @conn.create_statement
+    end
+
+    # errors_to_ignore should only be used where no return value is expected
+    def with_error_handling(options={})
+      errors_to_ignore = options.fetch(:errors_to_ignore, [])
+      return_value = yield
+      @retry_count = 0
+      return_value
+
+    rescue Exception => e
+      if errors_to_ignore.include?(e.message)
+        @retry_count = 0
+
+      elsif interpretable_error?(e.message)
+        @retry_count = 0
+        interpret_error(e.message)
+
+      elsif recoverable_error?(e.message) && @retry_count < 3
+        @retry_count += 1
+        sleep_time = sleep_time_for_error(e.message)
+        log "Failed with recoverable error: #{e.message}"
+        log "Retry attempt #{@retry_count} will occur after #{sleep_time} seconds"
+        sleep sleep_time
+        retry
+
+      else
+        @retry_count = 0
+        @error_occurred = true
+        log e.class.to_s
+        log e.message
+        log e.backtrace.join("\n") unless e.class == sql_exception_class
+
+        raise e if @abort_on_error
+      end
+    end
+
+    def interpretable_error?(err_msg)
+      err_msg =~ ERROR_UPDATE_COUNT_OVERFLOW_REGEX
+    end
+
+    def interpret_error(err_msg)
+      if err_msg =~ ERROR_UPDATE_COUNT_OVERFLOW_REGEX
+        get_update_count_from_error(err_msg)
+      else
+        fail ArgumentError, 'Unexpected interpretable_error'
+      end
+    end
+
+    def get_update_count_from_error(err_msg)
+      err_msg.split(' ').last.split('.').first
+    end
+
+    def recoverable_error?(err_msg)
+      false
+    end
+
+    def sleep_time_for_error(err_msg)
+      60
+    end
+
+    def sql_exception_class
+      fail NotImplementedError, 'sql_exception_class'
+    end
+
+    # TODO add other transaction keywords
+    def transaction_statement?(sql)
+      sql =~ /begin\s*;?/i || sql =~ /end\s*;?/i
+    end
+
+    def remove_comments(sql)
+      lines = sql.split("\n")
+      stripped_lines = lines.reject {|line| starts_with_double_dash?(line) }
+      stripped_lines.join("\n")
+    end
+
+    def starts_with_double_dash?(line)
+      line =~ /\A\s*--/
+    end
+
+    def substitute_variables(sql)
+      sql
+    end
+
+    def execute_each_statement(sql, options={})
+      sql.split(/;\s*$/).each do |statement|
+        execute("#{statement};", options)
+      end
+    end
+
+    def log(msg)
+      @logger.log(msg)
+    end
+  end
+end

data/lib/jredshift/logger.rb

@@ -0,0 +1,32 @@
+require 'singleton'
+require 'time'
+require 'copperlight/time_util'
+
+
+module Jredshift
+  class Logger
+
+    def initialize(options={})
+      @secrets = options.fetch(:secrets, [])
+      @secrets << ENV['AWS_SECRET_ACCESS_KEY'] if ENV['AWS_SECRET_ACCESS_KEY']
+    end
+
+    def log(msg)
+      puts "#{now_utc} | #{hide_secrets(msg)}"
+    end
+
+    private
+
+    def hide_secrets(msg)
+      @secrets.inject(msg) {|message, secret| hide_secret(message, secret) }
+    end
+
+    def hide_secret(msg, secret)
+      msg.sub(secret, '*' * secret.length)
+    end
+
+    def now_utc
+      DateTime.now.new_offset(0).strftime('%Y-%m-%d %H:%M:%S')
+    end
+  end
+end

data/lib/jredshift/version.rb

@@ -0,0 +1,3 @@
+module Jredshift
+  VERSION = "0.0.1"
+end

data/lib/jredshift.rb

@@ -0,0 +1,85 @@
+require 'jredshift/jdbc_db'
+
+
+module Jredshift
+  class Redshift < JdbcDb
+    attr_reader :query_group, :query_slot_count
+
+    # I've encountered a Redshift bug where the DB breaks all connections and
+    # can't be connected to again for a short time.
+    ERROR_CONNECTION_REFUSED = 'Connection refused. Check that the hostname and ' +
+      'port are correct and that the postmaster is accepting TCP/IP connections.'
+    # This is seen when the connection is broken.
+    ERROR_IO = 'An I/O error occurred while sending to the backend'
+
+
+    def initialize(jdbc_url, user, password, options={})
+      super
+      @query_group = options.fetch(:query_group, nil)
+      @query_slot_count = options.fetch(:query_slot_count, nil)
+
+      set_query_group(@query_group) if @query_group
+      set_query_slot_count(@query_slot_count) if @query_slot_count
+    end
+
+    def set_query_group(query_group)
+      execute("SET query_group TO #{query_group};")
+    end
+
+    def set_query_slot_count(count)
+      execute("SET wlm_query_slot_count TO #{count};")
+    end
+
+    def drop_table_if_exists(table, options={})
+      cascade = options.fetch(:cascade, false) ? ' CASCADE' : ''
+      err_msg = "ERROR: table \"#{remove_schema(table)}\" does not exist"
+
+      execute("DROP TABLE #{table}#{cascade};", :quiet => true, :errors_to_ignore => [err_msg])
+    end
+
+    def drop_view_if_exists(view, options={})
+      cascade = options.fetch(:cascade, false) ? ' CASCADE' : ''
+      err_msg = "ERROR: view \"#{remove_schema(view)}\" does not exist"
+
+      execute("DROP VIEW #{view}#{cascade};", :quiet => true, :errors_to_ignore => [err_msg])
+    end
+
+    def table_exists?(schema, table)
+      sql = <<-SQL
+        SELECT count(*) FROM pg_tables
+        WHERE schemaname = '#{schema}' AND tablename = '#{table}'
+        ;
+      SQL
+      query(sql, :quiet => true).first['count'] == 1
+    end
+
+    private
+
+    def recoverable_error?(err_msg)
+      err_msg =~ /S3ServiceException:speed limit exceeded/i ||
+      err_msg =~ /Communications link failure/i ||
+      err_msg =~ /VACUUM is running/i ||
+      err_msg =~ /system is in maintenance mode/i ||
+      err_msg == ERROR_CONNECTION_REFUSED ||
+      super
+    end
+
+    def sleep_time_for_error(err_msg)
+      if err_msg =~ /system is in maintenance mode/i
+        900
+      elsif err_msg == ERROR_CONNECTION_REFUSED
+        600
+      else
+        super
+      end
+    end
+
+    def sql_exception_class
+      Java::OrgPostgresqlUtil::PSQLException
+    end
+
+    def remove_schema(table)
+      table.split('.').last
+    end
+  end
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: jredshift
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Dean Morin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jdbc-postgres
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |2
+      Convience wrapper for using Redshift with JRuby.
+email:
+- morin.dean@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- jredshift.gemspec
+- lib/jredshift.rb
+- lib/jredshift/array_of_hashes_result.rb
+- lib/jredshift/jdbc_db.rb
+- lib/jredshift/logger.rb
+- lib/jredshift/version.rb
+homepage: http://github.com/deanmorin/jredshift
+licenses:
+- The Unlicense
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: Redshift wrapper for JRuby.
+test_files: []