flare-up 0.1 → 0.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjZjOGExYWMyODg1Y2FjNzZkZGIwZDg5Zjc3YzE3ODgxY2NlOWQ5NA==
+    Y2M1MDAzODUxOGM5NjQwYzQ0ZGM4ZjI2Y2Q0Mzk1NjQ3NGNmZDBhMg==
   data.tar.gz: !binary |-
-    NTIwM2FiNWI1NGVmOGVkNzgzMGJjMDcwYzg2MTY3ZTFjNWQ5ZWE5Ng==
+    ZDhmZjQwZTA4ODJjYzZkZWM5N2FiZDFiYzY3ZTU1ZTBhYjBlN2VmNw==
 SHA512:
   metadata.gz: !binary |-
-    NDI5YzQ0Y2MzYWU0MjU5MDgzNjFmNzFkMWE0Y2U3OTU0ZDkzZWFkZjk4OTAy
-    NzEwNzhhMWMzNDBhZjdjNmQzYTdmYzQ0Y2RjODk2ZGY5N2UyMjUzYjIyZjY4
-    ZThhYmVjYzJiNTgxOWMzYWUwMTRhYjQ0NzU4ODIzMThiMGNmZmM=
+    MzM2YTRkMjhkYWRkZGUzZmU2Mzg3NjgzZDk5ZmVmOWVhZTBkMDA1OWQyMjEz
+    NmVmOTRkNTExYzk5Y2FlYWUzMTU2ZGRlMzA4ODhmMWZjZWZhNmE1Mzg4YmJh
+    OTAzYjBjNGVhYzE2NjM5NjJkOTFhZWM3ZjMyMTdkOWEzZjk3MWI=
   data.tar.gz: !binary |-
-    Y2QyMDc0NTg0ZGFjOTE3MGVmYzg2ZmYzZTA3YTQzMzdmMWEyZmZiZDEwZjc2
-    MGM1NTU2NGU0ZTFmMmQ2OWVkNmNhODlkZTY4N2VkNjdkOWEwNGMwMmYwYThj
-    NjFkZTg5MjA5ZDViMDAwMjMxOTBiNjE5NTVjY2JhNTMyN2ZkZTA=
+    MDgwNjcyMDAwN2VkYTIzOGVmNWQ4YzhiZWZmNWIyMTBhMDYwZDE1NTYyOTRl
+    YTNiYmJiZTA4ODU5MzhkMmRiNmU2ZmQ2ZjJmZmZjZTQ2ODAxNmFhYjdkMTk0
+    NWExZGQ5ZDU2Y2Q4YzZmMjJkOGNiMjg4MDBhYTk4MDVjNjNmMzQ=

data/.gitignore

@@ -1 +1,3 @@
-pkg/
+pkg/
+.DS_Store
+.env

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format progress
+--require spec_helper

data/.ruby-version

@@ -1 +1 @@
-1.9.3-p545
+1.9.3-p547

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.1
+  - jruby-19mode

data/Gemfile.lock

@@ -1,14 +1,38 @@
 PATH
   remote: .
   specs:
-    flare-up (0.1)
+    flare-up (0.2)
+      pg (~> 0.17)
+      thor (~> 0.19)
 
 GEM
   remote: http://rubygems.org/
   specs:
+    diff-lcs (1.2.5)
+    pg (0.17.1)
+    rake (10.3.2)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.3)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-its (1.0.1)
+      rspec-core (>= 2.99.0.beta1)
+      rspec-expectations (>= 2.99.0.beta1)
+    rspec-mocks (3.0.3)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.3)
+    thor (0.19.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   flare-up!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rspec-its (~> 1.0)

data/README.md

@@ -0,0 +1,29 @@
+## Overview
+```Flare-up``` provides a wrapper around the Redshift [```COPY```](http://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) command for scriptability, allowing you to issue the command directly from the CLI.  Much of the code is concerned with simplifying constructing the COPY command and providing easy access to the errors that may result from import.
+
+## Why?
+
+Redshift prefers a bulk COPY operation over indidivual INSERTs which Redshift is not optimized for, and Amazon does not recommend it as a strategy for bulk loading.  This means a tool like Sqoop is out, as the number of INSERTs would be prohibitvely large given the data sets many folks import into Redshift.  Additionally, COPY is a SQL command, not something issued via the AWS Redshift REST API, meaning you need a SQL connection to your Redshift instance to bulk load data.
+
+The astute consumer of the AWS toolchain will note that [Data Pipeline](http://aws.amazon.com/datapipeline/) is one way this import may be completed however, we use Azkaban and the only thing worse one than one job flow control tool is two job flow control tools :)
+
+Additionally, access to COPY errors is a bit cumbersome.  On failure, Redshift populates the ```stl_load_errors``` table which inherently must be accessed via SQL.  Flare-up will pretty print any errors that occur during import so that you may examine your logs to versus having to establish a connection to Redshift to understand what went wrong.
+
+```
+TODO PASTE EXAMPLE OF PRETTY PRINT ERROR HERE
+```
+
+## Sample Usage - Overview
+
+At Sharethrough we're somewhat opinionated about having credentials present as environment variables for security purposes, and recommend it as a productionalized approach.  That being said, it can be a pain to export variables when you're testing a tool and as such, we support specifying all of these on the command-line.
+
+### Environment Variables
+
+These will be queried if no command-line options are specified.
+
+```
+export AWS_ACCESS_KEY_ID=
+export AWS_SECRET_ACCESS_KEY=
+export REDSHIFT_USERNAME=
+export REDSHIFT_PASSWORD=
+```

data/Rakefile

@@ -1,2 +1,12 @@
 require 'bundler'
 Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = false
+end
+
+desc 'Run specs'
+task :default => :spec

data/bin/flare-up

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'flare_up'
+
+FlareUp::CLI.start(ARGV)

data/flare-up.gemspec

@@ -0,0 +1,24 @@
+$:.push File.expand_path('../lib', __FILE__)
+require 'flare_up/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'flare-up'
+  s.version     = FlareUp::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Robert Slifka']
+  s.homepage    = 'http://www.github.com/sharethrough/flare-up'
+  s.summary     = %q{Command-line access to bulk data loading via Redshift's COPY command.}
+  s.description = %q{Flare-up makes Redshift COPY scriptable by providing CLI access to the Redshift COPY command, with handy access to pretty printed errors as well.}
+
+  s.add_dependency('pg', '~> 0.17')
+  s.add_dependency('thor', '~> 0.19')
+
+  s.add_development_dependency('rake', '~> 10.0')
+  s.add_development_dependency('rspec', '~> 3.0')
+  s.add_development_dependency('rspec-its', '~> 1.0')
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = %w(lib)
+end

data/lib/flare_up/boot.rb

@@ -0,0 +1,51 @@
+module FlareUp
+
+  class Boot
+
+    # TODO: This control flow is untested
+    def self.boot(options)
+      conn = create_connection(options)
+      copy = create_copy_command(options)
+
+      begin
+        handle_load_errors(copy.execute(conn))
+      rescue CopyCommandError => e
+        CLI.output_error("\x1b[31m#{e.message}")
+        CLI.bailout(1)
+      end
+
+    end
+
+    def self.create_connection(options)
+      FlareUp::Connection.new(
+        options[:redshift_endpoint],
+        options[:database],
+        options[:redshift_username],
+        options[:redshift_password]
+      )
+    end
+
+    def self.create_copy_command(options)
+      copy = FlareUp::CopyCommand.new(
+        options[:table],
+        options[:data_source],
+        options[:aws_access_key],
+        options[:aws_secret_key]
+      )
+      copy.columns = options[:column_list] if options[:column_list]
+      copy.options = options[:copy_options] if options[:copy_options]
+      copy
+    end
+
+    # TODO: How can we test this?
+    def self.handle_load_errors(stl_load_errors)
+      return if stl_load_errors.empty?
+      puts "\x1b[31mThere was an error processing the COPY command:"
+      stl_load_errors.each do |e|
+        puts e.pretty_print
+      end
+    end
+
+  end
+
+end

data/lib/flare_up/cli.rb

@@ -0,0 +1,60 @@
+module FlareUp
+
+  class CLI < Thor
+
+    desc 'copy DATA_SOURCE REDSHIFT_ENDPOINT DATABASE TABLE', 'COPY data into REDSHIFT_ENDPOINT from DATA_SOURCE into DATABASE.TABLE'
+    long_desc <<-LONGDESC
+`flare-up copy` executes the Redshift COPY command, loading data from\x5
+DATA_SOURCE into DATABASE_NAME.TABLE_NAME at REDSHIFT_ENDPOINT.
+
+Documentation for this version can be found at:\x5
+https://github.com/sharethrough/flare-up/blob/v#{FlareUp::VERSION}/README.md
+    LONGDESC
+    option :aws_access_key, :type => :string, :desc => "Required unless ENV['AWS_ACCESS_KEY_ID'] is set."
+    option :aws_secret_key, :type => :string, :desc => "Required unless ENV['AWS_SECRET_ACCESS_KEY'] is set."
+    option :redshift_username, :type => :string, :desc => "Required unless ENV['REDSHIFT_USERNAME'] is set."
+    option :redshift_password, :type => :string, :desc => "Required unless ENV['REDSHIFT_PASSWORD'] is set."
+    option :column_list, :type => :array, :desc => 'A space-separated list of columns, should your DATA_SOURCE require it'
+    option :copy_options, :type => :string, :desc => "Appended to the end of the COPY command; enclose \"IN QUOTES\""
+
+    def copy(data_source, endpoint, database_name, table_name)
+      boot_options = {
+        :data_source => data_source,
+        :redshift_endpoint => endpoint,
+        :database => database_name,
+        :table => table_name
+      }
+      options.each { |k, v| boot_options[k.to_sym] = v }
+
+      begin
+        CLI.env_validator(boot_options, :aws_access_key, 'AWS_ACCESS_KEY_ID')
+        CLI.env_validator(boot_options, :aws_secret_key, 'AWS_SECRET_ACCESS_KEY')
+        CLI.env_validator(boot_options, :redshift_username, 'REDSHIFT_USERNAME')
+        CLI.env_validator(boot_options, :redshift_password, 'REDSHIFT_PASSWORD')
+      rescue ArgumentError => e
+        CLI.output_error(e.message)
+        CLI.bailout(1)
+      end
+
+      Boot.boot(boot_options)
+    end
+
+    def self.env_validator(options, option_name, env_variable_name)
+      options[option_name] ||= ENVWrap.get(env_variable_name)
+      return if options[option_name]
+      raise ArgumentError, "One of either the --#{option_name} option or the ENV['#{env_variable_name}'] must be set"
+    end
+
+    # TODO: Extract
+    def self.bailout(exit_code)
+      exit(1)
+    end
+
+    # TODO: Extract
+    def self.output_error(message)
+      puts "\x1b[31m#{message}" unless ENV['TESTING']
+    end
+
+  end
+
+end

data/lib/flare_up/connection.rb

@@ -0,0 +1,74 @@
+module FlareUp
+
+  class HostUnknownOrInaccessibleError < StandardError
+  end
+  class TimeoutError < StandardError
+  end
+  class NoDatabaseError < StandardError
+  end
+  class AuthenticationError < StandardError
+  end
+  class UnknownError < StandardError
+  end
+
+  class Connection
+
+    attr_accessor :host
+    attr_accessor :port
+    attr_accessor :dbname
+    attr_accessor :user
+    attr_accessor :password
+    attr_accessor :connect_timeout
+
+    def initialize(host, dbname, user, password)
+      @host = host
+      @dbname = dbname
+      @user = user
+      @password = password
+
+      @port = 5439
+      @connect_timeout = 5
+    end
+
+    # TODO - Not quite sure how to test this; perhaps fold connect/execute into
+    # TODO   one method so we can close connections in case of failure, etc.
+    def execute(statement)
+      @pg_conn ||= connect
+      @pg_conn.exec(statement)
+    end
+
+    private
+
+    def connect
+      begin
+        PG.connect(connection_parameters)
+      rescue PG::ConnectionBad => e
+        case e.message
+          when /nodename nor servname provided, or not known/
+            raise HostUnknownOrInaccessibleError, "Host unknown or unreachable: #{@host}"
+          when /timeout expired/
+            raise TimeoutError, 'Timeout connecting to the database (have you checked your Redshift security groups?)'
+          when /database ".+" does not exist/
+            raise NoDatabaseError, "Database #{@dbname} does not exist"
+          when /password authentication failed for user/
+            raise AuthenticationError, "Either username '#{@user}' or password invalid"
+          else
+            raise UnknownError
+        end
+      end
+    end
+
+    def connection_parameters
+      {
+        :host => @host,
+        :port => @port,
+        :dbname => @dbname,
+        :user => @user,
+        :password => @password,
+        :connect_timeout => @connect_timeout
+      }
+    end
+
+  end
+
+end

data/lib/flare_up/copy_command.rb

@@ -0,0 +1,73 @@
+module FlareUp
+
+  class CopyCommandError < StandardError
+  end
+  class DataSourceError < CopyCommandError
+  end
+  class OtherZoneBucketError < CopyCommandError
+  end
+  class SyntaxError < CopyCommandError
+  end
+
+  class CopyCommand
+
+    attr_reader :table_name
+    attr_reader :data_source
+    attr_reader :aws_access_key_id
+    attr_reader :aws_secret_access_key
+    attr_reader :columns
+    attr_accessor :options
+
+    def initialize(table_name, data_source, aws_access_key_id, aws_secret_access_key)
+      @table_name = table_name
+      @data_source = data_source
+      @aws_access_key_id = aws_access_key_id
+      @aws_secret_access_key = aws_secret_access_key
+      @columns = []
+      @options = ''
+    end
+
+    def get_command
+      "COPY #{@table_name} #{get_columns} FROM '#{@data_source}' CREDENTIALS '#{get_credentials}' #{@options}"
+    end
+
+    def columns=(columns)
+      raise ArgumentError, 'Columns must be an array' unless columns.is_a?(Array)
+      @columns = columns
+    end
+
+    def execute(connection)
+      begin
+        connection.execute(get_command)
+        []
+      rescue PG::InternalError => e
+        case e.message
+          when /Check 'stl_load_errors' system table for details/
+            return STLLoadErrorFetcher.fetch_errors(connection)
+          when /The specified S3 prefix '.+' does not exist/
+            raise DataSourceError, "A data source with prefix '#{@data_source}' does not exist."
+          when /The bucket you are attempting to access must be addressed using the specified endpoint/
+            raise OtherZoneBucketError, "Your Redshift instance appears to be in a different zone than your S3 bucket.  Specify the \"REGION 'bucket-region'\" option."
+          when /PG::SyntaxError/
+            matches = /syntax error (.+) \(PG::SyntaxError\)/.match(e.message)
+            raise SyntaxError, "Syntax error in the COPY command: [#{matches[1]}]."
+          else
+            raise e
+        end
+      end
+    end
+
+    private
+
+    def get_columns
+      return '' if columns.empty?
+      "(#{@columns.join(', ').strip})"
+    end
+
+    def get_credentials
+      "aws_access_key_id=#{@aws_access_key_id};aws_secret_access_key=#{@aws_secret_access_key}"
+    end
+
+  end
+
+end

data/lib/flare_up/env_wrap.rb

@@ -0,0 +1,11 @@
+module FlareUp
+
+  class ENVWrap
+
+    def self.get(variable)
+      ENV[variable]
+    end
+
+  end
+
+end

data/lib/flare_up/stl_load_error.rb

@@ -0,0 +1,67 @@
+module FlareUp
+
+  class STLLoadError
+
+    attr_reader :err_reason
+    attr_reader :raw_field_value
+    attr_reader :raw_line
+    attr_reader :col_length
+    attr_reader :type
+    attr_reader :colname
+    attr_reader :filename
+    attr_reader :position
+    attr_reader :line_number
+
+    def initialize(err_reason, raw_field_value, raw_line, col_length, type, colname, filename, position, line_number)
+      @err_reason = err_reason
+      @raw_field_value = raw_field_value
+      @raw_line = raw_line
+      @col_length = col_length
+      @type = type
+      @colname = colname
+      @filename = filename
+      @position = position
+      @line_number = line_number
+    end
+
+    def ==(other_error)
+      return false unless @err_reason == other_error.err_reason
+      return false unless @raw_field_value == other_error.raw_field_value
+      return false unless @raw_line == other_error.raw_line
+      return false unless @col_length == other_error.col_length
+      return false unless @type == other_error.type
+      return false unless @colname == other_error.colname
+      return false unless @filename == other_error.filename
+      return false unless @position == other_error.position
+      return false unless @line_number == other_error.line_number
+      true
+    end
+
+    def pretty_print
+      output = ''
+      output += "\e[33mREASON: \e[37m#{@err_reason}\n"
+      output += "\e[33mLINE  : \e[37m#{@line_number}\n"
+      output += "\e[33mPOS   : \e[37m#{@position}\n"
+      output += "\e[33mCOLUMN: \e[37m#{@colname} (LENGTH=#{@col_length})\n" if @colname.length > 0 && @col_length > 0
+      output += "\e[33mTYPE  : \e[37m#{@type}\n" if @type.length > 0
+      output += "\e[33mLINE  : \e[37m#{@raw_line}\n"
+      output += "              \e[37m#{' ' * @position}^"
+    end
+
+    def self.from_pg_results_row(row)
+      STLLoadError.new(
+        row['err_reason'].strip,
+        row['raw_field_value'].strip,
+        row['raw_line'].strip,
+        row['col_length'].strip.to_i,
+        row['type'].strip,
+        row['colname'].strip,
+        row['filename'].strip,
+        row['position'].strip.to_i,
+        row['line_number'].strip.to_i
+      )
+    end
+
+  end
+
+end

data/lib/flare_up/stl_load_error_fetcher.rb

@@ -0,0 +1,16 @@
+module FlareUp
+
+  class STLLoadErrorFetcher
+
+    def self.fetch_errors(connection)
+      query_result = connection.execute('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT 1')
+      errors = []
+      query_result.each do |row|
+        errors << STLLoadError.from_pg_results_row(row)
+      end
+      errors
+    end
+
+  end
+
+end

data/lib/flare_up/version.rb

@@ -1,3 +1,3 @@
 module FlareUp
-  VERSION = '0.1'
+  VERSION = '0.2'
 end

data/lib/flare_up.rb

@@ -1,3 +1,16 @@
+require 'pg'
+require 'thor'
+
+require 'flare_up/env_wrap'
+
+require 'flare_up/connection'
+require 'flare_up/stl_load_error'
+require 'flare_up/stl_load_error_fetcher'
+require 'flare_up/copy_command'
+
+require 'flare_up/boot'
+require 'flare_up/cli'
+
 module FlareUp
 
 end

data/resources/README.md

@@ -0,0 +1,7 @@
+## Resources
+Used to facilitate development of ```flare-up```, these are not packaged as part of the gem, nor do they constitute a public API.  Consume at your own risk :)
+
+- ```hearthstone_cards.csv```: Small list of Hearthstone cards to COPY.
+- ```load_hearthstone_cards.rb```: Utilize the internal API to COPY from a single S3 file into a test Redshift database.  Note that the internal design is subject to, and will definitely change.
+- ```postgresql-8.4-703.jdbc4.jar```: Used by [SQLWorkbenchJ](http://www.sql-workbench.net/index.html) to connect to Redshift.  The current version of the official Postgres client does not support connecting to Redshift because it is such an old version (supports >= 8.4, Redshift is 8.0).
+- ```test_schema.sql```: Prepare our test Redshift database for ingestion.

data/resources/hearthstone_cards.csv

@@ -0,0 +1,3 @@
+Al'Akir the Windlord,8,3,5,Windfury Charge Divine Shield Taunt
+Alexstraza,9,8,8,Battlecry: Set a hero's remaining health to 15.
+Archmage Antonidas,7,5,7,Whenever you cast a spell add a 'Fireball' spell to your hand.

data/resources/load_hearthstone_cards.rb

@@ -0,0 +1,64 @@
+require 'flare_up'
+
+host_name = 'flare-up-test.cskjnp4xvaje.us-west-2.redshift.amazonaws.com'
+db_name = 'dev'
+table_name = 'hearthstone_cards'
+data_source = 's3://slif-redshift/hearthstone_cards_short_list.csv'
+
+conn = FlareUp::Connection.new(host_name, db_name, ENV['REDSHIFT_USERNAME'], ENV['REDSHIFT_PASSWORD'])
+
+copy = FlareUp::CopyCommand.new(table_name, data_source, ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'])
+copy.columns = %w(name cost attack health description)
+copy.options = "REGION 'us-east-1' CSV"
+
+result = copy.execute(conn)
+
+good_command = <<-COMMAND
+flare-up copy \
+  s3://slif-redshift/hearthstone_cards_short_list.csv \
+  flare-up-test.cskjnp4xvaje.us-west-2.redshift.amazonaws.com \
+  dev \
+  hearthstone_cards \
+  --column-list name cost attack health description \
+  --copy_options "REGION 'us-east-1' CSV"
+COMMAND
+
+bad_data_source_command = <<-COMMAND
+flare-up copy \
+  s3://slif-redshift/hearthstone_cards_short_lissdsdsdsdsdsdsd.csv \
+  flare-up-test.cskjnp4xvaje.us-west-2.redshift.amazonaws.com \
+  dev \
+  hearthstone_cards \
+  --column-list name cost attack health description \
+  --copy_options "REGION 'us-east-1' CSV"
+COMMAND
+
+bad_data_command = <<-COMMAND
+flare-up copy \
+  s3://slif-redshift/hearthstone_cards_broken.csv \
+  flare-up-test.cskjnp4xvaje.us-west-2.redshift.amazonaws.com \
+  dev \
+  hearthstone_cards \
+  --column-list name cost attack health description \
+  --copy_options "REGION 'us-east-1' CSV"
+COMMAND
+
+other_zone_bucket = <<-COMMAND
+flare-up copy \
+  s3://slif-redshift/hearthstone_cards_short_list.csv \
+  flare-up-test.cskjnp4xvaje.us-west-2.redshift.amazonaws.com \
+  dev \
+  hearthstone_cards \
+  --column-list name cost attack health description \
+  --copy_options CSV
+COMMAND
+
+busted_options = <<-COMMAND
+flare-up copy \
+  s3://slif-redshift/hearthstone_cards_short_list.csv \
+  flare-up-test.cskjnp4xvaje.us-west-2.redshift.amazonaws.com \
+  dev \
+  hearthstone_cards \
+  --column-list name cost attack health description \
+  --copy_options "CSV ;lmlkmlk3"
+COMMAND

data/resources/test_schema.sql

@@ -0,0 +1,10 @@
+DROP TABLE hearthstone_cards;
+
+CREATE TABLE hearthstone_cards
+(
+  name        VARCHAR(255) NOT NULL,
+  cost        SMALLINT NOT NULL,
+  attack      SMALLINT,
+  health      SMALLINT,
+  description VARCHAR(255) NOT NULL
+);