flare-up 0.2 → 0.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Y2M1MDAzODUxOGM5NjQwYzQ0ZGM4ZjI2Y2Q0Mzk1NjQ3NGNmZDBhMg==
+    NmI5NGU5NTk4NjAzNDE1ODY4YTQ1ODRjNGVhZmZjYjFkNjlhYjEzNQ==
   data.tar.gz: !binary |-
-    ZDhmZjQwZTA4ODJjYzZkZWM5N2FiZDFiYzY3ZTU1ZTBhYjBlN2VmNw==
+    YzhiOTAyMDk5YTlmODg3NmJlM2M5OWI5ZWYyOTJjMzZjMDdhZmI5ZA==
 SHA512:
   metadata.gz: !binary |-
-    MzM2YTRkMjhkYWRkZGUzZmU2Mzg3NjgzZDk5ZmVmOWVhZTBkMDA1OWQyMjEz
-    NmVmOTRkNTExYzk5Y2FlYWUzMTU2ZGRlMzA4ODhmMWZjZWZhNmE1Mzg4YmJh
-    OTAzYjBjNGVhYzE2NjM5NjJkOTFhZWM3ZjMyMTdkOWEzZjk3MWI=
+    ZDQ1MjFiZjkyMGU1MTUyYjU2ZTVlYjQyNzAyMGYwMmM0MGVhYmZkMWM3NzFj
+    NDI0NmRmOGYwNDdiYWYwZTNhODBiZmZkZDVhYmUxNzhlODlkMGNlYTAzY2Nk
+    NmQ1MzFjMWQwODY1YjliMzM3NTkxNjY1ODRjYTlhN2Y4YjllY2Q=
   data.tar.gz: !binary |-
-    MDgwNjcyMDAwN2VkYTIzOGVmNWQ4YzhiZWZmNWIyMTBhMDYwZDE1NTYyOTRl
-    YTNiYmJiZTA4ODU5MzhkMmRiNmU2ZmQ2ZjJmZmZjZTQ2ODAxNmFhYjdkMTk0
-    NWExZGQ5ZDU2Y2Q4YzZmMjJkOGNiMjg4MDBhYTk4MDVjNjNmMzQ=
+    YjRlOGVmMGFjYzdiYTUyMWU3NTlhODcwNmM2NTc1NTJhNjU3MjBmYzZhY2Jj
+    ZTAwNmU4NzgwZGJkMmFjMDE3MTc5YTllOGFkOTVjMjkyZTc1MjEyMjkyZTA2
+    MjU2YjFhOGJhZTYzMGJhNTg2ODhiY2MwZTM4NGM1YmE5YmJmYTg=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    flare-up (0.2)
+    flare-up (0.3)
       pg (~> 0.17)
       thor (~> 0.19)
 

data/README.md

@@ -3,27 +3,50 @@
 
 ## Why?
 
-Redshift prefers a bulk COPY operation over indidivual INSERTs which Redshift is not optimized for, and Amazon does not recommend it as a strategy for bulk loading.  This means a tool like Sqoop is out, as the number of INSERTs would be prohibitvely large given the data sets many folks import into Redshift.  Additionally, COPY is a SQL command, not something issued via the AWS Redshift REST API, meaning you need a SQL connection to your Redshift instance to bulk load data.
+Redshift prefers a bulk COPY operation over indidivual INSERTs which Redshift is not optimized for, and Amazon does not recommend it as a strategy for loading.  COPY is a SQL command, not something issued via the AWS Redshift REST API, meaning you need a SQL connection to your Redshift instance to bulk load data.
 
 The astute consumer of the AWS toolchain will note that [Data Pipeline](http://aws.amazon.com/datapipeline/) is one way this import may be completed however, we use Azkaban and the only thing worse one than one job flow control tool is two job flow control tools :)
 
-Additionally, access to COPY errors is a bit cumbersome.  On failure, Redshift populates the ```stl_load_errors``` table which inherently must be accessed via SQL.  Flare-up will pretty print any errors that occur during import so that you may examine your logs to versus having to establish a connection to Redshift to understand what went wrong.
+Additionally, access to COPY errors is a bit cumbersome.  On failure, Redshift populates the ```stl_load_errors``` table which inherently must be accessed via SQL.  Flare-up will pretty print any errors that occur during import so that you may examine your logs rather than establishing a connection to Redshift to understand what went wrong.
+
+## Requirements and Installation
+
+The `pg` gem is a dependency (required to issue SQL commands to Redshift) and will be pulled down with flare-up.
 
 ```
-TODO PASTE EXAMPLE OF PRETTY PRINT ERROR HERE
+> gem install flare-up
 ```
 
-## Sample Usage - Overview
+## Syntax
+
+Available via `flare-up help copy`.
 
-At Sharethrough we're somewhat opinionated about having credentials present as environment variables for security purposes, and recommend it as a productionalized approach.  That being said, it can be a pain to export variables when you're testing a tool and as such, we support specifying all of these on the command-line.
+While we'd prefer if everyone stored configuration variables (esp. credentials) as environment variables (re: [Twelve-Factor App](http://12factor.net/)), it can be a pain to export variables when you're testing a tool and as such, we support specifying all of these on the command-line.
+
+```
+Usage:
+  flare-up copy DATA_SOURCE REDSHIFT_ENDPOINT DATABASE TABLE
+
+Options:
+  [--aws-access-key=AWS_ACCESS_KEY]        # Required unless ENV['AWS_ACCESS_KEY_ID'] is set.
+  [--aws-secret-key=AWS_SECRET_KEY]        # Required unless ENV['AWS_SECRET_ACCESS_KEY'] is set.
+  [--redshift-username=REDSHIFT_USERNAME]  # Required unless ENV['REDSHIFT_USERNAME'] is set.
+  [--redshift-password=REDSHIFT_PASSWORD]  # Required unless ENV['REDSHIFT_PASSWORD'] is set.
+  [--column-list=one two three]            # A space-separated list of columns, should your DATA_SOURCE require it
+  [--copy-options=COPY_OPTIONS]            # Appended to the end of the COPY command; enclose "IN QUOTES"
+```
 
-### Environment Variables
+## Sample Usage
 
-These will be queried if no command-line options are specified.
+Note that this example assumes you have credentials set as environment variables.
 
 ```
-export AWS_ACCESS_KEY_ID=
-export AWS_SECRET_ACCESS_KEY=
-export REDSHIFT_USERNAME=
-export REDSHIFT_PASSWORD=
+> flare-up                                                      \
+    copy                                                        \
+    s3://slif-redshift/hearthstone_cards_short_list.csv         \
+    flare-up-test.cskjnp4xvaje.us-west-2.redshift.amazonaws.com \
+    dev                                                         \
+    hearthstone_cards                                           \
+    --column-list name cost attack health description           \
+    --copy_options "REGION 'us-east-1' CSV"
 ```

data/lib/flare_up/boot.rb

@@ -40,7 +40,7 @@ module FlareUp
     # TODO: How can we test this?
     def self.handle_load_errors(stl_load_errors)
       return if stl_load_errors.empty?
-      puts "\x1b[31mThere was an error processing the COPY command:"
+      puts "\x1b[31mThere was an error processing the COPY command.  Displaying the last (#{stl_load_errors.length}) errors."
       stl_load_errors.each do |e|
         puts e.pretty_print
       end

data/lib/flare_up/stl_load_error.rb

@@ -11,8 +11,9 @@ module FlareUp
     attr_reader :filename
     attr_reader :position
     attr_reader :line_number
+    attr_reader :start_time
 
-    def initialize(err_reason, raw_field_value, raw_line, col_length, type, colname, filename, position, line_number)
+    def initialize(err_reason, raw_field_value, raw_line, col_length, type, colname, filename, position, line_number, start_time)
       @err_reason = err_reason
       @raw_field_value = raw_field_value
       @raw_line = raw_line
@@ -22,6 +23,7 @@ module FlareUp
       @filename = filename
       @position = position
       @line_number = line_number
+      @start_time = start_time
     end
 
     def ==(other_error)
@@ -34,11 +36,13 @@ module FlareUp
       return false unless @filename == other_error.filename
       return false unless @position == other_error.position
       return false unless @line_number == other_error.line_number
+      return false unless @start_time == other_error.start_time
       true
     end
 
     def pretty_print
       output = ''
+      output += "\e[33mSTART : \e[37m#{@start_time} (#{@start_time - 7 * 60 * 60} PST)\n"
       output += "\e[33mREASON: \e[37m#{@err_reason}\n"
       output += "\e[33mLINE  : \e[37m#{@line_number}\n"
       output += "\e[33mPOS   : \e[37m#{@position}\n"
@@ -58,7 +62,8 @@ module FlareUp
         row['colname'].strip,
         row['filename'].strip,
         row['position'].strip.to_i,
-        row['line_number'].strip.to_i
+        row['line_number'].strip.to_i,
+        Time.parse("#{row['starttime'].strip} UTC'")
       )
     end
 

data/lib/flare_up/stl_load_error_fetcher.rb

@@ -3,7 +3,7 @@ module FlareUp
   class STLLoadErrorFetcher
 
     def self.fetch_errors(connection)
-      query_result = connection.execute('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT 1')
+      query_result = connection.execute('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT 3')
       errors = []
       query_result.each do |row|
         errors << STLLoadError.from_pg_results_row(row)

data/lib/flare_up/version.rb

@@ -1,3 +1,3 @@
 module FlareUp
-  VERSION = '0.2'
+  VERSION = '0.3'
 end

data/spec/lib/flare_up/stl_load_error_fetcher_spec.rb

@@ -6,7 +6,7 @@ describe FlareUp::STLLoadErrorFetcher do
 
     before do
       expect(connection).to receive(:execute).
-        with('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT 1').
+        with('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT 3').
         and_return([
         {
           'err_reason' => 'TEST_REASON',
@@ -17,7 +17,8 @@ describe FlareUp::STLLoadErrorFetcher do
           'colname' => 'TEST_COLNAME',
           'filename' => 'TEST_FILENAME',
           'position' => '2',
-          'line_number' => '3'
+          'line_number' => '3',
+          'starttime' => '2014-08-11 06:06:59'
         }
       ])
     end
@@ -34,7 +35,8 @@ describe FlareUp::STLLoadErrorFetcher do
             'TEST_COLNAME',
             'TEST_FILENAME',
             2,
-            3
+            3,
+            Time.parse('2014-08-11 06:06:59 UTC')
           )
         ]
       )

data/spec/lib/flare_up/stl_load_error_spec.rb

@@ -10,7 +10,8 @@ describe FlareUp::STLLoadError do
       'TEST_COLNAME',
       'TEST_FILENAME',
       2,
-      3
+      3,
+      Time.parse('2014-08-11 06:06:59 UTC')
     )
   end
 
@@ -35,7 +36,8 @@ describe FlareUp::STLLoadError do
         'colname' => 'TEST_COLNAME     ',
         'filename' => 'TEST_FILENAME    ',
         'position' => '2   ',
-        'line_number' => '3      '
+        'line_number' => '3      ',
+        'starttime' => '2014-08-11 06:06:59      '
       }
     end
 
@@ -55,6 +57,7 @@ describe FlareUp::STLLoadError do
       expect(load_error_from_hash.filename).to eq('TEST_FILENAME')
       expect(load_error_from_hash.position).to eq(2)
       expect(load_error_from_hash.line_number).to eq(3)
+      expect(load_error_from_hash.start_time).to eq(Time.parse('2014-08-11 06:06:59 UTC'))
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flare-up
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '0.3'
 platform: ruby
 authors:
 - Robert Slifka