sneaql 0.0.13-java → 0.0.15-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1691a3f7c5849f131d0c93b224dec808c78877d1
-  data.tar.gz: 6ecfdbd3c59590059ae603b85c6694ebab7174d0
+  metadata.gz: 91f871ef54dcf89b3664fc1752bfc13cfb6ce787
+  data.tar.gz: fd38d7bf1c632db20ae61ae88ec57872a370edf9
 SHA512:
-  metadata.gz: d38ec6c6d5ffcafd8090163e47e0ae3d77691ed8411a469fe1e653b24e7d88dbda0fb6b16bcf7ca373c27b0988801355960abecbccc2c7f528165acb50dac26a
-  data.tar.gz: 8c584cfeb4e4dfd5655aaf3bc6e19bb1da042281ef9c09dab2a0b0f593d8b28abddccbb95d91068795c718f67f950a717e575aba910e1dc02a91b17b03e0ea7b
+  metadata.gz: a5d6fa48016600fde9277250a4b7bdde4f3d1e3cee7c6305af06b9b084ed00c8d2ea21739315cb68472a4b348b1b99858468d71a1bc9a08868e454c34fdb5df7
+  data.tar.gz: 9bd0b13170e52f50b961b65228a139f1c879ccd4780d3610665aecd6698741b15232fbd9023453f25c6ea121467a4126f51b878e3440a15003a5045d22cc2c5c

data/bin/sneaql

@@ -36,7 +36,6 @@ def configure_logging_format
   end
 end
 
-
 def local_gems
    Gem::Specification.sort_by{ |g| [g.name.downcase, g.version] }.group_by{ |g| g.name }
 end

data/lib/sneaql.rb

@@ -13,6 +13,7 @@ require_relative 'sneaql_lib/recordset.rb'
 require_relative 'sneaql_lib/database_manager.rb'
 require_relative 'sneaql_lib/standard_db_objects.rb'
 require_relative 'sneaql_lib/docker.rb'
+require_relative 'sneaql_lib/tokenizer.rb'
 
 # module for sneaql
 module Sneaql
@@ -77,13 +78,15 @@ module Sneaql
       @db_pass = @params[:db_pass]
       @database = @params[:database]
 
+      @expression_handler = create_expression_handler
+      @recordset_manager = create_recordset_manager
+      @exception_manager = create_exception_manager
+
       run if @params[:run] == true
     end
 
     # validate the transform.
     def validate
-      @expression_handler = create_expression_handler
-      @recordset_manager = create_recordset_manager
       @repo_manager = create_repo_manager
       @steps = create_metadata_manager
       @parsed_steps = create_parsed_steps(@steps)
@@ -116,8 +119,6 @@ module Sneaql
 
     # Runs the actual transform.
     def run
-      @expression_handler = create_expression_handler
-      @recordset_manager = create_recordset_manager
       @repo_manager = create_repo_manager
       @lock_manager = create_lock_manager if @params[:locked_transform] == true
       @steps = create_metadata_manager
@@ -159,6 +160,11 @@ module Sneaql
       Sneaql::Core::ExpressionHandler.new(@logger)
     end
 
+    # Creates ExceptionHandler object
+    def create_exception_manager
+      Sneaql::Exceptions::ExceptionManager.new(@logger)
+    end
+
     # Creates a RepoDownloadManager object
     # The actual object returns depends upon params[:repo_type] provided at initialize.
     # @return [Sneaql::Core::RepoDownloadManager]
@@ -236,9 +242,9 @@ module Sneaql
     # not rollback automatically unless that is the default RDBMS
     # behavior for a connection that closes before a commit.
     def iterate_steps_and_statements
-      @parsed_steps.each do |this_step|
-        #special handling is required for the exit_step_if command
-        #because there is a nested loop the exit_step var is needed
+      @parsed_steps.each_with_index do |this_step|
+        # special handling is required for the exit_step_if command
+        # because there is a nested loop the exit_step var is needed
         exit_step = false
         break if exit_step == true
         # set this so that other processes can poll the state
@@ -247,7 +253,9 @@ module Sneaql
         this_step[:parser].statements.each_with_index do |this_stmt, stmt_index|
           # set this so that other processes can poll the state
           @current_statement = stmt_index + 1
-
+          
+          @exception_manager.output_pending_error
+          
           # log some useful info
           @logger.info("step: #{@current_step} statement: #{@current_statement}")
           @expression_handler.output_all_session_variables
@@ -267,12 +275,24 @@ module Sneaql
             c = Sneaql::Core.find_class(:command, this_cmd[:command]).new(
               @jdbc_connection,
               @expression_handler,
+              @exception_manager,
               @recordset_manager,
               @expression_handler.evaluate_all_expressions(this_stmt),
               @logger
             )
 
             c.action(*this_cmd[:arguments])
+
+            # check if there was an error from the action
+            if @exception_manager.pending_error != nil
+              # if there was an error... check to see if this is the last stmt in step
+              if stmt_index == (this_step[:parser].statements.length - 1)
+                # last step... so we know there is no error handler in this step
+                # therefore we should propagate the error
+                raise @exception_manager.pending_error
+              end
+            end
+
           rescue Sneaql::Exceptions::SQLTestStepExitCondition => e
             exit_step = true
             @logger.info e.message

data/lib/sneaql_lib/base.rb

@@ -2,7 +2,7 @@ require 'zip/zip'
 require 'fileutils'
 require 'logger'
 
-#top level namespace for sneaql objects
+# top level namespace for sneaql objects
 module Sneaql
   # contains the base classes for the extendable parts of sneaql:
   #   commands (the actual commands specified in sneaql tags)
@@ -74,11 +74,12 @@ module Sneaql
       # @param [Sneaql::Core::RecordsetManager] recordset_manager
       # @param [String] statement SQL statement provided in body, with all variables resolved
       # @param [Logger] logger object otherwise will default to new Logger
-      def initialize(jdbc_connection, expression_handler, recordset_manager, statement, logger = nil)
+      def initialize(jdbc_connection, expression_handler, exception_manager, recordset_manager, statement, logger = nil)
         @logger = logger ? logger : Logger.new(STDOUT)
 
         @jdbc_connection = jdbc_connection
         @expression_handler = expression_handler
+        @exception_manager = exception_manager
         @statement = statement
         @recordset_manager = recordset_manager
       end
@@ -110,6 +111,8 @@ module Sneaql
             valid << valid_operator?(a)
           when arg_definition[i] == :recordset then
             valid << valid_recordset?(a)
+          when arg_definition[i] == :symbol then
+            valid << valid_symbol?(a)
           else valid << false end
         end
         @logger.debug("arg validation results: #{valid}")
@@ -143,6 +146,16 @@ module Sneaql
       def valid_recordset?(a)
         @recordset_manager.valid_recordset_name?(a.to_s.strip)
       end
+      
+      # note that this is not a ruby symbol, but a sneaql symbol
+      # should be a contigiuous string consisting of only letters, digits
+      # and underscores
+      # @param [String] a value to test
+      # @return [Boolean]
+      def valid_symbol?(a)
+        return true if a.match(/^\w+$/)
+        false
+      end
 
       private
 

data/lib/sneaql_lib/core.rb

@@ -5,7 +5,7 @@ require_relative 'exceptions.rb'
 module Sneaql
   module Core
     # Core Sneaql language command tags.
-    # You can create your own tags by extending 
+    # You can create your own tags by extending
     # Sneaql::Core::SneaqlCommand and overriding the
     # action method.  You should also override arg_definition
     # and potentially validate_args if have a complex argument
@@ -23,6 +23,8 @@ module Sneaql
         # @param [String] value expression (must be a string)
         def action(var_name, value)
           @expression_handler.set_session_variable(var_name, value)
+        rescue => e
+          @exception_manager.pending_error = e
         end
 
         # argument types
@@ -46,6 +48,8 @@ module Sneaql
             target_var_name,
             sql_result
           )
+        rescue => e
+          @exception_manager.pending_error = e
         end
 
         # argument types
@@ -78,6 +82,8 @@ module Sneaql
             'last_statement_rows_affected',
             rows_affected
           )
+        rescue => e
+          @exception_manager.pending_error = e
         end
 
         # @return [Fixnum] rows affected by SQL statement
@@ -108,6 +114,8 @@ module Sneaql
               rows_affected
             )
           end
+        rescue => e
+          @exception_manager.pending_error = e
         end
 
         # argument types
@@ -145,6 +153,8 @@ module Sneaql
           )
             raise Sneaql::Exceptions::SQLTestExitCondition
           end
+        rescue => e
+          @exception_manager.pending_error = e
         end
 
         # argument types
@@ -223,6 +233,8 @@ module Sneaql
           r = query_results
           @logger.debug "adding #{r.length} recs as #{recordset_name}"
           @recordset_manager.store_recordset(recordset_name, r)
+        rescue => e
+          @exception_manager.pending_error = e
         end
 
         # argument types
@@ -255,6 +267,8 @@ module Sneaql
           elsif ((args.size - 1) % 4) == 0
             iterate_records_conditionally(*args)
           end
+        rescue => e
+          @exception_manager.pending_error = e
         end
 
         # custom method for argument validation
@@ -295,6 +309,9 @@ module Sneaql
               rows_affected_current_statement(tmp)
             )
           end
+        rescue => e
+          @exception_manager.pending_error = e
+          raise e
         end
 
         # @param [*Array] args all the arguments passed to the calling function
@@ -312,6 +329,9 @@ module Sneaql
               rows_affected_current_statement(tmp)
             )
           end
+        rescue => e
+          @exception_manager.pending_error = e
+          raise e
         end
 
         # @return [Fixnum] rows affected by the SQL statement
@@ -339,6 +359,111 @@ module Sneaql
           r.map! { |d| { 'path_name' => d.to_s } }
           @logger.debug "adding #{r.length} recs as #{recordset_name}"
           @recordset_manager.store_recordset(recordset_name, r)
+        rescue => e
+          @exception_manager.pending_error = e
+        end
+      end
+      
+      # stores all the file paths matching the dir glob into a recordset
+      class SneaqlOnError < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'on_error',
+          Sneaql::Core::Commands::SneaqlOnError
+        )
+
+        # argument types
+        def arg_definition
+          [:symbol]
+        end
+        
+        # @param [String] recordset_name
+        # @param [String] dirglob directory glob with optional wildcards
+        def action(handler_action)
+          # only take action if there is an error
+          if @exception_manager.pending_error
+            case handler_action.downcase
+            when 'continue' then 
+              @logger.error("#{@exception_manager.pending_error.message}")
+              if @exception_manager.pending_error.backtrace
+                @exception_manager.pending_error.backtrace.each {|b| @logger.debug(b) }
+              end
+              @exception_manager.pending_error = nil
+              @exception_manager.last_iterated_record = nil
+              @logger.info("continuing after error due to on_error handling")
+              
+            when 'exit_step' then
+              @logger.error("#{@exception_manager.pending_error.message}")
+              if @exception_manager.pending_error.backtrace
+                @exception_manager.pending_error.backtrace.each {|b| @logger.debug(b) }
+              end
+              @exception_manager.pending_error = nil
+              @exception_manager.last_iterated_record = nil
+              
+              @logger.info("exiting step due to on_error handling")
+              raise Sneaql::Exceptions::SQLTestStepExitCondition
+
+            when 'execute' then
+              @logger.error("#{@exception_manager.pending_error.message}")
+              if @exception_manager.pending_error.backtrace
+                @exception_manager.pending_error.backtrace.each {|b| @logger.debug(b) }
+              end
+              @logger.info("executing sql block due to on_error handling")
+              
+              @expression_handler.set_session_variable(
+                'last_statement_rows_affected',
+                rows_affected
+              )
+              
+              @exception_manager.pending_error = nil
+              @exception_manager.last_iterated_record = nil
+            end
+          end
+        end
+
+        def rows_affected
+          tmp = @statement
+          tmp = replace_last_record(tmp)
+          tmp = replace_error_details(tmp)
+          JDBCHelpers::Execute.new(
+            @jdbc_connection,
+            tmp,
+            @logger
+          ).rows_affected
+        end
+        
+        # replaces a string with references to the field values in the last
+        # record iterated (if present). use :err_record.field_name syntax
+        # @param [String] input_string containing potential :err_record references
+        # @return [String] string with :err_record references replaced
+        def replace_last_record(input_string)
+          tmp = input_string
+          if @exception_manager.last_iterated_record != nil
+            @exception_manager.last_iterated_record.keys.sort.reverse.each do |k|
+              puts "replacing #{k}"
+              tmp = tmp.gsub(
+                ":err_record.#{k}",
+                @exception_manager.last_iterated_record[k].to_s
+              )
+            end
+          end
+          tmp
+        end
+        
+        # replaces text `:err_message` and `:err_type` with appropriate
+        # values from the pending error
+        # @param [String] input_string string containing potential err detail references
+        # @return [String] string with err details replaced
+        def replace_error_details(input_string)
+          tmp = input_string
+          tmp = tmp.gsub(
+            ':err_message',
+            @exception_manager.pending_error.message.to_s
+          )
+          tmp = tmp.gsub(
+            ':err_type',
+            @exception_manager.pending_error.class.to_s
+          )
         end
       end
     end

data/lib/sneaql_lib/database_manager.rb

@@ -7,7 +7,7 @@ module Sneaql
         return d[:text] if jdbc_url.match(d[:text])
       end
     end
-    
+
     # Manages preferences for a specific RDBMS implementation.
     class DatabaseManager
       attr_reader(
@@ -19,7 +19,7 @@ module Sneaql
         :commit_statement,
         :rollback_statement
       )
-      
+
       # @param [Hash] options values to override defaults
       def initialize(options = {})
         @has_boolean = options.fetch(:has_boolean, default_has_boolean)
@@ -30,17 +30,17 @@ module Sneaql
         @commit_statement = options.fetch(:commit_statement, default_commit_statement)
         @rollback_statement = options.fetch(:rollback_statement, default_rollback_statement)
       end
-      
+
       # @return [Boolean]
       def default_has_boolean
         false
       end
-      
+
       # @return [String]
       def default_autocommit_off_statement
         nil
       end
-      
+
       # @return [Boolean]
       def default_supports_transactions
         true
@@ -50,7 +50,7 @@ module Sneaql
       def default_supports_table_locking
         false
       end
-           
+
       # @return [String] begin statement
       def default_begin_statement
         "begin;"
@@ -60,12 +60,12 @@ module Sneaql
       def default_commit_statement
         "commit;"
       end
-      
+
       # @return [String] rollback statement
       def default_rollback_statement
         "rollback;"
       end
-      
+
       # @param [String] table_name
       # @return [String] rollback statement
       def lock_table_statement(table_name)

data/lib/sneaql_lib/database_prefs/redshift.rb

@@ -6,17 +6,16 @@ module Sneaql
         :database,
         'redshift',
         Sneaql::Core::RedshiftDatabaseManager
-      ) 
-      
-      def initialize()
+      )
+
+      def initialize
         super(
           {
             has_boolean: true,
             autocommit_off_statement: 'set autocommit=off;'
           }
-        )  
+        )
       end
-      
     end
   end
-end
+end

data/lib/sneaql_lib/database_prefs/sqlite.rb

@@ -6,7 +6,7 @@ module Sneaql
         :database,
         'sqlite',
         Sneaql::Core::SqliteDatabaseManager
-      ) 
+      )
     end
   end
-end
+end

data/lib/sneaql_lib/database_prefs/vertica.rb

@@ -6,16 +6,16 @@ module Sneaql
         :database,
         'vertica',
         Sneaql::Core::VerticaDatabaseManager
-      ) 
-      
+      )
+
       def initialize
         super(
           {
             has_boolean: true,
             autocommit_off_statement: 'set session autocommit to off;'
           }
-        )  
+        )
       end
     end
   end
-end
+end

data/lib/sneaql_lib/docker.rb

@@ -3,26 +3,26 @@ require 'json'
 module Sneaql
   module Docker
     class LocalTransformDockerfile
-      
+
       def initialize(repo_dir, repo_tag)
         @repo_dir = repo_dir
         @repo_tag = repo_tag
         @steps = JSON.parse(
           File.read("#{repo_dir}/sneaql.json")
         )
-        
+
         create_step_files
         create_dockerfile
       end
-      
-      def create_step_files()
+
+      def create_step_files
         @step_files = []
         @step_files << {
           docker_path: 'sneaql.json',
           local_path: 'sneaql.json' #File.expand_path("#{repo_dir}/sneaql.json")
         }
-        
-        @steps.each { |s| 
+
+        @steps.each { |s|
           @step_files << {
             docker_path: s['step_file'],
             local_path: s['step_file'] # File.expand_path(s['step_file'])
@@ -30,9 +30,9 @@ module Sneaql
         }
       end
 
-      def dockerfile()
+      def dockerfile
 %{FROM full360/sneaql:latest
-RUN mkdir /repo 
+RUN mkdir /repo
 #{@step_files.map {|s| "ADD #{s[:local_path]} /repo/#{s[:docker_path]}"}.join("\n")}
 }
       end
@@ -56,4 +56,4 @@ RUN mkdir /repo
       end
     end
   end
-end
+end

data/lib/sneaql_lib/exceptions.rb

@@ -1,19 +1,39 @@
 module Sneaql
   # Exceptions for SneaQL
   module Exceptions
+    class ExceptionManager
+      attr_accessor :pending_error
+      attr_accessor :last_iterated_record
+      
+      def initialize(logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+      end
+      
+      def output_pending_error
+        @logger.error "pending error: #{@pending_error}" if @pending_error
+      end
+    end
+     
     # Base error class for Sneaql
     class BaseError < StandardError; end
+
+    # Exception used to to gracefully exit test
+    class UnhandledException < BaseError
+      def initialize(msg = 'Previous error was not handled.')
+        super
+      end
+    end
     
     # Exception used to to gracefully exit test
     class SQLTestExitCondition < BaseError
-      def initialize(msg = "Exit condition met by test, this is not an error")
+      def initialize(msg = 'Exit condition met by test, this is not an error')
         super
       end
     end
-    
+
     # Exception used to gracefully exit test step
     class SQLTestStepExitCondition < BaseError
-      def initialize(msg = "Exit condition for this step has been met, this is not an error")
+      def initialize(msg = 'Exit condition for this step has been met, this is not an error')
         super
       end
     end
@@ -21,56 +41,56 @@ module Sneaql
     # Transform is locked by another process.  This is a
     # possibility when using the LockManager
     class TransformIsLocked < BaseError
-      def initialize(msg = "This transform is locked by another process")
+      def initialize(msg = 'This transform is locked by another process')
         super
       end
     end
 
     # Recordset check failure indicator
     class RecordsetContainsInconsistentOrInvalidTypes < BaseError
-      def initialize(msg = "Recordsets must have identical keys in every record")
+      def initialize(msg = 'Recordsets must have identical keys in every record')
         super
       end
     end
-    
+
     # Recordset check failure indicator
     class RecordsetIsNotAnArray < BaseError
-      def initialize(msg = "Recordset must be an array of hashes with identical keys")
+      def initialize(msg = 'Recordset must be an array of hashes with identical keys')
         super
       end
     end
 
-    # General error evaluating expression. 
+    # General error evaluating expression.
     class ExpressionEvaluationError < BaseError
-      def initialize(msg = "Error evaluating expression")
+      def initialize(msg = 'Error evaluating expression')
         super
       end
     end
-    
+
     # Comparison operator must be explicitly supported.
     class InvalidComparisonOperator < BaseError
-      def initialize(msg = "Invalid or no comparison operator provided")
+      def initialize(msg = 'Invalid or no comparison operator provided')
         super
       end
     end
-    
+
     # Error raised during parser validation process
     class StatementParsingError < BaseError
-      def initialize(msg = "General error parsing Sneaql tag and statement")
+      def initialize(msg = 'General error parsing Sneaql tag and statement')
         super
       end
     end
-    
+
     # Sneaql step files must not be empty
     class NoStatementsFoundInFile < BaseError
-      def initialize(msg = "No statements found in step file.")
+      def initialize(msg = 'No statements found in step file.')
         super
       end
     end
 
     # Sneaql command tags must be formed correctly
     class MalformedSneaqlCommandsInStep < BaseError
-      def initialize(msg = "Sneaql command tag is malformed.")
+      def initialize(msg = 'Sneaql command tag is malformed.')
         super
       end
     end