sneaql 0.0.8-java

data/lib/sneaql_lib/exceptions.rb

@@ -0,0 +1,78 @@
+module Sneaql
+  # Exceptions for SneaQL
+  module Exceptions
+    # Base error class for Sneaql
+    class BaseError < StandardError; end
+    
+    # Exception used to to gracefully exit test
+    class SQLTestExitCondition < BaseError
+      def initialize(msg = "Exit condition met by test, this is not an error")
+        super
+      end
+    end
+    
+    # Exception used to gracefully exit test step
+    class SQLTestStepExitCondition < BaseError
+      def initialize(msg = "Exit condition for this step has been met, this is not an error")
+        super
+      end
+    end
+
+    # Transform is locked by another process.  This is a
+    # possibility when using the LockManager
+    class TransformIsLocked < BaseError
+      def initialize(msg = "This transform is locked by another process")
+        super
+      end
+    end
+
+    # Recordset check failure indicator
+    class RecordsetContainsInconsistentOrInvalidTypes < BaseError
+      def initialize(msg = "Recordsets must have identical keys in every record")
+        super
+      end
+    end
+    
+    # Recordset check failure indicator
+    class RecordsetIsNotAnArray < BaseError
+      def initialize(msg = "Recordset must be an array of hashes with identical keys")
+        super
+      end
+    end
+
+    # General error evaluating expression. 
+    class ExpressionEvaluationError < BaseError
+      def initialize(msg = "Error evaluating expression")
+        super
+      end
+    end
+    
+    # Comparison operator must be explicitly supported.
+    class InvalidComparisonOperator < BaseError
+      def initialize(msg = "Invalid or no comparison operator provided")
+        super
+      end
+    end
+    
+    # Error raised during parser validation process
+    class StatementParsingError < BaseError
+      def initialize(msg = "General error parsing Sneaql tag and statement")
+        super
+      end
+    end
+    
+    # Sneaql step files must not be empty
+    class NoStatementsFoundInFile < BaseError
+      def initialize(msg = "No statements found in step file.")
+        super
+      end
+    end
+
+    # Sneaql command tags must be formed correctly
+    class MalformedSneaqlCommandsInStep < BaseError
+      def initialize(msg = "Sneaql command tag is malformed.")
+        super
+      end
+    end
+  end
+end

data/lib/sneaql_lib/expressions.rb

@@ -0,0 +1,238 @@
+require 'logger'
+
+module Sneaql
+  module Core
+    # Handles variables, expression evaluation, and comparisons.
+    # A single ExpressionHandler is created per transform.  This
+    # object will get passed around to the various commands as well
+    # as other manager objects attached to the transform class.
+    class ExpressionHandler
+      # @param [Hash] environment_variables pass in a set of ENV
+      # @param [Logger] logger object otherwise will default to new Logger
+      def initialize(environment_variables, logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+        @environment_variables = environment_variables
+        @session_variables = {}
+      end
+
+      # @param [String] var_name identifier for variable
+      # @param [String, Fixnum, Float] var_value value to store, expressions here will not be evaluated
+      def set_session_variable(var_name, var_value)
+        @logger.info("setting session var #{var_name} to #{var_value}")
+        raise "can't set environment variable #{var_name}" unless valid_session_variable_name?(var_name)
+        @session_variables[var_name] = var_value
+      end
+
+      # validates that this would make a suitable variable name
+      # @param [String] var_name
+      # @return [Boolean]
+      def valid_session_variable_name?(var_name)
+        r = (var_name.to_s.match(/^\w+$/) && !var_name.to_s.match(/env\_\w*/) && !var_name.to_s.match(/^\d+/)) ? true : false
+        @logger.debug "validating #{var_name} as valid variable identifier indicates #{r}"
+        r
+      end
+
+      # @param [String] var_name identifier for variable
+      # @return [String, Fixnum, Float]
+      def get_session_variable(var_name)
+        @session_variables[var_name]
+      end
+
+      # convenience method, outputs all session variables to the logger
+      def output_all_session_variables
+        @logger.debug("current session variables: #{@session_variables}")
+      end
+
+      # @param [String] var_name identifier for environment variable as defined in ENV
+      # @return [String]
+      def get_environment_variable(var_name)
+        @environment_variables[var_name]
+      end
+
+      # @param [String] expression either a numeric constant, string constant in '',
+      # or reference to session or environment variable
+      # @return [String, Fixnum, Float]
+      def evaluate_expression(expression)
+        return expression unless expression.class == String
+
+        # reference to an environment variable
+        # :env_var_name or :ENV_var_name
+        # env variable references are case insensitive in this case
+        if expression =~ /\:env\_\w+/i
+          return @environment_variables[expression.gsub(/\:env\_/i, '').strip]
+
+        # reference to a variable
+        # ANSI dynamic SQL :var_name
+        # variable names are case sensitive
+        elsif expression =~ /\:\w+/
+          return @session_variables[expression.gsub(/\:/, '').strip]
+
+        # deprecated
+        elsif expression =~ /\{.*\}/
+          @logger.warn '{var_name} deprecated. use dynamic SQL syntax :var_name'
+          return @session_variables[expression.gsub(/\{|\}/, '').strip]
+
+        # string literal enclosed in single quotes
+        # only works for a single word... no whitespace allowed at this time
+        elsif expression =~ /\'.*\'/
+          return expression.delete("'").strip
+
+        # else assume it is a numeric literal
+        # need some better thinking here
+        else
+          return expression.strip
+        end
+      rescue => e
+        @logger.error("error evaluating expression: #{e.message}")
+        e.backtrace.each { |b| logger.error(b.to_s) }
+        raise Sneaql::Exceptions::ExpressionEvaluationError
+      end
+
+      # evaluates all expressions in a given SQL statement.
+      #   replaces...
+      #     environment variables in the form :env_HOSTNAME
+      #     session variables in the form :variable_name
+      #     session variables in the deprecated form {variable_name}
+      # @param [String] statement SQL statement to have all expressions evaluated
+      # @return [String] SQL statement with all variable references resolved
+      def evaluate_all_expressions(statement)
+        evaluate_session_variables(statement)
+        evaluate_environment_variables(statement)
+        evaluate_session_variables_braces(statement)
+        return statement
+      rescue => e
+        @logger.error "evaluation error #{e.message}"
+        e.backtrace.each { |b| logger.error b.to_s }
+        raise Sneaql::Exceptions::ExpressionEvaluationError
+      end
+
+      # evaluates all environment variables in a given SQL statement.
+      #   replaces...
+      #     environment variables in the form :env_HOSTNAME
+      # @param [String] statement SQL statement to have all environment variables evaluated
+      # @return [String] SQL statement with all variable references resolved
+      def evaluate_session_variables(statement)
+        # replaces :var_name in provided statement
+        @session_variables.keys.each do |k|
+          statement.gsub!(/\:#{k}/, @session_variables[k].to_s)
+        end
+      end
+
+      # evaluates all session variables in a given SQL statement.
+      #   replaces...
+      #     session variables in the form :variable_name
+      # @param [String] statement SQL statement to have all session variables evaluated
+      # @return [String] SQL statement with all variable references resolved
+      def evaluate_environment_variables(statement)
+        # replace env vars in the form :env_HOSTNAME
+        @environment_variables.keys.each do |e|
+          statement.gsub!(/\:env\_#{e}/i, @environment_variables[e])
+        end
+      end
+
+      # evaluates all session variables in a given SQL statement.
+      #   replaces...
+      #     session variables in the deprecated form {variable_name}
+      # @param [String] statement SQL statement to have all deprecated form variable references evaluated
+      # @return [String] SQL statement with all variable references resolved
+      # @deprecated
+      def evaluate_session_variables_braces(statement)
+        # deprecated
+        @session_variables.keys.each do |k|
+          statement.gsub!(/\{#{k}\}/, @session_variables[k].to_s)
+        end
+      end
+
+      # validates that this would make a suitable reference at run time.
+      # checks to see this is single quoted string, :variable_name, {var_name) or number (1, 1.031, etc.)
+      # @param [String] expr value to check
+      def valid_expression_reference?(expr)
+        return expr.to_s.match(/(^\'.+\'$|^\:\w+$|^\{\w+\}$|^\d+$|^\d+\.\d*$)/) ? true : false
+      end
+
+      # Operators valid for expression comparison
+      # @return [Array<String>]
+      def valid_operators
+        ['=', '!=', '>', '<', '>=', '<=', 'like', 'notlike']
+      end
+
+      # provides a standardized method of comparing two expressions.
+      # note that this only works for variables and constants.
+      # current version supports float, integer, and contigious strings.
+      # @param [String] operator comparison operator @see valid_operators
+      # @param [String] exp1 expression for left operand
+      # @param [String] exp2 expression for right operand
+      def compare_expressions(operator, exp1, exp2)
+        unless valid_operators.include?(operator)
+          raise Sneaql::Exceptions::InvalidComparisonOperator
+        end
+
+        @logger.debug "evaluating #{exp1} #{operator} #{exp2}"
+
+        # evaluate exps and coerce data types
+        coerced = coerce_data_types(
+          evaluate_expression(exp1),
+          evaluate_expression(exp2)
+        )
+
+        compare_values(operator, coerced[0], coerced[1])
+      end
+
+      # coerces the data types for both expressions to match for valid comparison
+      # @param [String, Float, Fixnum] exp1 expression for left operand
+      # @param [String, Float, Fixnum] exp2 expression for right operand
+      # @return [Array<Float, Fixnum, String>] returns array with both input expressions coerced to the same data type
+      def coerce_data_types(exp1, exp2)
+        # coerce data types to make for a good comparison
+        if exp1.class == exp2.class
+          nil # nothing to do... continue with comparison
+        elsif [exp1.class, exp2.class].include? Float
+          # if either is a float then make sure they are both floats
+          exp1 = exp1.to_f
+          exp2 = exp2.to_f
+        elsif [exp1.class, exp2.class].include? Fixnum
+          # otherwise... if one is an integer make them both integers
+          exp1 = exp1.to_i
+          exp2 = exp2.to_i
+        end
+        [exp1, exp2]
+      end
+
+      # performs the actual comparison between two values
+      # @param [String] operator comparison operator @see valid_operators
+      # @param [String] exp1 expression for left operand
+      # @param [String] exp2 expression for right operand
+      # @return [Boolean]
+      def compare_values(operator, exp1, exp2)
+        # below are all the valid comparison operators
+        @logger.debug("comparing #{exp1} #{operator} #{exp2}")
+        case operator
+        when '=' then return exp1 == exp2
+        when '!=' then return exp1 != exp2
+        when '>=' then return exp1 >= exp2
+        when '<=' then return exp1 <= exp2
+        when '>' then return exp1 > exp2
+        when '<' then return exp1 < exp2
+        when 'like' then return like_operator(exp1, exp2)
+        when 'notlike' then return !like_operator(exp1, exp2)
+        end
+      end
+
+      # performs SQL style LIKE comparison between inputs
+      # @param [String] left_operand
+      # @param [String] like_right_operand this will be the like expression
+      # @return [Boolean]
+      def like_operator(left_operand, like_right_operand)
+        #converts to string before comparison
+        return left_operand.to_s.match(wildcard_to_regex(like_right_operand.to_s)) ? true : false
+      end
+
+      # converts a SQL LIKE wildcard expression to a Regexp
+      # @param [String] wildcard like expression
+      # @return [Regexp] returns regexp object for use in match comparison
+      def wildcard_to_regex(wildcard)
+        Regexp.new("^#{wildcard}$".gsub('%','.*').gsub('_','.'))
+      end
+    end
+  end
+end

data/lib/sneaql_lib/lock_manager.rb

@@ -0,0 +1,176 @@
+require 'jdbc_helpers'
+
+module Sneaql
+  module Core
+    # manages transform locking operations using a standardized
+    # table for storing the locks.
+    class TransformLockManager
+      # set instance variables that will be used to manage the locks
+      def initialize(params, logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+        @transform_name = params[:transform_name]
+        @transform_lock_id = params[:transform_lock_id]
+        @transform_lock_table = params[:transform_lock_table]
+        @jdbc_url = params[:jdbc_url]
+        @db_user = params[:db_user]
+        @db_pass = params[:db_pass]
+        @database_manager = Sneaql::Core.find_class(
+          :database,
+          params[:database]
+        ).new
+      rescue => e
+        @logger.error e.message
+        e.backtrace.each { |b| @logger.error b}
+      end
+
+      # Creates a connection in the current JDBC context
+      def create_jdbc_connection
+        JDBCHelpers::ConnectionFactory.new(
+          @jdbc_url,
+          @db_user,
+          @db_pass,
+          @logger
+        ).connection
+      end
+
+      # Checks to see if the current transform is locked
+      # @return [Boolean]
+      def acquire_lock
+        # check to see if this transform is locked by
+        # another transform returns true if locked
+        jdbc_connection = create_jdbc_connection
+
+        # initialize lock value
+        lock_value = false
+
+        if @database_manager.supports_transactions == true
+          l = JDBCHelpers::Execute.new(
+            jdbc_connection,
+            @database_manager.begin_statement,
+            @logger
+          )
+        end
+
+        if @database_manager.supports_table_locking == true
+          l = JDBCHelpers::Execute.new(
+            jdbc_connection,
+            @database_manager.lock_table_statement(@transform_lock_table),
+            @logger
+          )
+        end
+
+        # query the number of rows which match the condition...
+        # should be 1 or 0... 1 indicating a lock
+        r = JDBCHelpers::SingleValueFromQuery.new(
+          jdbc_connection,
+          %(select
+            count(*)
+          from
+            #{@transform_lock_table}
+          where
+            transform_name='#{@transform_name}'
+            and
+            transform_lock_id!=#{@transform_lock_id};),
+            @logger
+        ).result
+
+        # table is unlocked
+        if r == 0
+          l = JDBCHelpers::Execute.new(
+            jdbc_connection,
+            %{insert into #{@transform_lock_table}
+              (
+                transform_lock_id,
+                transform_name,
+                transform_lock_time
+              )
+              values
+              (
+                #{@transform_lock_id},
+                '#{@transform_name}',
+                current_timestamp
+              );},
+              @logger
+          )
+
+          if @database_manager.supports_transactions == true
+            l = JDBCHelpers::Execute.new(
+              jdbc_connection,
+              @database_manager.commit_statement,
+              @logger
+            )
+          end
+
+          lock_value = true
+        else
+          if @database_manager.supports_transactions == true
+            l = JDBCHelpers::Execute.new(
+              jdbc_connection,
+              @database_manager.rollback_statement,
+              @logger
+            )
+          end
+          lock_value = false
+        end
+
+        if lock_value == true
+          @logger.info("#{@transform_name} transform lock acquired;")
+        else
+          @logger.info("#{@transform_name} is locked by another process")
+        end
+      ensure
+        # close this connection
+        jdbc_connection.close
+
+        lock_value
+      end
+
+      # Removes transform lock if it's present.
+      def remove_lock
+        # get a fresh jdbc connection...
+        # to avoid committing the main transform unnecessarily
+        jdbc_connection = create_jdbc_connection
+
+        if @database_manager.supports_transactions == true
+          l = JDBCHelpers::Execute.new(
+            jdbc_connection,
+            @database_manager.begin_statement,
+            @logger
+          )
+        end
+
+        if @database_manager.supports_table_locking == true
+          l = JDBCHelpers::Execute.new(
+            jdbc_connection,
+            @database_manager.lock_table_statement(@transform_lock_table),
+            @logger
+          )
+        end
+
+        # delete the lock record and commit
+        JDBCHelpers::Execute.new(
+          jdbc_connection,
+          %(delete from #{@transform_lock_table}
+          where transform_name='#{@transform_name}'
+          and transform_lock_id=#{@transform_lock_id};),
+          @logger
+        )
+
+        c = JDBCHelpers::Execute.new(
+          jdbc_connection,
+          @database_manager.commit_statement,
+          @logger
+        )
+      ensure
+        jdbc_connection.close
+        
+        return true
+      end
+
+      # TBD
+      def lock_all_available_transforms
+        # undefined at this time
+      end
+    end
+  end
+end

data/lib/sneaql_lib/parser.rb

@@ -0,0 +1,89 @@
+module Sneaql
+  module Core
+    # Parses a step file into discrete statements.
+    # Also performs validation of all Sneaql tags.
+    class StepParser
+      # array of raw statement text
+      attr_reader :statements
+      attr_reader :expression_handler
+
+      # @param [String] file_path pathname to step file
+      # @param [Sneaql::ExpressionHandler] expression_handler
+      # @param [Sneaql::RecordsetManager] recordset_manager
+      # @param [Logger] logger optional logger, if omitted default logger will be used
+      def initialize(file_path, expression_handler, recordset_manager, logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+        @expression_handler = expression_handler
+        @recordset_manager = recordset_manager
+
+        # parse the statements from the file and store them in an array
+        # this is a simple text parsing based upon the /*- delimiter
+        @statements = parse_statements_from_file(file_path)
+
+        raise Sneaql::Exceptions::NoStatementsFoundInFile if @statements == []
+      end
+
+      # Performs the actual parsing from file
+      # @param [String] file_path
+      def parse_statements_from_file(file_path)
+        @logger.info("parsing statements from step file #{file_path}")
+        stmt = []
+        File.read(file_path).split('/*-').each { |s| stmt << "/*-#{s.strip}" }
+        # delete the first element because of the way it splits
+        stmt.delete_at(0)
+        @logger.info("#{stmt.length} statements found")
+        stmt
+      rescue => e
+        @logger.error("file parsing error :#{e.message}")
+        e.backtrace.each { |b| @logger.error b.to_s }
+        raise Sneaql::Exceptions::StatementParsingError
+      end
+
+      # Extracts tag and splits into an array
+      # @param [String] statement_text_with_command
+      # @return [Array]
+      def tag_splitter(statement_text_with_command)
+        # splits out all the tag elements into an array
+        statement_text_with_command.split('-*/')[0].gsub('/*-', '').strip.split
+      end
+
+      # Returns command tag from statement at specified index.  Allows for
+      # @param [Fixnum] indx index of statement in statements array
+      # @return [Hash]
+      def command_at_index(indx)
+        parsed_tag = tag_splitter(@statements[indx])
+        { command: parsed_tag[0], arguments: parsed_tag[1..parsed_tag.length - 1] }
+      end
+
+      # Validates the Sneaql command tag and arguments
+      # @return [Boolean]
+      def valid_arguments_in_all_statements?
+        all_statements_valid = true
+        @statements.each_with_index do |_s, i|
+          cmd = command_at_index(i)
+          @logger.debug("validating #{cmd}")
+          unless statement_args_are_valid?(cmd)
+            all_statements_valid = false
+            @logger.info "argument validation error: #{cmd}"
+          end
+        end
+        return all_statements_valid
+      end
+
+      # Checks to see if the arguments for a given command are valid.
+      # This is done by calling the validate_args method of the command class.
+      # @param [Hash] this_cmd parsed command tag
+      # @return [Boolean]
+      def statement_args_are_valid?(this_cmd)
+        c = Sneaql::Core.find_class(:command, this_cmd[:command]).new(
+          nil,
+          @expression_handler,
+          @recordset_manager,
+          nil,
+          @logger
+        )
+        c.validate_args(this_cmd[:arguments])
+      end
+    end
+  end
+end

data/lib/sneaql_lib/recordset.rb

@@ -0,0 +1,97 @@
+require 'logger'
+
+module Sneaql
+  module Core
+    #manages stored recordsets in sneaql transforms
+    class RecordsetManager
+      attr_reader :recordset
+
+      def initialize(expression_manager, logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+        @expression_manager = expression_manager
+        @recordset = {}
+      end
+
+      # Stores a recordset if it is in a valid format.
+      # @param [String] name name for recordset
+      # @param [Array<Hash>] rs recordset to store
+      def store_recordset(name, rs)
+        raise Sneaql::RecordsetIsNotAnArray unless rs.class == Array
+        raise Sneaql::RecordsetContainsInconsistentOrInvalidTypes unless recordset_valid?(rs)
+        recordset[name] = rs
+      end
+
+      # Validates recordset.  Must be an array of hashes with identical keys.
+      # @param [Array<Hash>] rs recordset to validate
+      def recordset_valid?(rs)
+        return false unless rs.class == Array
+        r1 = rs[0].keys
+
+        rs.each do |record|
+          return false unless record.class == Hash
+          return false unless r1 == record.keys
+          record.keys {|k| puts k; return false unless valid_element_data_types.include?(record[k].class)}
+        end
+        true
+      end
+
+      # Ruby data types that are valid as recordset fields.
+      # @return [Array<Class>]
+      def valid_element_data_types
+        [Fixnum, String, Float]
+      end
+
+      # Validates that the string will make a valid recordset name.
+      # @param [String] name
+      # @return [Boolean]
+      def valid_recordset_name?(name)
+        return false unless name.match(/^\w+/)
+        h = {}
+        h[name] == 1
+      rescue
+        return false
+      else
+        return true
+      end
+
+      # Validates that the recordset name doesn't conflict with session var names
+      # @param [String] name
+      # @return [Boolean]
+      def recordset_name_conflicts_with_variables?(name)
+        @expression_manager.session_variables.key?(name)
+      end
+
+      # Parses a recordset expression.
+      # @param [Array<Hash>] args
+      # @return [Array<Hash>]
+      def parse_recordset_expression(args)
+        # takes in argument array as an argument
+        # returns array of expressions to be checked at run time
+        args.delete_at(0) # get rid of the first element, recordset ref
+        args.each_slice(4).to_a.map{ |x| { condition: x[0].downcase, field: x[1], operator: x[2], expression: x[3]}}
+      end
+
+      # applies a conditional expression set against a record.
+      # @param [Hash] record
+      # @param [Array<Hash>] expressions
+      # @return [Boolean]
+      def evaluate_expression_against_record(record, expressions)
+        conditions = []
+        expressions.each do |exp|
+          @logger.debug("applying #{exp} to #{record}")
+          raw_result = @expression_manager.compare_expressions(
+            exp[:operator],
+            record[exp[:field]],
+            exp[:expression]
+          )
+          if exp[:condition] == 'include'
+            conditions << raw_result
+          elsif exp[:condition] == 'exclude'
+            conditions << !raw_result
+          end
+        end
+        return !conditions.include?(false)
+      end
+    end
+  end
+end