sneaql 0.0.13-java → 0.0.15-java

data/lib/sneaql_lib/expressions.rb

@@ -1,4 +1,5 @@
 require 'logger'
+require 'time'
 
 module Sneaql
   module Core
@@ -21,7 +22,7 @@ module Sneaql
       # @param [String, Fixnum, Float] var_value value to store, expressions here will not be evaluated
       def set_session_variable(var_name, var_value)
         @logger.info("setting session var #{var_name} to #{var_value}")
-        raise "can't set environment variable #{var_name}" unless valid_session_variable_name?(var_name)
+        raise "can't set variable #{var_name}" unless valid_session_variable_name?(var_name)
         @session_variables[var_name] = var_value
       end
 
@@ -29,8 +30,8 @@ module Sneaql
       # @param [String] var_name
       # @return [Boolean]
       def valid_session_variable_name?(var_name)
-        r = (var_name.to_s.match(/^\w+$/) && !var_name.to_s.match(/env\_\w*/) && !var_name.to_s.match(/^\d+/)) ? true : false
-        @logger.debug "validating #{var_name} as valid variable identifier indicates #{r}"
+        r = var_name.to_s.match(/^\w+$/) && !var_name.to_s.match(/env\_\w*/) && !var_name.to_s.match(/^\d+/) ? true : false
+        @logger.debug("validating #{var_name} as valid variable identifier indicates #{r}")
         r
       end
 
@@ -57,21 +58,25 @@ module Sneaql
       def evaluate_expression(expression)
         return expression unless expression.class == String
 
+        # first handle empty string
+        if expression == "''"
+          return ''
+
         # reference to an environment variable
         # :env_var_name or :ENV_var_name
         # env variable references are case insensitive in this case
-        if expression =~ /\:env\_\w+/i
+        elsif expression =~ /^\:env\_\w+/i
           return @environment_variables[expression.gsub(/\:env\_/i, '').strip]
 
         # reference to a variable
         # ANSI dynamic SQL :var_name
         # variable names are case sensitive
-        elsif expression =~ /\:\w+/
+        elsif expression =~ /^\:\w+/
           return @session_variables[expression.gsub(/\:/, '').strip]
 
         # deprecated
-        elsif expression =~ /\{.*\}/
-          @logger.warn '{var_name} deprecated. use dynamic SQL syntax :var_name'
+        elsif expression =~ /^\{.*\}$/
+          @logger.warn('{var_name} deprecated. use dynamic SQL syntax :var_name')
           return @session_variables[expression.gsub(/\{|\}/, '').strip]
 
         # boolean
@@ -81,9 +86,9 @@ module Sneaql
           return false if expression.downcase == 'false'
 
         # string literal enclosed in single quotes
-        # only works for a single word... no whitespace allowed at this time
-        elsif expression =~ /\'.*\'/
-          return expression.delete("'").strip
+
+        elsif expression.match(/^\s*\'.*\'\s*$/)
+          return expression.strip[1..expression.strip.length - 2]
 
         # else assume it is a numeric literal
         # need some better thinking here
@@ -92,7 +97,7 @@ module Sneaql
         end
       rescue => e
         @logger.error("error evaluating expression: #{e.message}")
-        e.backtrace.each { |b| logger.error(b.to_s) }
+        e.backtrace.each { |b| @logger.error(b.to_s) }
         raise Sneaql::Exceptions::ExpressionEvaluationError
       end
 
@@ -117,11 +122,13 @@ module Sneaql
       # evaluates all environment variables in a given SQL statement.
       #   replaces...
       #     environment variables in the form :env_HOSTNAME
+      # variables are sorted and reversed before performing substitutions
+      # this prevents :var from partially overwriting :var_foo
       # @param [String] statement SQL statement to have all environment variables evaluated
       # @return [String] SQL statement with all variable references resolved
       def evaluate_session_variables(statement)
         # replaces :var_name in provided statement
-        @session_variables.keys.each do |k|
+        @session_variables.keys.sort.reverse.each do |k|
           statement.gsub!(/\:#{k}/, @session_variables[k].to_s)
         end
       end
@@ -129,11 +136,13 @@ module Sneaql
       # evaluates all session variables in a given SQL statement.
       #   replaces...
       #     session variables in the form :variable_name
+      # variables are sorted and reversed before performing substitutions
+      # this prevents :var from partially overwriting :var_foo
       # @param [String] statement SQL statement to have all session variables evaluated
       # @return [String] SQL statement with all variable references resolved
       def evaluate_environment_variables(statement)
         # replace env vars in the form :env_HOSTNAME
-        @environment_variables.keys.each do |e|
+        @environment_variables.keys.sort.reverse.each do |e|
           statement.gsub!(/\:env\_#{e}/i, @environment_variables[e])
         end
       end
@@ -155,7 +164,7 @@ module Sneaql
       # checks to see this is single quoted string, :variable_name, {var_name) or number (1, 1.031, etc.)
       # @param [String] expr value to check
       def valid_expression_reference?(expr)
-        return expr.to_s.match(/(^\'.+\'$|^\:\w+$|^\{\w+\}$|^\d+$|^\d+\.\d*$|true|false)/i) ? true : false
+        expr.to_s.match(/(^\s*\'.+\'\s*$|^\:\w+$|^\{\w+\}$|^\d+$|^\d+\.\d*$|true|false)/i) ? true : false
       end
 
       # Operators valid for expression comparison
@@ -183,6 +192,7 @@ module Sneaql
           exp1,
           exp2
         )
+
         compare_values(operator, coerced[0], coerced[1])
       end
 
@@ -195,16 +205,31 @@ module Sneaql
         @logger.debug("coercing types #{[exp1.class, exp2.class]}")
         if exp1.class == exp2.class
           nil # nothing to do... continue with comparison
-        elsif ([exp1.class, exp2.class].include?(FalseClass)) or ([exp1.class, exp2.class].include?(TrueClass))
+        elsif array_has_boolean_value?([exp1, exp2])
+          # if one of the values is an actual Boolean object...
           unless [coerce_boolean(exp1), coerce_boolean(exp2)].include?(nil)
             exp1 = coerce_boolean(exp1)
             exp2 = coerce_boolean(exp2)
           end
-        elsif [exp1.class, exp2.class].include? Float
+        elsif exp1.class == Time
+          tmp = coerce_datetime(exp2)
+          if tmp
+            exp2 = tmp
+          else
+            exp1, exp2 = exp1.to_s, exp2.to_s
+          end
+        elsif exp2.class == Time
+          tmp = coerce_datetime(exp1)
+          if tmp
+            exp1 = tmp
+          else
+            exp1, exp2 = exp1.to_s, exp2.to_s
+          end
+        elsif [exp1.class, exp2.class].include?(Float)
           # if either is a float then make sure they are both floats
           exp1 = exp1.to_f
           exp2 = exp2.to_f
-        elsif [exp1.class, exp2.class].include? Fixnum
+        elsif [exp1.class, exp2.class].include?(Fixnum)
           # otherwise... if one is an integer make them both integers
           exp1 = exp1.to_i
           exp2 = exp2.to_i
@@ -213,26 +238,52 @@ module Sneaql
         [exp1, exp2]
       end
 
+      def coerce_datetime(obj)
+        retobj = nil
+        case
+          when obj.class == Fixnum then retobj = Time.at(obj)
+          when obj.class == String then retobj = Time.parse(obj)
+          when obj.class == Time then retobj = obj
+        end
+        retobj
+      end
+
+      # checks to see if any element of the array has a boolean value
+      # @param [Array<Object>] arr array of objects
+      # @return [Boolean]
+      def array_has_boolean_value?(arr)
+        tmp = arr.map { |e| e.class }
+        # [exp1.class, exp2.class].include?(FalseClass) || [exp1.class, exp2.class].include?(TrueClass)
+        tmp.include?(FalseClass) || tmp.include?(TrueClass)
+      end
+
       # evaluates string or fixnum values to coerce into boolean
       # @param [Object] value
       # @return [Object]
       def coerce_boolean(value)
         retval = nil
-        if [TrueClass, FalseClass].include?(value.class)
+        if array_has_boolean_value?([value])
           retval = value
         elsif value.class == Fixnum
           retval = true if value == 1
-          retval = false if value == 0
+          retval = false if value.zero?
         elsif value.class == String
-          tmp = value.downcase.strip
-          case tmp
-          when 'f', 'false', '0'
-            retval = false
-          when 't', 'true', '1'
-            retval = true
-          end
+          retval = text_to_boolean(value)
+        end
+        retval
+      end
+
+      # converts text representation of boolean to boolean
+      # @param [String] value
+      # @return [Boolean]
+      def text_to_boolean(value)
+        case value.downcase.strip
+        when 'f', 'false', '0'
+          retval = false
+        when 't', 'true', '1'
+          retval = true
         end
-        return retval
+        retval
       end
 
       # performs the actual comparison between two values
@@ -260,15 +311,15 @@ module Sneaql
       # @param [String] like_right_operand this will be the like expression
       # @return [Boolean]
       def like_operator(left_operand, like_right_operand)
-        #converts to string before comparison
-        return left_operand.to_s.match(wildcard_to_regex(like_right_operand.to_s)) ? true : false
+        # converts to string before comparison
+        left_operand.to_s.match(wildcard_to_regex(like_right_operand.to_s)) ? true : false
       end
 
       # converts a SQL LIKE wildcard expression to a Regexp
       # @param [String] wildcard like expression
       # @return [Regexp] returns regexp object for use in match comparison
       def wildcard_to_regex(wildcard)
-        Regexp.new("^#{wildcard}$".gsub('%','.*').gsub('_','.'))
+        Regexp.new("^#{wildcard}$".gsub('%', '.*').gsub('_', '.'))
       end
 
       # create a hash built from supplied environment variables.
@@ -278,14 +329,14 @@ module Sneaql
       def filtered_environment_variables
         env_vars = {}
         if ENV['SNEAQL_AVAILABLE_ENV_VARS']
-          @logger.debug("filtering environment variables")
+          @logger.debug('filtering environment variables')
           available = ENV['SNEAQL_AVAILABLE_ENV_VARS'].split(',')
           ENV.keys.each { |k| env_vars[k] = ENV[k] if available.include?(k) }
         else
-          @logger.debug("setting environment variables")
+          @logger.debug('setting environment variables')
           ENV.keys.each { |k| env_vars[k] = ENV[k] }
         end
-        return env_vars
+        env_vars
       end
 
       # basic regex filtering to help combat sql injection
@@ -294,7 +345,7 @@ module Sneaql
       # @return [Boolean] returns true if value is safe
       def sql_injection_filter(value)
         return false if value.to_s.match(/(\'|\;|(drop|alter).*(table|user|view|column|database|schema|function|sequence|procedure))/i)
-        return true
+        true
       end
 
       # insures that all environment variables pass SQL injection test

data/lib/sneaql_lib/lock_manager.rb

@@ -44,7 +44,7 @@ module Sneaql
         lock_value = false
 
         if @database_manager.supports_transactions == true
-          l = JDBCHelpers::Execute.new(
+          JDBCHelpers::Execute.new(
             jdbc_connection,
             @database_manager.begin_statement,
             @logger
@@ -52,7 +52,7 @@ module Sneaql
         end
 
         if @database_manager.supports_table_locking == true
-          l = JDBCHelpers::Execute.new(
+          JDBCHelpers::Execute.new(
             jdbc_connection,
             @database_manager.lock_table_statement(@transform_lock_table),
             @logger
@@ -94,7 +94,7 @@ module Sneaql
           )
 
           if @database_manager.supports_transactions == true
-            l = JDBCHelpers::Execute.new(
+            JDBCHelpers::Execute.new(
               jdbc_connection,
               @database_manager.commit_statement,
               @logger
@@ -104,7 +104,7 @@ module Sneaql
           lock_value = true
         else
           if @database_manager.supports_transactions == true
-            l = JDBCHelpers::Execute.new(
+            JDBCHelpers::Execute.new(
               jdbc_connection,
               @database_manager.rollback_statement,
               @logger
@@ -132,7 +132,7 @@ module Sneaql
         jdbc_connection = create_jdbc_connection
 
         if @database_manager.supports_transactions == true
-          l = JDBCHelpers::Execute.new(
+          JDBCHelpers::Execute.new(
             jdbc_connection,
             @database_manager.begin_statement,
             @logger
@@ -140,7 +140,7 @@ module Sneaql
         end
 
         if @database_manager.supports_table_locking == true
-          l = JDBCHelpers::Execute.new(
+          JDBCHelpers::Execute.new(
             jdbc_connection,
             @database_manager.lock_table_statement(@transform_lock_table),
             @logger
@@ -163,7 +163,7 @@ module Sneaql
         )
       ensure
         jdbc_connection.close
-        
+
         return true
       end
 

data/lib/sneaql_lib/parser.rb

@@ -1,3 +1,5 @@
+require_relative 'tokenizer.rb'
+
 module Sneaql
   module Core
     # Parses a step file into discrete statements.
@@ -10,7 +12,7 @@ module Sneaql
       # @param [String] file_path pathname to step file
       # @param [Sneaql::ExpressionHandler] expression_handler
       # @param [Sneaql::RecordsetManager] recordset_manager
-      # @param [Logger] logger optional logger, if omitted default logger will be used
+      # @param [Logger] logger optional, if omitted default logger will be used
       def initialize(file_path, expression_handler, recordset_manager, logger = nil)
         @logger = logger ? logger : Logger.new(STDOUT)
         @expression_handler = expression_handler
@@ -39,12 +41,16 @@ module Sneaql
         raise Sneaql::Exceptions::StatementParsingError
       end
 
-      # Extracts tag and splits into an array
+      # Extracts array of tokens from tag
       # @param [String] statement_text_with_command
       # @return [Array]
       def tag_splitter(statement_text_with_command)
+        # updated to use tokenizer
         # splits out all the tag elements into an array
-        statement_text_with_command.split('-*/')[0].gsub('/*-', '').strip.split
+        # statement_text_with_command.split('-*/')[0].gsub('/*-', '').strip.split
+        command = statement_text_with_command.split('-*/')[0].gsub('/*-', '').strip
+        t = Sneaql::Core::Tokenizer.new
+        t.tokenize(command)
       end
 
       # Returns command tag from statement at specified index.  Allows for
@@ -78,6 +84,7 @@ module Sneaql
         c = Sneaql::Core.find_class(:command, this_cmd[:command]).new(
           nil,
           @expression_handler,
+          nil,
           @recordset_manager,
           nil,
           @logger

data/lib/sneaql_lib/repo_manager.rb

@@ -6,7 +6,7 @@ require_relative 'base.rb'
 module Sneaql
   # Classes to manage repositories full of SneaQL code.
   module RepoManagers
-    
+
     # tells you the repo type based upon the url
     # either git or http
     # @param [String] repo_url
@@ -14,8 +14,7 @@ module Sneaql
       return 'git' if repo_url.match(/\.*git.*/i)
       return 'http' if repo_url.match(/\.*http.*/i)
     end
-    
-    
+
     # pulls a branch from a remote git repo
     class GitRepoManager < Sneaql::Core::RepoDownloadManager
       Sneaql::Core::RegisterMappedClass.new(

data/lib/sneaql_lib/standard_db_objects.rb

@@ -39,19 +39,19 @@ module Sneaql
         )
         return true
       end
-      
+
       # Coerces a boolean to the appropriate value for the database type.
       # May return a 0 or 1 in RDBMS where boolean is not supported.
       # @param [Boolean] boolean_value
       # @return [Boolean, Fixnum]
       def coerce_boolean(boolean_value)
-        if @database_manager.has_boolean 
-          boolean_value 
-        else 
-          boolean_value == true ? 1 : 0 
+        if @database_manager.has_boolean
+          boolean_value
+        else
+          boolean_value == true ? 1 : 0
         end
       end
-      
+
       # Create table statement for primary transform table.
       # @param [String] transform_table_name fully qualified name for this table
       # @return [String]
@@ -68,7 +68,7 @@ module Sneaql
           	,updated_ts timestamp
           );}
       end
-      
+
       # Creates a record in the transforms table.
       # @param [String] transform_table_name
       # @param [Hash] params Hash of parameters with symbols matching column names
@@ -106,7 +106,7 @@ module Sneaql
             ,current_timestamp
           );}
       end
-      
+
       # Drops and recreates the transform steps table.
       # @param [String] transform_steps_table_name fully qualified name for this table
       # @return [Boolean]

data/lib/sneaql_lib/step_manager.rb

@@ -10,14 +10,21 @@ module Sneaql
         'local_file',
         Sneaql::StepManagers::JSONFileStepManager
       )
-  
+
       # Manages steps from a local JSON file.
       def manage_steps
-        @steps = JSON.parse(File.read(@params[:step_metadata_file_path])).sort_by! { |h| h['step_number'] }
-        @steps.map! { |j| { step_number: j['step_number'], step_file: j['step_file'] } }
+        @steps = JSON.parse(
+          File.read(@params[:step_metadata_file_path])
+        ).sort_by! { |h| h['step_number'] }
+        @steps.map! do |j|
+          {
+            step_number: j['step_number'],
+            step_file: j['step_file']
+          }
+        end
       end
     end
-  
+
     # source step metadata from a standardized table in the target database
     class TransformStepTableManager < Sneaql::Core::StepMetadataManager
       Sneaql::Core::RegisterMappedClass.new(
@@ -25,7 +32,7 @@ module Sneaql
         'transform_steps_table',
         Sneaql::StepManagers::TransformStepTableManager
       )
-      
+
       # Manages steps based in a standardized table.
       def manage_steps
         jdbc_connection = JDBCHelpers::ConnectionFactory.new(
@@ -33,28 +40,35 @@ module Sneaql
           @params[:db_user],
           @params[:db_pass]
         ).connection
-  
+
         @steps = JDBCHelpers::QueryResultsToArray.new(
           jdbc_connection,
-          %(select
-            transform_step
-            ,sql_file_path_in_repo
-          from
-            #{@params[:transform_steps_table]}
-          where
-            transform_name='#{@params[:transform_name]}'
-            and
-            is_active=#{ if @params[:database_manager].has_boolean then 'true' else 1 end }
-          order by
-            transform_step asc;)
+          steps_sql
         ).results
-  
-        @steps.map! do |s| 
-          { step_number: s['transform_step'], step_file: s['sql_file_path_in_repo'] }
+
+        @steps.map! do |s|
+          {
+            step_number: s['transform_step'],
+            step_file: s['sql_file_path_in_repo']
+          }
         end
-  
+
         jdbc_connection.close
       end
+
+      def steps_sql
+        %(select
+          transform_step
+          ,sql_file_path_in_repo
+        from
+          #{@params[:transform_steps_table]}
+        where
+          transform_name='#{@params[:transform_name]}'
+          and
+          is_active=#{@params[:database_manager].has_boolean ? 'true' : 1}
+        order by
+          transform_step asc;)
+      end
     end
   end
 end