sneaql 0.0.8-java

data/lib/sneaql_lib/base.rb

@@ -0,0 +1,224 @@
+require 'zip/zip'
+require 'fileutils'
+require 'logger'
+
+#top level namespace for sneaql objects
+module Sneaql
+  # contains the base classes for the extendable parts of sneaql:
+  #   commands (the actual commands specified in sneaql tags)
+  #   repo_managers (used to pull the sql files from a remote or local source)
+  #   metadata managers (to get information about each step)
+  # in addition to the base classes, the mapped class/class_map
+  # utilities are used to provide a dynamic class system
+  # this class system allows you to register a class under a type
+  # you can later look up the class by type and a text identifier
+  # then instantiate a new instance from there
+  module Core
+    # global map of all classes
+    @@class_map = {}
+
+    # allows external access to class_map for testing purposes
+    def self.class_map
+      @@class_map
+    end
+
+    # adds a new class to the map
+    # @param [Symbol] type class type (user settable, can be :command, :repo_manager, etc)
+    # @param [Hash] mapped_class_hash in the format { text: text, mapped_class: mapped_class }
+    def self.add_mapped_class(type, mapped_class_hash)
+      @@class_map[type] == [] unless @@class_map.keys.include?(type)
+      @@class_map[type] << mapped_class_hash
+    end
+
+    # makes sure that the type exists before appending the class information
+    # @param [Symbol] type class type (user settable, can be :command, :repo_manager, etc)
+    def self.insure_type_exists(type)
+      @@class_map[type] = [] unless @@class_map.key?(type)
+    end
+
+    # returns the class referenced by the type/text combination
+    # @param [Symbol] type class type (user settable, can be :command, :repo_manager, etc)
+    # @param [String] text to when searching within this type
+    # @return [Class] returns the class you are searching for
+    def self.find_class(type, text)
+      @@class_map[type].each do |t|
+        return t[:mapped_class] if t[:text] == text
+      end
+    end
+
+    # Handles registration of a class to the class_map
+    # Ignores duplicate definitions if they occur
+    class RegisterMappedClass
+      # Registers the class into the class_map.
+      # @param [Symbol] type class type (user settable, can be :command, :repo_manager, etc)
+      # @param [String] text to when searching within this type
+      # @param [Class] mapped_class class to be returned when search matches type and text
+      def initialize(type, text, mapped_class)
+        Sneaql::Core.insure_type_exists(type)
+        # check to see if the reference text is already being used by this type
+        unless Sneaql::Core.class_map[type].map { |c| c[:text] }.include?(text)
+          Sneaql::Core.add_mapped_class(
+            type,
+            { text: text, mapped_class: mapped_class }
+          )
+        end
+      end
+    end
+
+    # Base class for SneaQL command tags
+    class SneaqlCommand
+      # this is the base object for a sneaql command
+      # subclass this and override the action method
+      # @param [Object] jdbc_connection JDBC connection object to database
+      # @param [Sneaql::Core::ExpressionHandler] expression_handler
+      # @param [Sneaql::Core::RecordsetManager] recordset_manager
+      # @param [String] statement SQL statement provided in body, with all variables resolved
+      # @param [Logger] logger object otherwise will default to new Logger
+      def initialize(jdbc_connection, expression_handler, recordset_manager, statement, logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+
+        @jdbc_connection = jdbc_connection
+        @expression_handler = expression_handler
+        @statement = statement
+        @recordset_manager = recordset_manager
+      end
+
+      # override this method with the actual code for your command
+      def action
+        nil
+      end
+
+      # override with an array in the form [:expression, :operator]
+      def arg_definition
+        []
+      end
+
+      # override this if you have a complex tag structure
+      # @param [Array] args argument array to validate
+      # @return [Boolean] true if all arguments are valid
+      def validate_args(args)
+        return false if args.length != arg_definition.length
+        return true if (arg_definition == []) and (args == [])
+        valid = []
+        args.each_with_index do |a, i|
+          case
+          when arg_definition[i] == :variable then
+            valid << valid_variable?(a)
+          when arg_definition[i] == :expression then
+            valid << valid_expression?(a)
+          when arg_definition[i] == :operator then
+            valid << valid_operator?(a)
+          when arg_definition[i] == :recordset then
+            valid << valid_recordset?(a)
+          else valid << false end
+        end
+        @logger.debug(valid)
+        !valid.include?(false)
+      end
+
+      # validates that the value is a valid variable name
+      # @param [String] a value to test
+      # @return [Boolean]
+      def valid_variable?(a)
+        @expression_handler.valid_session_variable_name?(a.to_s.strip)
+      end
+
+      # validates that the value is a valid expression
+      # @param [String, Float, Fixnum] a value to test
+      # @return [Boolean]
+      def valid_expression?(a)
+        @expression_handler.valid_expression_reference?(a.to_s.strip)
+      end
+
+      # validates that the value is a valid operator
+      # @param [String] a value to test
+      # @return [Boolean]
+      def valid_operator?(a)
+        @expression_handler.valid_operators.include?(a.to_s.strip)
+      end
+
+      # validates that the value is a valid recordset name
+      # @param [String] a value to test
+      # @return [Boolean]
+      def valid_recordset?(a)
+        @recordset_manager.valid_recordset_name?(a.to_s.strip)
+      end
+
+      private
+
+      # these are set during initialize
+      # reference to jdbc connection object
+      attr_accessor :jdbc_connection
+
+      # reference to expression handler object for this transform
+      attr_accessor :expression_handler
+
+      # actual sql statement with all variables dereferenced
+      attr_accessor :statement
+    end # class
+
+    # base class for repo managers
+    class RepoDownloadManager
+      # this is the directory that the repo operates in
+      attr_reader :repo_base_dir
+
+      # @param [Hash] params parameters passed to transform will be passed here
+      # @param [Logger] logger object otherwise will default to new Logger
+      def initialize(params, logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+
+        @repo_base_dir = "#{params[:repo_base_dir]}/#{params[:transform_name]}"
+        @params = params
+
+        # perform the actual actions of managing the repo
+        manage_repo
+      end
+
+      # method to drop and rebuild the specified directory
+      # all files and subdirectories will be destroyed
+      # @param [String] directory
+      def drop_and_rebuild_directory(directory)
+        @logger.info("dropping and recreating repo directory #{directory}")
+        FileUtils.remove_dir(directory) if Dir.exist?(directory)
+        FileUtils.mkdir_p(directory)
+      end
+
+      # override in your implementation
+      def manage_repo
+        nil
+      end
+
+      # copied this from the internet
+      # http://www.markhneedham.com/blog/2008/10/02/ruby-unzipping-a-file-using-rubyzip/
+      def unzip_file(file, destination)
+        ::Zip::ZipFile.open(file) do |zip_file|
+          zip_file.each do |f|
+            f_path = File.join(destination, f.name)
+            FileUtils.mkdir_p(File.dirname(f_path))
+            zip_file.extract(f, f_path) unless File.exist?(f_path)
+          end
+        end
+      end
+    end
+
+    # abstracted to allow this metadata to come from any source
+    class StepMetadataManager
+      # value should be array of metadata hashes in the form `{ step_number: j['step_number'], step_file: j['step_file'] }`
+      attr_reader :steps
+
+      # @param [Hash] params parameters passed to transform will be passed here
+      # @param [Logger] logger object otherwise will default to new Logger
+      def initialize(params, logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+        @params = params
+        manage_steps
+      end
+
+      # override with a method that will override steps with an array of
+      # steps in the format :step_number, :step_file
+      def manage_steps
+        nil
+      end
+    end
+  end
+end

data/lib/sneaql_lib/core.rb

@@ -0,0 +1,346 @@
+require 'jdbc_helpers'
+require_relative 'base.rb'
+require_relative 'exceptions.rb'
+
+module Sneaql
+  module Core
+    # Core Sneaql language command tags.
+    # You can create your own tags by extending 
+    # Sneaql::Core::SneaqlCommand and overriding the
+    # action method.  You should also override arg_definition
+    # and potentially validate_args if have a complex argument
+    # structure.
+    module Commands
+      # assigns a session variable to a provided value
+      class SneaqlAssign < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'assign',
+          Sneaql::Core::Commands::SneaqlAssign
+        )
+
+        # @param [String] var_name
+        # @param [String] value expression (must be a string)
+        def action(var_name, value)
+          @expression_handler.set_session_variable(var_name, value)
+        end
+
+        # argument types
+        def arg_definition
+          [:variable, :expression]
+        end
+      end
+
+      # assigns a session variable to a value returned from a sql query
+      class SneaqlAssignResult < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'assign_result',
+          Sneaql::Core::Commands::SneaqlAssignResult
+        )
+
+        # run the query... then assign the result to a session variable
+        # @param [String] target_var_name
+        def action(target_var_name)
+          @expression_handler.set_session_variable(
+            target_var_name,
+            sql_result
+          )
+        end
+
+        # argument types
+        def arg_definition
+          [:variable]
+        end
+
+        # returns value at first row/field in result set
+        def sql_result
+          JDBCHelpers::SingleValueFromQuery.new(
+            @jdbc_connection,
+            @statement,
+            @logger
+          ).result
+        end
+      end
+
+      # executes a sql statement
+      class SneaqlExecute < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'execute',
+          Sneaql::Core::Commands::SneaqlExecute
+        )
+
+        # execute sql statement
+        # last_statement_rows_affected is always set...
+        def action
+          @expression_handler.set_session_variable(
+            'last_statement_rows_affected',
+            rows_affected
+          )
+        end
+
+        # @return [Fixnum] rows affected by SQL statement
+        def rows_affected
+          JDBCHelpers::Execute.new(
+            @jdbc_connection,
+            @statement,
+            @logger
+          ).rows_affected
+        end
+      end
+
+      # executes a sql statement if the condition evaluates to true
+      class SneaqlExecuteIf < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'execute_if',
+          Sneaql::Core::Commands::SneaqlExecuteIf
+        )
+
+        # @param [String] left_value expression as left operand
+        # @param [String] operator comparison operator supported by expression handler
+        # @param [String] right_value expression as right operand
+        def action(left_value, operator, right_value)
+          if @expression_handler.compare_expressions(operator, left_value, right_value)
+            @expression_handler.set_session_variable(
+              'last_statement_rows_affected',
+              rows_affected
+            )
+          end
+        end
+
+        # argument types
+        def arg_definition
+          [:expression, :operator, :expression]
+        end
+
+        # @return [Fixnum] rows affected by SQL statement
+        def rows_affected
+          JDBCHelpers::Execute.new(
+            @jdbc_connection,
+            @statement,
+            @logger
+          ).rows_affected
+        end
+      end
+
+      # compares the result of a sql statement against an argument
+      # raises error if the comparison does not evaluate to true
+      # the first field of the first record is used for the comparison
+      class SneaqlTest < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'test',
+          Sneaql::Core::Commands::SneaqlTest
+        )
+
+        # @param [String] operator comparison operator supported by expression handler
+        # @param [String] value_to_test expression as right operand
+        def action(operator, value_to_test)
+          unless @expression_handler.compare_expressions(
+            operator,
+            sql_result,
+            value_to_test
+          )
+            raise Sneaql::Exceptions::SQLTestExitCondition
+          end
+        end
+
+        # argument types
+        def arg_definition
+          [:operator, :expression]
+        end
+
+        # returns value at first row/field in result set
+        def sql_result
+          JDBCHelpers::SingleValueFromQuery.new(
+            @jdbc_connection,
+            @statement,
+            @logger
+          ).result
+        end
+      end
+
+      # raises an error to exit the transform if the condition evaluates to true
+      class SneaqlExitIf < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'exit_if',
+          Sneaql::Core::Commands::SneaqlExitIf
+        )
+
+        # @param [String] operand1 expression as left operand
+        # @param [String] operator comparison operator supported by expression handler
+        # @param [String] operand2 expression as right operand
+        def action(operand1, operator, operand2)
+          if @expression_handler.compare_expressions(operator, operand1, operand2)
+            raise Sneaql::Exceptions::SQLTestExitCondition
+          end
+        end
+
+        # argument types
+        def arg_definition
+          [:expression, :operator, :expression]
+        end
+      end
+
+      # raises an error to exit the transform step if the comdition evaluates to true
+      # note that this error needs to be handled accordingly in the calling
+      # procedure as all other errors will end the transform
+      class SneaqlExitStepIf < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'exit_step_if',
+          Sneaql::Core::Commands::SneaqlExitStepIf
+        )
+
+        # @param [String] operand1 expression as left operand
+        # @param [String] operator comparison operator supported by expression handler
+        # @param [String] operand2 expression as right operand
+        def action(operand1, operator, operand2)
+          if @expression_handler.compare_expressions(operator, operand1, operand2)
+            raise Sneaql::Exceptions::SQLTestStepExitCondition
+          end
+        end
+
+        # argument types
+        def arg_definition
+          [:expression, :operator, :expression]
+        end
+      end
+
+      # runs the query then stores the array of hashes into the recordset hash
+      class SneaqlRecordsetFromQuery < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'recordset',
+          Sneaql::Core::Commands::SneaqlRecordsetFromQuery
+        )
+
+        # @param [String] recordset_name name of the recordset in which to store the results
+        def action(recordset_name)
+          r = query_results
+          @logger.debug "adding #{r.length} recs as #{recordset_name}"
+          @recordset_manager.store_recordset(recordset_name, r)
+        end
+
+        # argument types
+        def arg_definition
+          [:recordset]
+        end
+
+        # @return [Array] returns array of hashes from SQL results
+        def query_results
+          JDBCHelpers::QueryResultsToArray.new(
+            @jdbc_connection,
+            @statement,
+            @logger
+          ).results
+        end
+      end
+
+      # iterates a recordset and runs the sql statement for each record
+      class SneaqlIterateRecordset < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'iterate',
+          Sneaql::Core::Commands::SneaqlIterateRecordset
+        )
+
+        # @param [*Array] args parameters for recordset expression in the format
+        def action(*args)
+          if args.size == 1
+            iterate_all_records(*args)
+          elsif ((args.size - 1) % 4) == 0
+            iterate_records_conditionally(*args)
+          end
+        end
+
+        # custom method for argument validation
+        # @param [Array] args
+        # @return [Boolean]
+        def validate_args(args)
+          if args.size == 1
+            return valid_recordset?(args[0])
+          elsif ((args.size - 1) % 4) == 0
+            valid = []
+            valid << valid_recordset?(args[0])
+            args[1..args.length - 1].each_slice(4) do |s|
+              if ['include', 'exclude'].include?(s[0])
+                valid << true
+              else
+                valid << false
+              end
+              # field names have the same rules as recordset names for now
+              valid << valid_recordset?(s[1])
+              valid << valid_operator?(s[2])
+              valid << valid_expression?(s[3])
+            end
+            !valid.include?(false)
+          else
+            return false
+          end
+        end
+
+        # @param [String] recordset recordset to iterate
+        def iterate_all_records(recordset)
+          @logger.info "iterating recordset #{recordset}..."
+          @recordset_manager.recordset[recordset].each_with_index do |i, n|
+            @logger.debug("#{n + 1} of #{recordset}: #{i}")
+            tmp = @statement
+            i.keys.each { |k| tmp = tmp.gsub(":#{recordset}.#{k}", i[k].to_s) }
+            @expression_handler.set_session_variable(
+              'last_statement_rows_affected',
+              rows_affected_current_statement(tmp)
+            )
+          end
+        end
+
+        # @param [*Array] args all the arguments passed to the calling function
+        def iterate_records_conditionally(*args)
+          recordset = args.to_a[0]
+          @logger.info "iterating recordset #{recordset}..."
+          conditions = @recordset_manager.parse_recordset_expression(args.to_a)
+          @recordset_manager.recordset[recordset].each_with_index do |i, n|
+            @logger.debug("#{n + 1} of #{recordset}: #{i}")
+            next unless @recordset_manager.evaluate_expression_against_record(i, conditions)
+            tmp = @statement
+            i.keys.each { |k| tmp = tmp.gsub(":#{recordset}.#{k}", i[k].to_s) }
+            @expression_handler.set_session_variable(
+              'last_statement_rows_affected',
+              rows_affected_current_statement(tmp)
+            )
+          end
+        end
+
+        # @return [Fixnum] rows affected by the SQL statement
+        def rows_affected_current_statement(stmt)
+          JDBCHelpers::Execute.new(
+            @jdbc_connection,
+            stmt,
+            @logger
+          ).rows_affected
+        end
+      end
+
+      # stores all the file paths matching the dir glob into a recordset
+      class SneaqlRecordsetFromDirGlob < Sneaql::Core::SneaqlCommand
+        Sneaql::Core::RegisterMappedClass.new(
+          :command,
+          'rs_from_local_dir',
+          Sneaql::Core::Commands::SneaqlRecordsetFromDirGlob
+        )
+
+        # @param [String] recordset_name
+        # @param [String] dirglob directory glob with optional wildcards
+        def action(recordset_name, dirglob)
+          r = Dir.glob(dirglob)
+          r.map! { |d| { 'path_name' => d.to_s } }
+          @logger.debug "adding #{r.length} recs as #{recordset_name}"
+          @recordset_manager.store_recordset(recordset_name, r)
+        end
+      end
+    end
+  end
+end

data/lib/sneaql_lib/database_manager.rb

@@ -0,0 +1,79 @@
+module Sneaql
+  module Core
+    # returns the database type based upon the jdbc url
+    # @param [String]
+    def self.database_type(jdbc_url)
+      Sneaql::Core.class_map[:database].each do |d|
+        return d[:text] if jdbc_url.match(d[:text])
+      end
+    end
+    
+    # Manages preferences for a specific RDBMS implementation.
+    class DatabaseManager
+      attr_reader(
+        :has_boolean,
+        :autocommit_off_statement,
+        :supports_transactions,
+        :supports_table_locking,
+        :begin_statement,
+        :commit_statement,
+        :rollback_statement
+      )
+      
+      # @param [Hash] options values to override defaults
+      def initialize(options = {})
+        @has_boolean = options.fetch(:has_boolean, default_has_boolean)
+        @autocommit_off_statement = options.fetch(:autocommit_off_statement, default_autocommit_off_statement)
+        @supports_transactions = options.fetch(:supports_transactions, default_supports_transactions)
+        @supports_table_locking = options.fetch(:supports_table_locking, default_supports_table_locking)
+        @begin_statement = options.fetch(:begin_statement, default_begin_statement)
+        @commit_statement = options.fetch(:commit_statement, default_commit_statement)
+        @rollback_statement = options.fetch(:rollback_statement, default_rollback_statement)
+      end
+      
+      # @return [Boolean]
+      def default_has_boolean
+        false
+      end
+      
+      # @return [String]
+      def default_autocommit_off_statement
+        nil
+      end
+      
+      # @return [Boolean]
+      def default_supports_transactions
+        true
+      end
+
+      # @return [Boolean]
+      def default_supports_table_locking
+        false
+      end
+           
+      # @return [String] begin statement
+      def default_begin_statement
+        "begin;"
+      end
+
+      # @return [String] commit statement
+      def default_commit_statement
+        "commit;"
+      end
+      
+      # @return [String] rollback statement
+      def default_rollback_statement
+        "rollback;"
+      end
+      
+      # @param [String] table_name
+      # @return [String] rollback statement
+      def lock_table_statement(table_name)
+        "lock table #{table_name};"
+      end
+    end
+  end
+end
+
+# Require all tested RDBMS
+Dir.glob("#{File.dirname(__FILE__)}/database_prefs/*.rb").each { |f| require File.expand_path(f) }

data/lib/sneaql_lib/database_prefs/redshift.rb

@@ -0,0 +1,22 @@
+module Sneaql
+  module Core
+    # Settings for interaction with Amazon Redshift
+    class RedshiftDatabaseManager < Sneaql::Core::DatabaseManager
+      Sneaql::Core::RegisterMappedClass.new(
+        :database,
+        'redshift',
+        Sneaql::Core::RedshiftDatabaseManager
+      ) 
+      
+      def initialize()
+        super(
+          {
+            has_boolean: true,
+            autocommit_off_statement: 'set autocommit=off;'
+          }
+        )  
+      end
+      
+    end
+  end
+end

data/lib/sneaql_lib/database_prefs/sqlite.rb

@@ -0,0 +1,12 @@
+module Sneaql
+  module Core
+    # Settings for interacting with SQLite
+    class SqliteDatabaseManager < Sneaql::Core::DatabaseManager
+      Sneaql::Core::RegisterMappedClass.new(
+        :database,
+        'sqlite',
+        Sneaql::Core::SqliteDatabaseManager
+      ) 
+    end
+  end
+end

data/lib/sneaql_lib/database_prefs/vertica.rb

@@ -0,0 +1,21 @@
+module Sneaql
+  module Core
+    # Settings for interacting with HP Vertica
+    class VerticaDatabaseManager < Sneaql::Core::DatabaseManager
+      Sneaql::Core::RegisterMappedClass.new(
+        :database,
+        'vertica',
+        Sneaql::Core::VerticaDatabaseManager
+      ) 
+      
+      def initialize
+        super(
+          {
+            has_boolean: true,
+            autocommit_off_statement: 'set session autocommit to off;'
+          }
+        )  
+      end
+    end
+  end
+end