sneaql 0.0.8-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 007ecaf278e3fd4e9fbed5c695fa84713d240e0c
+  data.tar.gz: bd7bb2e4dc227543ddf499ef7f8092463f4fb6f9
+SHA512:
+  metadata.gz: bfcd8b46a3005f5d75b0b122063413ea537d20b5dec17221aec83d4ba7c4d7aed4ae7d51de29d0fd67f1d3c81c0fdff940baed2c5cf5b46fa6e2381f32d731d6
+  data.tar.gz: ed18029224f57ff17604ec1edb57754bd6a88ecd0319cbd15d8adc4e3ed86ea33c435d1dd1ca61c66e76067b1accf32b16a0e70572cd0dae402f7886931ffeb8

data/bin/sneaql

@@ -0,0 +1,273 @@
+#!/usr/bin/env ruby
+
+require 'sneaql'
+require 'thor'
+require 'dotenv'
+
+# configure global logger
+module SneaqlStandard
+  Log = Logger.new(STDOUT)
+end
+
+def logger
+  SneaqlStandard::Log
+end
+
+def configure_logging_format
+  # custom formatter provides logging with thread id and multi-line
+  # entries each receiving their own log prefix
+  if ENV['SNEAQL_LOG_UTC']
+    logger.formatter = proc do |severity, datetime, _progname, msg|
+      t = ''
+      msg.to_s.split(/\n+/).each do |line|
+        t += "[#{severity}] #{datetime.utc} tid#{Thread.current.object_id}: #{line}\n"
+      end
+      t
+    end
+  else
+    # default to local TZ
+    logger.formatter = proc do |severity, datetime, _progname, msg|
+      t = ''
+      msg.to_s.split(/\n+/).each do |line|
+        t += "[#{severity}] #{datetime} tid#{Thread.current.object_id}: #{line}\n"
+      end
+      t
+    end
+  end
+end
+
+
+def local_gems
+   Gem::Specification.sort_by{ |g| [g.name.downcase, g.version] }.group_by{ |g| g.name }
+end
+
+def require_sneaql_extensions()
+  extensions = ENV['SNEAQL_EXTENSIONS'].split(',')
+  local_gems.keys.each do |r| 
+    if extensions.include?(r) 
+      require r
+      logger.info "loading sneaql extension #{r}"
+    end
+  end
+end
+
+# sneaql
+module Sneaql
+  # standard implementation of sneaql
+  module Standard
+    # extends thor for cli functionality
+    class CLI < Thor
+
+  		desc "exec PATH_TO_TRANFORM_ROOT", "run sneaql, default action"
+  		
+  		method_option :step_file, :type => :string, :aliases => "-s"
+      method_option :env_file, :type => :string, :aliases => "-e"
+      method_option :jdbc_url, :type => :string, :aliases => "-url"
+      method_option :db_user, :type => :string, :aliases => "-u"
+      method_option :db_pass, :type => :string, :aliases => "-p"
+      method_option :jdbc_driver_jar, :type => :string, :aliases => "-j"
+      method_option :jdbc_driver_class, :type => :string, :aliases => "-c"
+      method_option :debug , :default => false, :type => :boolean
+      
+  		long_desc <<-LONGDESC
+executes the transform provided
+
+PARAMETERS
+
+you can provide the following parameters via an environment variable or by putting them in a sneaql.env file
+\n
+* export SNEAQL_JDBC_URL=jdbc:redshift://your-redshift-hostname:5439/dbname \n
+* export SNEAQL_DB_USER="dbadmin" \n
+* export SNEAQL_DB_PASS="password" \n
+* export SNEAQL_JDBC_DRIVER_JAR="java/RedshiftJDBC4-1.1.6.1006.jar" \n
+* export SNEAQL_JDBC_DRIVER_CLASS="com.amazon.redshift.jdbc4.Driver" \n
+\n
+note that if you provide an attribute on the command line it will override the environment variable value.
+\n
+EXAMPLE \n
+$ sneaql exec . \n
+the above will execute the transform in the current directory.  note that sneaql.json file must be in this directory or provided via -s option.
+      LONGDESC
+      
+      def exec(transform_path)
+        begin
+          logger.level = Logger::INFO
+          logger.level = Logger::DEBUG if options[:debug] == true
+          
+          Dotenv.load(options[:env_file] ? options[:env_file] : 'sneaql.env')
+          
+          base_path = File.expand_path(transform_path)
+          raise 'base_path not provided' if base_path == nil
+          
+          step_file = options[:step_file] ? options[:step_file] : "#{base_path}/sneaql.json"
+          raise 'step_file not provided' if step_file == nil
+          
+          jdbc_url = options[:jdbc_url] ? options[:jdbc_url] : ENV['SNEAQL_JDBC_URL']
+          raise 'jdbc_url not provided' if jdbc_url == nil
+          
+          db_user = options[:db_user] ? options[:db_user] : ENV['SNEAQL_DB_USER']
+          raise 'db_user not provided' if db_user == nil
+          
+          db_pass = options[:db_pass] ? options[:db_pass] : ENV['SNEAQL_DB_PASS']
+          raise 'db_pass not provided' if db_pass == nil
+          
+          jdbc_driver_jar = options[:jdbc_driver_jar] ? options[:jdbc_driver_jar] : ENV['SNEAQL_JDBC_DRIVER_JAR']
+          raise 'jdbc_driver_jar not provided' if jdbc_driver_jar == nil
+          
+          jdbc_driver_class = options[:jdbc_driver_class] ? options[:jdbc_driver_class] : ENV['SNEAQL_JDBC_DRIVER_CLASS']
+          raise 'jdbc_driver_class not provided' if step_file == nil
+        rescue => e
+          puts e.message
+          exit 1
+        end
+        
+        configure_logging_format()
+        require_sneaql_extensions() if ENV['SNEAQL_EXTENSIONS']
+        
+        logger.debug("available sneaql resources: #{Sneaql::Core.class_map}")
+        
+        database = Sneaql::Core.database_type(jdbc_url)
+        unless database
+          raise 'database specified in jdbc url is not supported, or malformed jdbc url'
+          exit 1 
+        end
+        
+        begin  
+          require_relative jdbc_driver_jar
+          java_import jdbc_driver_class
+        rescue => e
+          raise 'jdbc driver error'
+          exit 1
+        end
+
+        t = Sneaql::Transform.new(
+          {  
+            transform_name: base_path,
+            repo_base_dir: base_path,
+            repo_type: 'local',
+            database: database,
+            jdbc_url: jdbc_url,
+            db_user: db_user,
+            db_pass: db_pass,
+            step_metadata_manager_type: 'local_file',
+            step_metadata_file_path: step_file
+          },
+          logger
+        )
+        t.run
+        
+        exit t.exit_code
+      rescue => e
+        logger.error e.message
+        e.backtrace.each { |b| logger.error b }
+        exit 1
+  		end
+  		
+  		desc "dryrun PATH_TO_TRANFORM_ROOT", "validate transform"
+  		
+  		method_option :step_file, :type => :string, :aliases => "-s"
+      method_option :env_file, :type => :string, :aliases => "-e"
+      method_option :jdbc_url, :type => :string, :aliases => "-url"
+      method_option :db_user, :type => :string, :aliases => "-u"
+      method_option :db_pass, :type => :string, :aliases => "-p"
+      method_option :jdbc_driver_jar, :type => :string, :aliases => "-j"
+      method_option :jdbc_driver_class, :type => :string, :aliases => "-c"
+      
+  		long_desc <<-LONGDESC
+executes the transform provided
+
+PARAMETERS
+
+you can provide the following parameters via an environment variable or by putting them in a sneaql.env file
+\n
+* export SNEAQL_JDBC_URL=jdbc:redshift://your-redshift-hostname:5439/dbname \n
+* export SNEAQL_DB_USER="dbadmin" \n
+* export SNEAQL_DB_PASS="password" \n
+* export SNEAQL_JDBC_DRIVER_JAR="java/RedshiftJDBC4-1.1.6.1006.jar" \n
+* export SNEAQL_JDBC_DRIVER_CLASS="com.amazon.redshift.jdbc4.Driver" \n
+\n
+note that if you provide an attribute on the command line it will override the environment variable value.
+\n
+EXAMPLE \n
+$ sneaql exec . \n
+the above will execute the transform in the current directory.  note that sneaql.json file must be in this directory or provided via -s option.
+      LONGDESC
+      
+      def dryrun(transform_path)
+        begin
+          logger.level = Logger::DEBUG
+          
+          Dotenv.load(options[:env_file] ? options[:env_file] : 'sneaql.env')
+
+          base_path = File.expand_path(transform_path)
+          raise 'base_path not provided' if base_path == nil
+          
+          step_file = options[:step_file] ? options[:step_file] : "#{base_path}/sneaql.json"
+          raise 'step_file not provided' if step_file == nil
+          
+          jdbc_url = options[:jdbc_url] ? options[:jdbc_url] : ENV['SNEAQL_JDBC_URL']
+          raise 'jdbc_url not provided' if jdbc_url == nil
+          
+          db_user = options[:db_user] ? options[:db_user] : ENV['SNEAQL_DB_USER']
+          raise 'db_user not provided' if db_user == nil
+          
+          db_pass = options[:db_pass] ? options[:db_pass] : ENV['SNEAQL_DB_PASS']
+          raise 'db_pass not provided' if db_pass == nil
+          
+          jdbc_driver_jar = options[:jdbc_driver_jar] ? options[:jdbc_driver_jar] : ENV['SNEAQL_JDBC_DRIVER_JAR']
+          raise 'jdbc_driver_jar not provided' if jdbc_driver_jar == nil
+          
+          jdbc_driver_class = options[:jdbc_driver_class] ? options[:jdbc_driver_class] : ENV['SNEAQL_JDBC_DRIVER_CLASS']
+          raise 'jdbc_driver_class not provided' if step_file == nil
+        rescue => e
+          puts e.message
+          exit 1
+        end
+        
+        configure_logging_format()
+        require_sneaql_extensions() if ENV['SNEAQL_EXTENSIONS']
+        
+        logger.debug("available sneaql resources: #{Sneaql::Core.class_map}")
+        
+        database = Sneaql::Core.database_type(jdbc_url)
+        unless database
+          raise 'database specified in jdbc url is not supported, or malformed jdbc url'
+          exit 1 
+        end
+        
+        begin  
+          require_relative jdbc_driver_jar
+          java_import jdbc_driver_class
+        rescue => e
+          raise 'jdbc driver error'
+          exit 1
+        end
+
+        t = Sneaql::Transform.new(
+          {  
+            transform_name: base_path,
+            repo_base_dir: base_path,
+            repo_type: 'local',
+            database: database,
+            jdbc_url: jdbc_url,
+            db_user: db_user,
+            db_pass: db_pass,
+            step_metadata_manager_type: 'local_file',
+            step_metadata_file_path: step_file
+          },
+          logger
+        )
+        t.validate
+        
+        exit t.exit_code
+      rescue => e
+        logger.error e.message
+        e.backtrace.each { |b| logger.error b }
+        exit 1
+  		end
+
+    end
+  end
+end
+
+Sneaql::Standard::CLI.start(ARGV)

data/lib/sneaql.rb

@@ -0,0 +1,284 @@
+require 'jdbc_helpers'
+require 'logger'
+
+#require_relative 'sneaql_lib/logging.rb'
+require_relative 'sneaql_lib/exceptions.rb'
+require_relative 'sneaql_lib/core.rb'
+require_relative 'sneaql_lib/lock_manager.rb'
+require_relative 'sneaql_lib/repo_manager.rb'
+require_relative 'sneaql_lib/step_manager.rb'
+require_relative 'sneaql_lib/parser.rb'
+require_relative 'sneaql_lib/expressions.rb'
+require_relative 'sneaql_lib/recordset.rb'
+require_relative 'sneaql_lib/database_manager.rb'
+require_relative 'sneaql_lib/standard_db_objects.rb'
+
+# module for sneaql
+module Sneaql
+  # Manages and executes a SneaQL transform.
+  class Transform
+    attr_reader :current_step
+    attr_reader :current_statement
+    attr_reader :start_time
+    attr_reader :end_time
+    attr_reader :exit_code
+    attr_reader :transform_error
+    attr_reader :status
+
+    # Valid transform statuses
+    # :initializing, :connecting_to_database, :running, :completed, :error
+    # @return [Array] array of valid transform statuses
+    def valid_statuses
+      [:initializing, :connecting_to_database, :running, :completed, :error, :validating, :validated]
+    end
+
+    # Sets the current status of the transform.
+    # Must be a valid status type or it will not be set.
+    # Override this if you want to implement a custom status communication to
+    # an external target.
+    # @param [Symbol] status
+    # @see valid_status
+    def current_status(status)
+      @status = status if valid_statuses.include?(status)
+    end
+
+    # Creates a SneaQL transform object.
+    # @example
+    #   t=Sneaql::Transform.new({
+    #     transform_name: 'test-transform',
+    #     repo_base_dir: "test/fixtures/test-transform",
+    #     repo_type: 'local',
+    #     database: 'sqlite',
+    #     jdbc_url: 'jdbc:sqlite:memory',
+    #     db_user: 'dbuser',
+    #     db_pass: 'password',
+    #     step_metadata_manager_type: 'local_file',
+    #     step_metadata_file_path: "test/fixtures/test-transform/steps.json"
+    #   }, logger)
+    #
+    #   t.run
+    # @param [Hash] params various parameters are passed to define the transform
+    # @param [Logger] logger customer logger if provided (otherwise default logger is created)
+    def initialize(params, logger = nil)
+      @logger = logger ? logger : Logger.new(STDOUT)
+
+      @start_time = Time.new.utc
+
+      current_status(:initializing)
+      @params = params
+
+      @exit_code = 0
+
+      @transform_name = @params[:transform_name]
+      @transform_lock_id = @params[:transform_lock_id]
+      @jdbc_url = @params[:jdbc_url]
+      @db_user = @params[:db_user]
+      @db_pass = @params[:db_pass]
+      @database = @params[:database]
+
+      run if @params[:run] == true
+    end
+
+    # validate the transform.
+    def validate
+      @expression_handler = create_expression_handler
+      @recordset_manager = create_recordset_manager
+      @repo_manager = create_repo_manager
+      @steps = create_metadata_manager
+      @parsed_steps = create_parsed_steps(@steps)
+      current_status(:validating)
+      validate_parsed_steps(@parsed_steps)
+    rescue Sneaql::Exceptions::TransformIsLocked => e
+      @transform_error = e
+      @logger.info(e.message)
+    rescue Sneaql::Exceptions::SQLTestExitCondition => e
+      @transform_error = nil
+      @logger.info(e.message)
+    rescue => e
+      @exit_code = 1
+      @transform_error = e
+      current_status(:error)
+      @logger.error(e.message)
+      e.backtrace { |b| @logger.error b }
+    ensure
+      @end_time = Time.new.utc
+
+      if @transform_error.nil?
+        current_status(:validated)
+      else
+        current_status(:error)
+      end
+
+      @logger.info("#{@transform_name} validation time #{@end_time - @start_time}s")
+      @logger.info("#{@transform_name} exit code: #{@exit_code} status: #{@status}")
+    end
+    
+    # Runs the actual transform.
+    def run
+      @expression_handler = create_expression_handler
+      @recordset_manager = create_recordset_manager
+      @repo_manager = create_repo_manager
+      @lock_manager = create_lock_manager if @params[:locked_transform] == true
+      @steps = create_metadata_manager
+      @parsed_steps = create_parsed_steps(@steps)
+      validate_parsed_steps(@parsed_steps)
+      @jdbc_connection = create_jdbc_connection
+      current_status(:running)
+      iterate_steps_and_statements
+    rescue Sneaql::Exceptions::TransformIsLocked => e
+      @transform_error = e
+      @logger.info(e.message)
+    rescue Sneaql::Exceptions::SQLTestExitCondition => e
+      @transform_error = nil
+      @logger.info(e.message)
+    rescue => e
+      @exit_code = 1
+      @transform_error = e
+      current_status(:error)
+      @logger.error(e.message)
+      e.backtrace { |b| @logger.error b }
+    ensure
+      @lock_manager.remove_lock if @params[:locked_transform] == true
+      @jdbc_connection.close if @jdbc_connection
+      @end_time = Time.new.utc
+
+      if @transform_error.nil?
+        current_status(:completed)
+      else
+        current_status(:error)
+      end
+
+      @logger.info("#{@transform_name} runtime #{@end_time - @start_time}s")
+      @logger.info("#{@transform_name} exit code: #{@exit_code} status: #{@status}")
+    end
+
+    # Creates an ExpressionHandler object
+    # @return [Sneaql::Core::ExpressionHandler]
+    def create_expression_handler
+      Sneaql::Core::ExpressionHandler.new(ENV, @logger)
+    end
+
+    # Creates a RepoDownloadManager object
+    # The actual object returns depends upon params[:repo_type] provided at initialize.
+    # @return [Sneaql::Core::RepoDownloadManager]
+    def create_repo_manager
+      Sneaql::Core.find_class(:repo_manager, @params[:repo_type]).new(@params, @logger)
+    end
+
+    # Creates a TransformLockManager object
+    # The actual object returns depends upon params[:locked_transform] provided at initialize.
+    # @return [Sneaql::Core::RepoDownloadManager]
+    def create_lock_manager
+      # create a lock manager for this transform (uses a separate connection)
+      lock_manager = Sneaql::TransformLockManager.new(@params, @logger)
+      raise Sneaql::Exceptions::TransformIsLocked unless lock_manager.acquire_lock == true
+      lock_manager
+    end
+
+    # Creates a StepMetadataManager object
+    # The actual object returns depends upon params[:step_metadata_manager_type] provided at initialize.
+    # @return [Sneaql::Core::StepMetadataManager]
+    def create_metadata_manager
+      Sneaql::Core.find_class(
+        :step_metadata_manager,
+        @params[:step_metadata_manager_type]
+      ).new(@params, @logger).steps
+    end
+
+    # Creates a StepParser object for each step file defined by the metadata manager.
+    # @param [Array] steps takes an array of step definitions
+    # @return [Array] of Sneaql::Core::StepParser
+    def create_parsed_steps(steps)
+      steps.map do |s|
+        {
+          parser: Sneaql::Core::StepParser.new(
+            "#{@repo_manager.repo_base_dir}/#{s[:step_file]}",
+            @expression_handler,
+            @recordset_manager,
+            @logger
+          ),
+          step_number: s[:step_number]
+        }
+      end
+    end
+
+    # Validates the arguments for all tags.
+    # @param [Array<Sneaql::Core::StepParser>] steps
+    def validate_parsed_steps(steps)
+      steps.each do |s|
+        raise Sneaql::Exceptions::StatementParsingError unless s[:parser].valid_arguments_in_all_statements?
+      end
+    end
+
+    # Creates an RecordsetManager object
+    # @return [Sneaql::Core::RecordsetManager]
+    def create_recordset_manager
+      Sneaql::Core::RecordsetManager.new(@expression_handler, @logger)
+    end
+
+    # Creates a JDBC connection
+    # JDBC drivers must loaded into jruby before this will work.
+    def create_jdbc_connection
+      # db specific driver should have been handled by the calling procedure
+      current_status(:connecting_to_database)
+      JDBCHelpers::ConnectionFactory.new(
+        @jdbc_url,
+        @db_user,
+        @db_pass,
+        @logger
+      ).connection
+    end
+
+    # Performs the actual work of running the transform steps.
+    # This method operates within the context of a single
+    # database session across all steps.  If it fails, it will
+    # not rollback automatically unless that is the default RDBMS
+    # behavior for a connection that closes before a commit.
+    def iterate_steps_and_statements
+      @parsed_steps.each do |this_step|
+        #special handling is required for the exit_step_if command
+        #because there is a nested loop the exit_step var is needed
+        exit_step = false
+        break if exit_step == true
+        # set this so that other processes can poll the state
+        @current_step = this_step[:step_number]
+        # within a step... iterate through each statement
+        this_step[:parser].statements.each_with_index do |this_stmt, stmt_index|
+          # set this so that other processes can poll the state
+          @current_statement = stmt_index + 1
+
+          # log some useful info
+          @logger.info("step: #{@current_step} statement: #{@current_statement}")
+          @expression_handler.output_all_session_variables
+
+          # get the command hash for the current statement
+          this_cmd = this_step[:parser].command_at_index(stmt_index)
+          @logger.debug(this_cmd)
+
+          # evaluate any variable references in the arguments
+          if this_cmd[:arguments]
+            this_cmd[:arguments].map! { |a| @expression_handler.evaluate_expression(a) }
+          end
+
+          begin
+            # instantiate a new instance of the command class
+            # and call it's action method with arguments
+            c = Sneaql::Core.find_class(:command, this_cmd[:command]).new(
+              @jdbc_connection,
+              @expression_handler,
+              @recordset_manager,
+              @expression_handler.evaluate_all_expressions(this_stmt),
+              @logger
+            )
+
+            c.action(*this_cmd[:arguments])
+          rescue Sneaql::Exceptions::SQLTestStepExitCondition => e
+            exit_step = true
+            @logger.info e.message
+            break
+          end
+        end
+      end
+    end
+  end
+end