sneaql-standard 0.0.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 93a14cbef7a60a4a02e32472240d9039a8299ec5
+  data.tar.gz: 025aa637977c8ec97d79283cd1e3a1366705265f
+SHA512:
+  metadata.gz: c770787a6e7188206f0702f8c2934616a24949ef456338d38e813f38a7c7d73fc6c19273977c51bff0cf263587401b0172334eec6cd407f874c58e7b8178d6b5
+  data.tar.gz: 3c42b563381fcd46d67c97bd91fbe2a65f97e26b48944947a74c439dc9ee9fb05751968f55be600bbe067beaf38fb673d5ff3208bd646475fec28ea8028d798d

data/bin/sneaql

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require 'sneaql_standard'
+require 'thor'
+
+module Sneaql
+  module Standard
+    class CLI < Thor
+
+  		desc "run_transforms", "run sneaql transforms"
+  		long_desc <<-LONGDESC
+runs transforms based upon current ENV vars
+
+      LONGDESC
+      def run_transforms()
+        Sneaql::SneaqlStandard.new().run()
+  		end
+  		
+  		desc "configure_db", "configure database"
+  		long_desc <<-LONGDESC
+creates the sneaql.transforms table
+
+      LONGDESC
+      def configure_db()
+        Sneaql::SneaqlStandard.new().create_db_objects()
+  		end
+  		
+    end
+  end
+end
+
+Sneaql::Standard::CLI.start(ARGV)

data/lib/sneaql_standard.rb

@@ -0,0 +1,252 @@
+require 'sneaql'
+require 'jdbc_helpers'
+require 'dotenv'
+require 'thread'
+
+require_relative 'sneaql_standard_lib/logging.rb'
+require_relative 'sneaql_standard_lib/parallelize.rb'
+require_relative 'sneaql_standard_lib/jdbc_drivers.rb'
+
+Dotenv.load
+
+module Sneaql
+  # top level class for interacting with sneaql standard
+  class SneaqlStandard
+    # exposed for unit testing
+    attr_reader :params
+    attr_accessor :q
+
+    # runs all transforms
+    def run
+      set_params
+      configure_jdbc_driver
+      build_transform_queue
+      run_transforms
+    end
+
+    # creates database objects
+    # @param [String] transform_table_name if provided will override sneaql.transforms
+    def create_db_objects(transform_table_name = nil)
+      set_params
+      configure_jdbc_driver
+      create_transforms_table(transform_table_name)
+    end
+
+    # creates transform_table
+    # @param [String] transform_table_name if provided will override sneaql.transforms
+    def create_transforms_table(transform_table_name = nil)
+      transform_table_name = 'sneaql.transforms' unless transform_table_name
+
+      connection = create_connection
+
+      db_manager = Sneaql::Core.find_class(
+        :database,
+        @params[:database]
+      ).new
+
+      if transform_table_name =~ /\w+\.\w+/
+        # indicates schema qualfied object
+        # make sure db supports schemas
+        unless ['sqlite'].include?(@params[:database])
+          # create schema if needed
+          JDBCHelpers::Execute.new(
+            connection,
+            "create schema if not exists #{transform_table_name.match(/^\w+/)};"
+          )
+        end
+      end
+
+      creator = Sneaql::Standard::DBObjectCreator.new(
+        connection,
+        db_manager,
+        logger
+      )
+      creator.create_transforms_table(transform_table_name)
+
+    ensure
+      connection.close
+    end
+
+    # processes environment variables
+    def set_params
+      @params = {}
+
+      # each of these lil' hashes represents an env_var
+      # that is required, as well as an optional
+      # regex validation
+      [
+        { var: 'SNEAQL_JDBC_URL', sym: :jdbc_url, validation: /^jdbc\:.+/i },
+        { var: 'SNEAQL_DB_USER', sym: :db_user },
+        { var: 'SNEAQL_DB_PASS', sym: :db_pass },
+        { var: 'SNEAQL_JDBC_DRIVER_JAR', sym: :jdbc_driver_jar, validation: /^(http\:\/\/.+|file\:\/\/.+|s3\:\/\/.+)/i },
+        { var: 'SNEAQL_JDBC_DRIVER_CLASS', sym: :jdbc_driver_class }
+      ].each do |env_var|
+        raise "required environment variable #{env_var[:var]} not provided" unless ENV[env_var[:var]]
+        # assign the value of the env_var to the symbol key of @params
+        @params[env_var[:sym]] = ENV[env_var[:var]]
+
+        # validate if a validation is provided
+        if env_var[:validation]
+          unless @params[env_var[:sym]] =~ (env_var[:validation])
+            raise "required environment variable #{env_var[:var]} looks invalid"
+          end
+        end
+      end
+
+      # optional env vars are iterated in a similar manner
+      # but instead of validation they have a default
+      [
+        {
+          var: 'SNEAQL_JDBC_DRIVER_JAR_MD5',
+          sym: :jdbc_driver_jar_md5,
+          default: nil
+        },
+        {
+          var: 'SNEAQL_METADATA_MANAGER_TYPE',
+          sym: :step_metadata_manager_type,
+          default: 'transform_steps_table'
+        },
+        {
+          var: 'SNEAQL_REPO_BASE_DIR',
+          sym: :repo_base_dir,
+          default: '/tmp/sneaql/repos'
+        },
+        {
+          var: 'SNEAQL_TRANSFORM_CONCURRENCY',
+          sym: :concurrency,
+          default: 1
+        },
+        {
+          var: 'SNEAQL_TRANSFORM_TABLE_NAME',
+          sym: :transform_table_name,
+          default: 'sneaql.transforms'
+        }
+      ].each do |env_var|
+        @params[env_var[:sym]] = ENV[env_var[:var]] ? ENV[env_var[:var]] : env_var[:default]
+      end
+
+      # numeric parameter provided by env var should be casted
+      @params[:concurrency] = @params[:concurrency].to_i
+
+      # determine database type based jdbc url
+      # while technically any jdbc driver should work
+      # with sneaql, the database type allows for better
+      # handling of transactions, boolean, etc.
+      @params[:database] = Sneaql::Core.database_type(@params[:jdbc_url])
+    rescue => e
+      logger.error(e.message)
+      raise e
+    end
+
+    # creates a threadsafe queue with all active transforms
+    def build_transform_queue
+      # creates a queue to hold all the transform parameter hashes
+      @q = Queue.new
+
+      transforms = get_transforms
+      logger.info("#{transforms.length} transforms found in database...")
+
+      # push transforms on to queue
+      transforms.each do |t|
+        tmp = {}.merge(@params)
+        tmp[:transform_name] = t['transform_name']
+
+        # repo must be http or git https
+        raise 'malformed transform definition' unless t['sql_repository'] =~ /^http.*/i
+
+        tmp[:repo_url] = t['sql_repository']
+
+        # determine repo type based upon the the presence or absence of branch
+        # this comes from sql which is why the casting and strip
+        if t['sql_repository_branch'].to_s.strip == ''
+          tmp[:repo_type] = 'http'
+        else
+          tmp[:repo_type] = 'git'
+          tmp[:sql_repository_branch] = t['sql_repository_branch']
+        end
+
+        tmp[:compression] = 'zip' if tmp[:repo_url] =~ /.*\.zip$/
+
+        # only step manager option
+        tmp[:step_metadata_manager_type] = 'local_file'
+
+        # must be sneaql.json in the base of the sneaql repo
+        tmp[:step_metadata_file_path] = "#{@params[:repo_base_dir]}/#{tmp[:transform_name]}/sneaql.json"
+
+        @q.push tmp
+      end
+    rescue => e
+      logger.error(e.message)
+      e.backtrace.each { |b| logger.error(b) }
+    end
+
+    def get_transforms
+      # configure driver and db manager
+      configure_jdbc_driver
+      db_manager = Sneaql::Core.find_class(
+        :database,
+        @params[:database]
+      ).new
+
+      # connect and retrieve transform list
+      connection = create_connection
+
+      # fetch an array of active transforms
+      transforms = JDBCHelpers::QueryResultsToArray.new(
+        connection,
+        %(select
+            transform_name
+            ,sql_repository
+            ,sql_repository_branch
+          from
+            #{@params[:transform_table_name]}
+          where
+            is_active = #{db_manager.has_boolean ? 'true' : 1}
+          order by
+            transform_name;),
+        logger
+      ).results
+    ensure
+      connection.close
+      return transforms
+    end
+
+    # perform concurrent transform run
+    def run_transforms
+      # instantiate parallelize
+      ParallelizeSneaqlTransforms.new(
+        @q,
+        @params[:concurrency],
+        logger
+      )
+    end
+
+    # creates a jdbc connection based upon
+    # current driver context
+    # @return [JDBCHelpers::ConnectionFactory.connection]
+    def create_connection
+      JDBCHelpers::ConnectionFactory.new(
+        @params[:jdbc_url],
+        @params[:db_user],
+        @params[:db_pass],
+        logger
+      ).connection
+    end
+
+    # creates a database manager
+    # @return [Class]
+    def create_db_manager
+      Sneaql::Core.find_class(
+        :database,
+        @params[:database]
+      ).new
+    end
+
+    # configures the jdbc driver into the current context
+    def configure_jdbc_driver
+      j = Sneaql::JDBCDriverHandler.new(@params)
+      j.confirm_jdbc_driver
+      j.require_jdbc_driver
+    end
+  end
+end

data/lib/sneaql_standard_lib/jdbc_drivers.rb

@@ -0,0 +1,112 @@
+require 'digest'
+require 'open-uri'
+require 'aws-sdk'
+
+module Sneaql
+  # idempotent handling for jdbc driver
+  class JDBCDriverHandler
+    # exposed for unit tests
+    attr_accessor :confirmed_path
+    attr_accessor :target_path
+
+    # pulls down the jdbc driver and loads it
+    # param [Hash] params parameter hash
+    def initialize(params)
+      @params = params
+    end
+
+    # driver info must be provided
+    # jar file should be one of the following:
+    #   http store http://path/to/jarfile.jar
+    #   inside container file://path/to/jarfile.jar
+    #   s3 bucket s3://path/to/jarfile.jar requires aws credentials to be provided
+    # this method confirms the existence of the jdbc driver jar file
+    # if the file exists, no action is taken.  if file does not exist
+    # it is downloaded from the source location, either http or s3.
+    def confirm_jdbc_driver
+      @confirmed_path = nil
+      if @params[:jdbc_driver_jar] =~ /^http.*/i
+        @target_path = '/tmp/jdbc.jar'
+        @confirmed_path = File.exist?(@target_path) ? @target_path : download_driver_http
+      elsif @params[:jdbc_driver_jar] =~ /^file.*/i
+        @target_path = @params[:jdbc_driver_jar].gsub(/^file\:\/\//i, '')
+        @confirmed_path = @target_path if File.exist?(@target_path)
+      elsif @params[:jdbc_driver_jar] =~ /^s3.*/i
+        @target_path = '/tmp/jdbc.jar'
+        @confirmed_path = File.exist?(@target_path) ? @target_path : download_driver_s3
+      else raise 'no suitable driver provided'
+      end
+
+      # rubocop says to turn this into a guard statement
+      # but this needs the driver to be present before running
+      if @params[:jdbc_driver_jar_md5]
+        raise 'driver jar md5 mismatch' unless md5_check(
+          @confirmed_path,
+          @params[:jdbc_driver_jar_md5]
+        )
+      end
+    end
+
+    # downloads driver from an http source assuming no credentials
+    # need to be provided
+    def download_driver_http
+      File.write(
+        @target_path,
+        open(@params[:jdbc_driver_jar]).read
+      )
+    end
+
+    # downloads jar file from s3 source
+    # uses standard AWS environment variables
+    # or instance profile for credentials
+    def download_driver_s3
+      bucket_name = @params[:jdbc_driver_jar].match(
+        /^s3\:\/\/([a-zA-Z0-9]|\.|\-)+/i
+      )[0].gsub(/s3\:\/\//i, '')
+
+      object_key = @params[:jdbc_driver_jar].gsub(
+        /^s3\:\/\/([a-zA-Z0-9]|\.|\-)+\//i,
+        ''
+      )
+
+      aws_creds =
+      if ENV['AWS_ACCESS_KEY_ID']
+        Aws::Credentials.new(
+          ENV['AWS_ACCESS_KEY_ID'],
+          ENV['AWS_SECRET_ACCESS_KEY']
+        )
+      else
+        Aws::InstanceProfileCredentials.new
+      end
+
+      s3 = Aws::S3.new(
+        region: ENV['AWS_REGION'],
+        credentials: aws_creds
+      )
+
+      s3.get_object(
+        response_target: @target_path,
+        bucket: bucket_name,
+        key: object_key
+      )
+    end
+
+    # confirms that file md5 matches value provided
+    # @param [String] file_path path to file
+    # @param [String] file_md5 known md5 of file
+    # @return [Boolean]
+    def md5_check(file_path, file_md5)
+      m = Digest::MD5.file(file_path)
+      return true if m.hexdigest == file_md5
+      false
+    end
+
+    # requires the jar file and jdbc driver class
+    # into the current jruby context.  after this
+    # runs all jdbc connections will use this driver class.
+    def require_jdbc_driver
+      require @confirmed_path
+      java_import @params[:jdbc_driver_class]
+    end
+  end
+end

data/lib/sneaql_standard_lib/logging.rb

@@ -0,0 +1,19 @@
+require 'logger'
+
+module SneaqlStandard
+  Log = Logger.new(STDOUT)
+end
+
+def logger
+  SneaqlStandard::Log
+end
+
+# custom formatter provides logging with thread id and multi-line
+# entries each receiving their own log prefix
+logger.formatter = proc do |severity, datetime, _progname, msg|
+  t = ''
+  msg.to_s.split(/\n+/).each do |line|
+    t += "[#{severity}] #{datetime} tid#{Thread.current.object_id}: #{line}\n"
+  end
+  t
+end

data/lib/sneaql_standard_lib/parallelize.rb

@@ -0,0 +1,48 @@
+require 'thread'
+
+# used to run concurrent sneaql transforms
+# from a threadsafe queue.
+class ParallelizeSneaqlTransforms
+  # initialize object and run concurrent transforms.
+  # @param [Queue] queue_to_process queue of hashes with all params needed for transform
+  # @param [Fixnum] concurrency number of threads
+  # @param [Logger] logger optional logger object
+  def initialize(queue_to_process, concurrency, logger = nil)
+    @logger = logger ? logger : Logger.new(STDOUT)
+    @queue_to_process = queue_to_process
+    @concurrency = concurrency
+    parallelize
+  end
+
+  # performs the actual parallel execution
+  def parallelize
+    @logger.info(
+      "processing #{@queue_to_process} with a concurrency of #{@concurrency}..."
+    )
+
+    threads = []
+    @concurrency.times do
+      threads << Thread.new do
+        # loop until there are no more things to do
+        until @queue_to_process.empty?
+          begin
+            object_to_process = @queue_to_process.pop(true) rescue nil
+            # logger.debug(object_to_process)
+            t = Sneaql::Transform.new(
+              object_to_process,
+              @logger
+            )
+            t.run
+          rescue => e
+            @logger.error(e.message)
+            e.backtrace.each { |b| @logger.error(b) }
+          ensure
+            @logger.info("finished processing #{object_to_process['transform_name']}")
+          end
+        end
+      end
+    end
+    threads.each { |t| t.join }
+    threads = nil
+  end
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: sneaql-standard
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: java
+authors:
+- jeremy winters
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-01-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  name: sneaql
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+  name: minitest
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  name: aws-sdk
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  name: dotenv
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  name: thor
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+description: provides a cli and runtime environment for sneaql
+email: jeremy.winters@full360.com
+executables:
+- sneaql
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/sneaql
+- lib/sneaql_standard.rb
+- lib/sneaql_standard_lib/jdbc_drivers.rb
+- lib/sneaql_standard_lib/logging.rb
+- lib/sneaql_standard_lib/parallelize.rb
+homepage: https://www.full360.com
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.6
+signing_key:
+specification_version: 4
+summary: standard sneaql deployment
+test_files: []