batch-kit 0.3

data/lib/batch-kit/database/schema.rb

@@ -0,0 +1,229 @@
+require 'sequel'
+require_relative '../logging'
+
+
+class BatchKit
+
+    class Database
+
+        # Manages the database tables and connection used to record batch-kit
+        # events.
+        class Schema
+
+            attr_reader :log
+
+
+            # Create a batch-kit schema instance
+            #
+            # @param options [Hash] An options hash.
+            # @option options [Symbol] :log_level The level of database messages
+            #   that should be visible (default is :error).
+            def initialize(options = {})
+                @log = BatchKit::LogManager.logger('batch-kit.database.schema')
+                @log.level = options.fetch(:log_level, :error)
+            end
+
+
+            # Returns the Sequel database connection.
+            def connection
+                @conn
+            end
+
+
+            # Connects to the database determined by +args+.
+            #
+            # @see Sequel#connect for details on the different arguments that
+            #   are required to connect to your database.
+            def connect(*args)
+                Sequel.default_timezone = :utc
+                @conn = Sequel.connect(*args)
+                @conn.loggers << @log
+                @conn.autosequence = true
+
+                create_tables unless deployed?
+            end
+
+
+            # @return true if the batch-kit database tables have been deployed.
+            def deployed?
+                @conn.table_exists?(:batch_md5)
+            end
+
+
+            # Drops all database tables used for tracking batch-kit processes.
+            def drop_tables
+                @conn.drop_table(:batch_requestor)
+                @conn.drop_table(:batch_request)
+                @conn.drop_table(:batch_lock)
+                @conn.drop_table(:batch_task_run)
+                @conn.drop_table(:batch_task)
+                @conn.drop_table(:batch_job_run_failure)
+                @conn.drop_table(:batch_job_run_log)
+                @conn.drop_table(:batch_job_run_arg)
+                @conn.drop_table(:batch_job_run)
+                @conn.drop_table(:batch_job)
+                @conn.drop_table(:batch_md5)
+            end
+
+
+            # Creates all database tables needed for tracking batch-kit processes.
+            def create_tables
+                # MD5 table, used to hold hashes of objects to detect version changes
+                @conn.create_table?(:batch_md5) do
+                    primary_key :md5_id
+                    String :object_type, size: 30, null: false
+                    String :object_name, size: 255, null: false
+                    Fixnum :object_version, null: false
+                    String :md5_digest, size: 32, null: false
+                    DateTime :md5_created_at, null: false
+                    unique [:object_type, :object_name, :object_version]
+                end
+
+                # Job table, holds details of job definitions
+                @conn.create_table?(:batch_job) do
+                    primary_key :job_id
+                    String :job_name, size: 80, null: false
+                    String :job_class, size: 80, null: false
+                    String :job_method, size: 80, null: false
+                    String :job_desc, size: 255, null: true
+                    String :job_host, size: 50, null: false
+                    String :job_file, size: 255, null: false
+                    Fixnum :job_version, null: false
+                    foreign_key :job_file_md5_id, :batch_md5, null: false
+                    Fixnum :job_run_count, null: false
+                    Fixnum :job_success_count, null: false
+                    Fixnum :job_fail_count, null: false
+                    Fixnum :job_abort_count, null: false
+                    Bignum :job_min_success_duration_ms, null: false
+                    Bignum :job_max_success_duration_ms, null: false
+                    Bignum :job_mean_success_duration_ms, null: false
+                    Bignum :job_m2_success_duration_ms, null: false
+                    DateTime :job_created_at, null: false
+                    DateTime :job_modified_at, null: false
+                    DateTime :job_last_run_at, null: true
+                    unique [:job_host, :job_name]
+                end
+
+                # Task table, holds details of task definitions
+                @conn.create_table?(:batch_task) do
+                    primary_key :task_id
+                    foreign_key :job_id, :batch_job, null: false
+                    Fixnum :job_version, null: false
+                    String :task_name, size: 80, null: false
+                    String :task_class, size: 80, null: false
+                    String :task_method, size: 80, null: false
+                    String :task_desc, size: 255, null: true
+                    TrueClass :task_current_flag, null: false, default: true
+                    Fixnum :task_run_count, null: false
+                    Fixnum :task_success_count, null: false
+                    Fixnum :task_fail_count, null: false
+                    Fixnum :task_abort_count, null: false
+                    Bignum :task_min_success_duration_ms, null: false
+                    Bignum :task_max_success_duration_ms, null: false
+                    Bignum :task_mean_success_duration_ms, null: false
+                    Bignum :task_m2_success_duration_ms, null: false
+                    DateTime :task_created_at, null: false
+                    DateTime :task_modified_at, null: false
+                    DateTime :task_last_run_at, null: true
+                end
+
+                # Job run table, holds details of a single execution of a job
+                @conn.create_table?(:batch_job_run) do
+                    primary_key :job_run
+                    foreign_key :job_id, :batch_job, null: false
+                    String :job_instance, size: 80, null: true
+                    Fixnum :job_version, null: false
+                    String :job_run_by, size: 50, null: false
+                    String :job_cmd_line, size: 2000, null: true
+                    DateTime :job_start_time, null: false
+                    DateTime :job_end_time, null: true
+                    String :job_status, size: 12, null: false
+                    Fixnum :job_pid, null: true
+                    Fixnum :job_exit_code, null: true
+                    TrueClass :job_purged_flag, null: false, default: false
+                end
+
+                # Job run arguments table, holds details of the arguments used on a job
+                @conn.create_table?(:batch_job_run_arg) do
+                    foreign_key :job_run, :batch_job_run
+                    String :job_arg_name, size: 50, null: false
+                    String :job_arg_value, size: 255, null: true
+                    primary_key [:job_run, :job_arg_name]
+                end
+
+                # Job run log table, holds log records for a job
+                @conn.create_table?(:batch_job_run_log) do
+                    foreign_key :job_run, :batch_job_run
+                    Fixnum :log_line, null: false
+                    DateTime :log_time, null: false
+                    String :log_name, size: 40, null: false
+                    String :log_level, size: 8, null: false
+                    String :thread_id, size: 8, null: true
+                    String :log_message, size: 1000, null: false
+                    primary_key [:job_run, :log_line]
+                end
+
+                # Job failure table, holds exception details for job failures
+                @conn.create_table?(:batch_job_run_failure) do
+                    # We don't use an FK here, because we want to be able to retain
+                    # failure details longer than we retain job runs
+                    Fixnum :job_run, null: false
+                    foreign_key :job_id, :batch_job, null: false
+                    Fixnum :job_version, null: false
+                    DateTime :job_failed_at, null: false
+                    String :exception_message, size: 500, null: false
+                    String :exception_backtrace, size: 4000, null: false
+                end
+
+                # Task run table, holds details of a single execution of a task
+                @conn.create_table?(:batch_task_run) do
+                    primary_key :task_run
+                    foreign_key :task_id, :batch_task, null: false
+                    foreign_key :job_run, :batch_job_run, null: false
+                    String :task_instance, size: 80, null: true
+                    DateTime :task_start_time, null: false
+                    DateTime :task_end_time, null: true
+                    String :task_status, size: 12, null: false
+                    Fixnum :task_exit_code, null: true
+                end
+
+                # Lock table, holds details of the current locks
+                @conn.create_table?(:batch_lock) do
+                    String :lock_name, type: String, size: 50, unique_key: true
+                    foreign_key :job_run, :batch_job_run
+                    DateTime :lock_created_at, null: false
+                    DateTime :lock_expires_at, null: false
+                end
+
+                # Request table, holds details of job run requests
+                @conn.create_table?(:batch_request) do
+                    primary_key :request_id
+                    String :request_type, size: 80, null: false
+                    String :request_label, size: 80, null: false
+                    String :request_source, size: 80, null: false
+                    String :job_host, size: 30, null: false
+                    String :job_file, size: 255, null: false
+                    String :job_args, size: 2000, null: true
+                    String :job_working_dir, size: 255, null: true
+                    TrueClass :processed_flag, default: false, null: false
+                    foreign_key :job_run, :batch_job_run, null: true
+                    DateTime :request_created_at, null: false
+                    DateTime :request_start_at, null: true
+                    DateTime :request_launched_at, null: true
+                end
+
+                # Requestor table, holds details of job run requestors
+                @conn.create_table?(:batch_requestor) do
+                    foreign_key :request_id, :batch_request, null: false
+                    DateTime :requested_at, null: false
+                    String :requested_by, size: 80, null: false
+                    String :email_address, size: 100, null: true
+                end
+            end
+
+        end
+
+    end
+
+end
+

data/lib/batch-kit/encryption.rb

@@ -0,0 +1,7 @@
+if RUBY_PLATFORM == 'java'
+    # JRuby OpenSSL support does not include the PKCS5 module, so we use the
+    # built-in Java crypto classes instead
+    require_relative 'encryption/java_encryption'
+else
+    require_relative 'encryption/ruby_encryption'
+end

data/lib/batch-kit/encryption/java_encryption.rb

@@ -0,0 +1,178 @@
+class BatchKit
+
+    # Performs password-based encryption (PBE) and decryption using the AES 128-bit
+    # cipher algorithm. Password-based encryption uses a password or passphrase
+    # as the shared secret key with which to encrypt/decrypt other text.
+    #
+    # As with all key-based encryption schemes, the encrypted values are only as
+    # secure as the key used to encrypt them. If the key is accessible to a
+    # hacker, any values encrypted with the key can be decrypted, so the key
+    # should be stored separately from the encrypted values, and be kept as
+    # secure as possible.
+    module Encryption
+
+        include_package 'java.security'
+        include_package 'javax.crypto'
+        include_package 'javax.crypto.spec'
+
+        # Use PBKDF2 with SHA-1 as the key hashing algorithm
+        # The NIST specifically names SHA1 as an acceptable hashing algorithm
+        # for PKBDF2
+        KEY_ALGORITHM = 'PBKDF2WithHmacSHA1'
+
+        # Iteration count; NIST recommends at least 1000 iterations
+        KEY_ITERATION_COUNT = 5000
+
+        # Length of the generated key in bits
+        KEY_DERIVED_LENGTH = 128
+
+        # Algorithm to use for encryption
+        CIPHER_ALGORITHM = 'AES/CBC/PKCS5Padding'
+
+        # Salt; a random 8 bytes used to generate an encryption key from a key
+        # password / passphrase.
+        # @note The same value *must* be used for the salt when converting a key
+        # password/passphrase into an encryption key for both encryptionn and
+        # decryption.
+        SALT = [70, 211, 28, 57, 192, 6, 78, 163].pack("CCCCCCCC").to_java_bytes
+
+
+        # Generate a random master key that can be used to encrypt/decrypt other
+        # sensitive data such as passwords. The master key must be stored some
+        # place separate from the values it is used to encrypt.
+        #
+        # @return [String] A random string of text that can be used as a master
+        #   key for encrypting/decrypting other values.
+        def generate_master_key()
+            java.util.UUID.randomUUID().toString()
+        end
+        module_function :generate_master_key
+
+
+        # Encrypt the supplied +clear_text+, using +key_text+ as the pass-phrase.
+        #
+        # @param key_text [String] The clear-text pass-phrase to use as the key
+        #   for encryption.
+        # @param clear_text [String] The cleartext string to be encrypted.
+        # @param salt [Array<Byte>] An 8-byte array of random values to use as
+        #   the salt.
+        # @return [String] A base-64 encoded string representing the encrypted
+        #   +clear_text+ value.
+        def encrypt(key_text, clear_text, salt = SALT)
+            key = generate_key(key_text, salt)
+            encipher(key, clear_text)
+        end
+        module_function :encrypt
+
+
+        # Encipher the supplied +clear_text+, using +key+ as the encryption key.
+        #
+        # Note: this method is possibly less secure than #encrypt, since it uses
+        # the actual key in the enciphering, rather than an SHA1 hash of the key.
+        #
+        # @param key [SecretKeySpec] The key to use for encryption.
+        # @param clear_text [String] The cleartext string to be encrypted.
+        # @param cipher_algorithm [String] The name of the cipher algorithm to
+        #   use when encrypting the clear_text.
+        # @return [String] A base-64 encoded string representing the encrypted
+        #   +clear_text+ value.
+        def encipher(key, clear_text, cipher_algorithm = CIPHER_ALGORITHM)
+            cipher = Cipher.getInstance(cipher_algorithm)
+            cipher.init(Cipher::ENCRYPT_MODE, key)
+            params = cipher.getParameters()
+            iv = params.getParameterSpec(IvParameterSpec.java_class).getIV()
+            cipher_bytes = cipher.doFinal(clear_text.to_java_bytes)
+            # Combine IV and cipher bytes, and base-64 encode
+            buffer_bytes = Java::byte[KEY_DERIVED_LENGTH / 8 + cipher_bytes.length].new
+            buffer = java.nio.ByteBuffer.wrap(buffer_bytes)
+            buffer.put(iv)
+            buffer.put(cipher_bytes)
+            base64_encode(buffer_bytes)
+        end
+        module_function :encipher
+
+
+        # Decrypt the supplied +cipher_text+, using +key_text+ as the pass-phrase.
+        #
+        # @param key_text [String] The clear-text pass-phrase to use as the key
+        #   for decryption.
+        # @param cipher_text [String] A base-64 encoded cipher text string that
+        #   is to be decrypted.
+        # @param salt [Array<Byte>] An 8-byte array of random values to use as
+        #   the salt. Like the +key_text+, it is imperative that the same value
+        #   is used for the salt when decrypting a previously encrypted value.
+        # @return [String] The clear text that was encrypted.
+        def decrypt(key_text, cipher_text, salt = SALT)
+            key = generate_key(key_text, salt)
+            decipher(key, cipher_text)
+        end
+        module_function :decrypt
+
+
+        # Decipher the supplited +cipher_text+, using +key+ as the decipher key.
+        #
+        # @param key [SecretKeySpec] The key used for encryption.
+        # @param cipher_text [String] A base-64 encoded cipher text string that
+        #   is to be decrypted.
+        # @param cipher_algorithm [String] The name of the cipher algorithm used
+        #   to encrypt the clear_text.
+        # @return [String] The clear text that was encrypted.
+        def decipher(key, cipher_text, cipher_algorithm = CIPHER_ALGORITHM)
+            cipher = Cipher.getInstance(cipher_algorithm)
+            buffer_bytes = base64_decode(cipher_text)
+            # Unpack IV and cipher bytes
+            iv_bytes = Java::byte[KEY_DERIVED_LENGTH / 8].new
+            cipher_bytes = Java::byte[buffer_bytes.length - KEY_DERIVED_LENGTH / 8].new
+            buffer = java.nio.ByteBuffer.wrap(buffer_bytes)
+            buffer.get(iv_bytes)
+            buffer.get(cipher_bytes)
+            cipher.init(Cipher::DECRYPT_MODE, key, IvParameterSpec.new(iv_bytes))
+            String.from_java_bytes(cipher.doFinal(cipher_bytes))
+        end
+        module_function :decipher
+
+
+        # Convert a byte array to a base-64 encoded string representation.
+        #
+        # @param bytes [String, Array<byte>] A String or Java byte-array to be
+        #   encoded.
+        # @return [String] A base-64 encoded String.
+        def base64_encode(bytes)
+            bytes = bytes.to_java_bytes if bytes.is_a?(String)
+            javax.xml.bind.DatatypeConverter.printBase64Binary(bytes)
+        end
+        module_function :base64_encode
+
+
+        # Convert a base-64 encoded string to a byte array.
+        #
+        # @param str [String] A base-64 encoded String.
+        # @return [Array<byte>] A Java byte-array containing the decoded bytes.
+        def base64_decode(str)
+            javax.xml.bind.DatatypeConverter.parseBase64Binary(str)
+        end
+        module_function :base64_decode
+
+
+        private
+
+        # Generates the key to be used for encryption/decryption, based on
+        # +key_text+ and +salt+.
+        #
+        # @param key_text [String] A clear-text password or pass phrase that is
+        #   to be used to derive the encryption key.
+        # @param salt [Array<Byte>] An 8-byte array of random values to use as
+        #   the salt.
+        # @return [SecretKey] A key that can be used for encryption/decryption.
+        def generate_key(key_text, salt)
+            factory = SecretKeyFactory.getInstance(KEY_ALGORITHM)
+            key_spec = PBEKeySpec.new(key_text.to_s.to_java.toCharArray(), salt,
+                                      KEY_ITERATION_COUNT, KEY_DERIVED_LENGTH)
+            key = factory.generateSecret(key_spec)
+            SecretKeySpec.new(key.getEncoded(), CIPHER_ALGORITHM.split('/').first)
+        end
+        module_function :generate_key
+
+    end
+
+end

data/lib/batch-kit/encryption/ruby_encryption.rb

@@ -0,0 +1,175 @@
+require 'openssl'
+require 'digest/sha1'
+require 'base64'
+
+
+class BatchKit
+
+    # Performs password-based encryption (PBE) and decryption using the AES 128-bit
+    # cipher algorithm. Password-based encryption uses a password or passphrase
+    # as the shared secret key with which to encrypt/decrypt other text.
+    #
+    # As with all key-based encryption schemes, the encrypted values are only as
+    # secure as the key used to encrypt them. If the key is accessible to a
+    # hacker, any values encrypted with the key can be decrypted, so the key
+    # should be stored separately from the encrypted values, and be kept as
+    # secure as possible.
+    module Encryption
+
+        # Use PBKDF2 with SHA-1 as the key hashing algorithm
+        # The NIST specifically names SHA1 as an acceptable hashing algorithm
+        # for PKBDF2
+        KEY_ALGORITHM = :pbkdf2_hmac_sha1
+
+        # Iteration count; NIST recommends at least 1000 iterations
+        KEY_ITERATION_COUNT = 5000
+
+        # Length of the generated key in bits
+        KEY_DERIVED_LENGTH = 128
+
+        # Algorithm to use for encryption
+        CIPHER_ALGORITHM = 'AES-128-CBC'
+
+        # Salt; a random 8 bytes used to generate an encryption key from a key
+        # password / passphrase.
+        #
+        # @note The same value *must* be used for the salt when converting a key
+        # password/passphrase into an encryption key for both encryptionn and
+        # decryption.
+        SALT = [70, 211, 28, 57, 192, 6, 78, 163].pack("CCCCCCCC")
+
+
+        # Generate a random master key that can be used to encrypt/decrypt other
+        # sensitive data such as passwords. The master key must be stored some
+        # place separate from the values it is used to encrypt.
+        #
+        # @return [String] A random string of text that can be used as a master
+        #   key for encrypting/decrypting other values.
+        def generate_master_key()
+            require 'securerandom'
+            SecureRandom.uuid
+        end
+        module_function :generate_master_key
+
+
+        # Encrypt the supplied +clear_text+, using +key_text+ as the pass-phrase.
+        #
+        # @param key_text [String] The clear-text pass-phrase to use as the key
+        #   for encryption.
+        # @param clear_text [String] The cleartext string to be encrypted.
+        # @param salt [Array<Byte>] An 8-byte array of random values to use as
+        #   the salt.
+        # @return [String] A base-64 encoded string representing the encrypted
+        #   +clear_text+ value.
+        def encrypt(key_text, clear_text, salt = SALT)
+            key = generate_key(key_text, salt)
+            encipher(key, clear_text)
+        end
+        module_function :encrypt
+
+
+        # Encipher the supplied +clear_text+, using +key+ as the encryption key.
+        #
+        # Note: this method is possibly less secure than #encrypt, since it uses
+        # the actual key in the enciphering, rather than an SHA1 hash of the key.
+        #
+        # @param key [String] The key to use for encryption.
+        # @param clear_text [String] The cleartext string to be encrypted.
+        # @param cipher_algorithm [String] The name of the cipher algorithm to
+        #   use when encrypting the clear_text.
+        # @return [String] A base-64 encoded string representing the encrypted
+        #   +clear_text+ value.
+        def encipher(key, clear_text, cipher_algorithm = CIPHER_ALGORITHM)
+            cipher = OpenSSL::Cipher.new(cipher_algorithm)
+            cipher.encrypt
+            cipher.key = key
+
+            iv = cipher.random_iv
+            cipher_bytes = cipher.update(clear_text) + cipher.final
+
+            # Combine IV and cipher bytes, and base-64 encode
+            base64_encode([iv + cipher_bytes])
+        end
+        module_function :encipher
+
+
+        # Decrypt the supplied +cipher_text+, using +key_text+ as the pass-phrase.
+        #
+        # @param key_text [String] The clear-text pass-phrase to use as the key
+        #   for decryption.
+        # @param cipher_text [String] A base-64 encoded cipher text string that
+        #   is to be decrypted.
+        # @param salt [Array<Byte>] An 8-byte array of random values to use as
+        #   the salt. Like the +key_text+, it is imperative that the same value
+        #   is used for the salt when decrypting a previously encrypted value.
+        # @return [String] The clear text that was encrypted.
+        def decrypt(key_text, cipher_text, salt = SALT)
+            key = generate_key(key_text, salt)
+            decipher(key, cipher_text)
+        end
+        module_function :decrypt
+
+
+        # Decipher the supplited +cipher_text+, using +key+ as the decipher key.
+        #
+        # @param key [String] The key used for encryption.
+        # @param cipher_text [String] A base-64 encoded cipher text string that
+        #   is to be decrypted.
+        # @param cipher_algorithm [String] The name of the cipher algorithm used
+        #   to encrypt the clear_text.
+        # @return [String] The clear text that was encrypted.
+        def decipher(key, cipher_text, cipher_algorithm = CIPHER_ALGORITHM)
+            cipher = OpenSSL::Cipher.new(cipher_algorithm)
+            cipher.decrypt
+            cipher.key = key
+
+            # Base-64 decode, and unpack IV and cipher from cipher_text
+            iv_cipher = base64_decode(cipher_text)
+            iv_len = KEY_DERIVED_LENGTH / 8
+            cipher.iv = iv_cipher[0...iv_len]
+            cipher_bytes = iv_cipher[iv_len..-1]
+            cipher.update(cipher_bytes) + cipher.final
+        end
+        module_function :decipher
+
+
+        # Convert a byte array to a base-64 encoded string representation.
+        #
+        # @param bytes [String, Array<String>] A String or Java byte-array to be
+        #   encoded.
+        # @return [String] A base-64 encoded String.
+        def base64_encode(bytes)
+            bytes = [bytes] if bytes.is_a?(String)
+            bytes.pack('m0').chomp
+        end
+        module_function :base64_encode
+
+
+        # Convert a base-64 encoded string to a byte array.
+        #
+        # @param str [String] A base-64 encoded String.
+        # @return [Array<byte>] A String containing the decoded bytes.
+        def base64_decode(str)
+            str.unpack('m0').first
+        end
+        module_function :base64_decode
+
+
+        private
+
+        # Generates the key to be used for encryption/decryption, based on
+        # +key_text+ and +salt+.
+        #
+        # @param key_text [String] A clear-text password or pass phrase that is
+        #   to be used to derive the encryption key.
+        # @param salt [String] A String containing a random salt.
+        # @return [SecretKey] A key that can be used for encryption/decryption.
+        def generate_key(key_text, salt)
+            OpenSSL::PKCS5.send(KEY_ALGORITHM, key_text, salt,
+                                KEY_ITERATION_COUNT, KEY_DERIVED_LENGTH)
+        end
+        module_function :generate_key
+
+    end
+
+end