sneaql 0.0.8-java

data/lib/sneaql_lib/repo_manager.rb

@@ -0,0 +1,95 @@
+require 'git'
+require 'open-uri'
+
+require_relative 'base.rb'
+
+module Sneaql
+  # Classes to manage repositories full of SneaQL code.
+  module RepoManagers
+    
+    # tells you the repo type based upon the url
+    # either git or http
+    # @param [String] repo_url
+    def self.repo_type_from_url(repo_url)
+      return 'git' if repo_url.match(/\.*git.*/i)
+      return 'http' if repo_url.match(/\.*http.*/i)
+    end
+    
+    
+    # pulls a branch from a remote git repo
+    class GitRepoManager < Sneaql::Core::RepoDownloadManager
+      Sneaql::Core::RegisterMappedClass.new(
+        :repo_manager,
+        'git',
+        Sneaql::RepoManagers::GitRepoManager
+      )
+
+      # Pulls down the repo and checks out the branch.
+      def manage_repo
+        drop_and_rebuild_directory(@repo_base_dir)
+        clone_repo(@params[:repo_url])
+        checkout_branch(@params[:git_branch])
+      end
+
+      # Clones a git repo to the local file system.
+      # @param [String] repo_uri
+      def clone_repo(repo_uri)
+        Dir.chdir(@repo_base_dir) do
+          @logger.info("cloning git repo #{repo_uri}")
+          @local_repo = Git.clone(repo_uri, @repo_base_dir)
+        end
+      end
+
+      # Checks out specified branch/commit from local repo.
+      # @param [String] branch
+      def checkout_branch(branch)
+        Dir.chdir(@repo_base_dir) do
+          @logger.info("checking out branch #{branch}")
+          @local_repo.checkout(branch.to_s)
+        end
+      end
+    end
+
+    # pulls single archive file from http/https source
+    class HttpStoreRepoManager < Sneaql::Core::RepoDownloadManager
+      Sneaql::Core::RegisterMappedClass.new(
+        :repo_manager,
+        'http',
+        Sneaql::RepoManagers::HttpStoreRepoManager
+      )
+
+      # Pulls a zip file from an HTTP store down to local file system
+      # then unzips it for use in transform.
+      def manage_repo
+        drop_and_rebuild_directory(@repo_base_dir)
+        pull_transform_repo_from_http_store(@params[:repo_url])
+        if @params[:compression] == 'zip'
+          unzip_file("#{@repo_base_dir}/#{File.basename(@params[:repo_url])}", @repo_base_dir)
+        end
+      end
+
+      # Pulls the transfrom to local file system.
+      # @param [String] repo_url
+      def pull_transform_repo_from_http_store(repo_url)
+        @logger.info "repo #{repo_url} to #{@repo_base_dir}/#{File.basename(repo_url)}"
+        File.write("#{@repo_base_dir}/#{File.basename(repo_url)}", open(repo_url).read)
+      end
+    end
+
+    # refers to repo in local file system
+    class LocalFileSystemRepoManager < Sneaql::Core::RepoDownloadManager
+      Sneaql::Core::RegisterMappedClass.new(
+        :repo_manager,
+        'local',
+        Sneaql::RepoManagers::LocalFileSystemRepoManager
+      )
+
+      # Repo is simply a local file system reference.
+      def manage_repo
+        # overrides the value created by the
+        # constructor with the directory provided
+        @repo_base_dir = @params[:repo_base_dir]
+      end
+    end
+  end
+end

data/lib/sneaql_lib/standard.rb

@@ -0,0 +1,30 @@
+module Sneaql
+  # Classes to manage DB objects for standard SneaQL deployment.
+  module Standard
+    # Manages execution of a standard transform
+    class StandardTransform
+      
+      attr_accessor {
+        :jdbc_url,
+        :db_user,
+        :db_pass
+      }
+      
+      def initialize()
+        # set parameters
+        
+        # determine database type
+        
+      end
+      
+      def execute_all_transforms()
+      
+      end
+      
+      def execute_single_transform(transform_name)
+        
+      end
+      
+    end
+  end
+end

data/lib/sneaql_lib/standard_db_objects.rb

@@ -0,0 +1,232 @@
+module Sneaql
+  # Classes to manage DB objects for standard SneaQL deployment.
+  module Standard
+    # Facilitate creation/recreation of database table objects
+    # for standard SneaQL deployment.
+    class DBObjectCreator
+      attr_accessor :jdbc_connection
+
+      # @param [Object] jdbc_connection
+      # @param [Sneaql::Core::DatabaseManager] database_manager
+      # @param [Logger] logger
+      def initialize(jdbc_connection, database_manager, logger = nil)
+        @logger = logger ? logger : Logger.new(STDOUT)
+        @jdbc_connection = jdbc_connection
+        @database_manager = database_manager
+      end
+
+      # Drops and recreates the primary transform table.
+      # @param [String] transform_table_name fully qualified name for this table
+      # @return [Boolean]
+      def recreate_transforms_table(transform_table_name)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          "drop table if exists #{ transform_table_name };",
+          @logger
+        )
+        create_transforms_table(transform_table_name)
+        return true
+      end
+
+      # Creates the primary transform table.
+      # @param [String] transform_table_name fully qualified name for this table
+      # @return [Boolean]
+      def create_transforms_table(transform_table_name)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          transforms_table_create_statement(transform_table_name),
+          @logger
+        )
+        return true
+      end
+      
+      # Coerces a boolean to the appropriate value for the database type.
+      # May return a 0 or 1 in RDBMS where boolean is not supported.
+      # @param [Boolean] boolean_value
+      # @return [Boolean, Fixnum]
+      def coerce_boolean(boolean_value)
+        if @database_manager.has_boolean 
+          boolean_value 
+        else 
+          boolean_value == true ? 1 : 0 
+        end
+      end
+      
+      # Create table statement for primary transform table.
+      # @param [String] transform_table_name fully qualified name for this table
+      # @return [String]
+      def transforms_table_create_statement(transform_table_name)
+        %{create table if not exists #{ transform_table_name }
+          (
+          	transform_name varchar(255) not null
+          	,sql_repository varchar(255)
+          	,sql_repository_branch varchar(255)
+          	,is_active #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
+          	,notify_on_success #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
+          	,notify_on_non_precondition_failure #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
+          	,notify_on_precondition_failure #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
+          	,updated_ts timestamp
+          );}
+      end
+      
+      # Creates a record in the transforms table.
+      # @param [String] transform_table_name
+      # @param [Hash] params Hash of parameters with symbols matching column names
+      def create_transform(transform_table_name, params)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          create_transform_statement(transform_table_name, params),
+          @logger
+        )
+      end
+
+      # @param [String] transform_table_name
+      # @param [Hash] params Hash of parameters with symbols matching column names
+      def create_transform_statement(transform_table_name, params)
+        %{insert into #{transform_table_name}
+          (
+            transform_name
+            ,sql_repository
+            ,sql_repository_branch
+            ,is_active
+            ,notify_on_success
+            ,notify_on_non_precondition_failure
+            ,notify_on_precondition_failure
+            ,updated_ts
+          )
+          values
+          (
+            '#{params[:transform_name]}'
+            ,'#{params[:sql_repository]}'
+            ,'#{params[:sql_repository_branch]}'
+            ,#{coerce_boolean(params[:is_active])}
+            ,#{coerce_boolean(params[:notify_on_success])}
+            ,#{coerce_boolean(params[:notify_on_non_precondition_failure])}
+            ,#{coerce_boolean(params[:notify_on_precondition_failure])}
+            ,current_timestamp
+          );}
+      end
+      
+      # Drops and recreates the transform steps table.
+      # @param [String] transform_steps_table_name fully qualified name for this table
+      # @return [Boolean]
+      def recreate_transform_steps_table(transform_steps_table_name)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          "drop table if exists #{ transform_steps_table_name };",
+          @logger
+        )
+        create_transform_steps_table(transform_steps_table_name)
+        return true
+      end
+
+      # Creates the transform steps table.
+      # @param [String] transform_steps_table_name fully qualified name for this table
+      # @return [Boolean]
+      def create_transform_steps_table(transform_steps_table_name)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          transform_steps_table_create_statement(transform_steps_table_name),
+          @logger
+        )
+        return true
+      end
+
+      # Create table statement for transform steps table.
+      # @param [String] transform_steps_table_name fully qualified name for this table
+      # @return [String]
+      def transform_steps_table_create_statement(transform_steps_table_name)
+        %{create table if not exists #{ transform_steps_table_name }
+          (
+          	transform_name varchar(255) not null
+          	,transform_step integer not null
+          	,sql_file_path_in_repo varchar(1024)
+          	,is_active #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
+          	,is_precondition #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
+          	,updated_ts timestamp
+          );}
+      end
+
+      # Drops and recreates the transform lock table.
+      # @param [String] transform_lock_table_name fully qualified name for this table
+      # @return [Boolean]
+      def recreate_transform_lock_table(transform_lock_table_name)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          "drop table if exists #{ transform_lock_table_name };",
+          @logger
+        )
+        create_transform_lock_table(transform_lock_table_name)
+        return true
+      end
+
+      # Creates the transform lock table.
+      # @param [String] transform_lock_table_name fully qualified name for this table
+      # @return [Boolean]
+      def create_transform_lock_table(transform_lock_table_name)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          transform_lock_table_create_statement(transform_lock_table_name),
+          @logger
+        )
+        return true
+      end
+
+      # Create table statement for transform lock table.
+      # @param [String] transform_lock_table_name fully qualified name for this table
+      # @return [String]
+      def transform_lock_table_create_statement(transform_lock_table_name)
+        %{create table if not exists #{ transform_lock_table_name }
+          (
+          	transform_lock_id bigint
+          	,transform_name varchar(255)
+          	,transform_lock_time timestamp
+          );}
+      end
+
+      # Drops and recreates the transform log table.
+      # @param [String] transform_log_table_name fully qualified name for this table
+      # @return [Boolean]
+      def recreate_transform_log_table(transform_log_table_name)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          "drop table if exists #{ transform_log_table_name };",
+          @logger
+        )
+        create_transform_log_table(transform_log_table_name)
+        return true
+      end
+
+      # Creates the transform log table.
+      # @param [String] transform_log_table_name fully qualified name for this table
+      # @return [Boolean]
+      def create_transform_log_table(transform_log_table_name)
+        JDBCHelpers::Execute.new(
+          @jdbc_connection,
+          transform_log_table_create_statement(transform_log_table_name),
+          @logger
+        )
+        return true
+      end
+
+      # Create table statement for transform log table.
+      # @param [String] transform_log_table_name fully qualified name for this table
+      # @return [String]
+      def transform_log_table_create_statement(transform_log_table_name)
+        %{create table if not exists #{ transform_log_table_name }
+          (
+          	transform_run_id bigint
+          	,transform_lock_id bigint
+          	,transform_name varchar(255)
+          	,transform_step integer
+          	,transform_statement integer
+          	,all_steps_complete #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
+          	,failed_in_precondition #{ if @database_manager.has_boolean then 'boolean' else 'smallint' end }
+          	,message varchar(65000)
+          	,transform_start_time timestamp
+          	,transform_end_time timestamp
+          );}
+      end
+    end
+  end
+end

data/lib/sneaql_lib/step_manager.rb

@@ -0,0 +1,60 @@
+require 'json'
+
+module Sneaql
+  # Managers for transform steps
+  module StepManagers
+    # source step metadata from a json file
+    class JSONFileStepManager < Sneaql::Core::StepMetadataManager
+      Sneaql::Core::RegisterMappedClass.new(
+        :step_metadata_manager,
+        'local_file',
+        Sneaql::StepManagers::JSONFileStepManager
+      )
+  
+      # Manages steps from a local JSON file.
+      def manage_steps
+        @steps = JSON.parse(File.read(@params[:step_metadata_file_path])).sort_by! { |h| h['step_number'] }
+        @steps.map! { |j| { step_number: j['step_number'], step_file: j['step_file'] } }
+      end
+    end
+  
+    # source step metadata from a standardized table in the target database
+    class TransformStepTableManager < Sneaql::Core::StepMetadataManager
+      Sneaql::Core::RegisterMappedClass.new(
+        :step_metadata_manager,
+        'transform_steps_table',
+        Sneaql::StepManagers::TransformStepTableManager
+      )
+      
+      # Manages steps based in a standardized table.
+      def manage_steps
+        jdbc_connection = JDBCHelpers::ConnectionFactory.new(
+          @params[:jdbc_url],
+          @params[:db_user],
+          @params[:db_pass]
+        ).connection
+  
+        @steps = JDBCHelpers::QueryResultsToArray.new(
+          jdbc_connection,
+          %(select
+            transform_step
+            ,sql_file_path_in_repo
+          from
+            #{@params[:transform_steps_table]}
+          where
+            transform_name='#{@params[:transform_name]}'
+            and
+            is_active=#{ if @params[:database_manager].has_boolean then 'true' else 1 end }
+          order by
+            transform_step asc;)
+        ).results
+  
+        @steps.map! do |s| 
+          { step_number: s['transform_step'], step_file: s['sql_file_path_in_repo'] }
+        end
+  
+        jdbc_connection.close
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: sneaql
+version: !ruby/object:Gem::Version
+  version: 0.0.8
+platform: java
+authors:
+- jeremy winters
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-03-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: logger
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+  name: minitest
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: git
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  name: jdbc_helpers
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  name: zip
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: provides the base classes required to run and extend sneaql
+email: jeremy.winters@full360.com
+executables:
+- sneaql
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/sneaql
+- lib/sneaql.rb
+- lib/sneaql_lib/base.rb
+- lib/sneaql_lib/core.rb
+- lib/sneaql_lib/database_manager.rb
+- lib/sneaql_lib/database_prefs/redshift.rb
+- lib/sneaql_lib/database_prefs/sqlite.rb
+- lib/sneaql_lib/database_prefs/vertica.rb
+- lib/sneaql_lib/exceptions.rb
+- lib/sneaql_lib/expressions.rb
+- lib/sneaql_lib/lock_manager.rb
+- lib/sneaql_lib/parser.rb
+- lib/sneaql_lib/recordset.rb
+- lib/sneaql_lib/repo_manager.rb
+- lib/sneaql_lib/standard.rb
+- lib/sneaql_lib/standard_db_objects.rb
+- lib/sneaql_lib/step_manager.rb
+homepage: https://www.full360.com
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.6
+signing_key:
+specification_version: 4
+summary: sneaql language core
+test_files: []