egis 1.1.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d232fad45575196b2a8a2a65ccca79529a2fd72121bf5883a9c3b037a6c40850
-  data.tar.gz: d269d52126bf32592f3b3425abc76b1664b119684674db5e618d9cbacbc0d3f3
+  metadata.gz: 924727f52ac809df7ab4457a6a0c05469cd6c128adaff339018e964dc9ff4ea0
+  data.tar.gz: c8ba9d090641c899e832b73b062314bd76fa4fb39041fb175d7e3a717e943856
 SHA512:
-  metadata.gz: 37c73df4ff2e1bf1f280816f0a882046a02447932bfdb8c77893b27d2e106f263d22a0d7cd0951861898aef882b2ea9ddca737ef980b9bfc2e342609a2716845
-  data.tar.gz: 93e325830f6382c683430e7dac987b01dc9bc93890155f0c28f19373742610d8f24195e7c074684f813821f508a956a7ad936c7e4297a0738af52ec285445805
+  metadata.gz: fa429f550a68475a75f9dd9c0610b0d41c9a8017f0e46c0f9e467cfaeddfb7d6587d6e7df5b4440bbd75e4d4d9a12d5966f40e4ce72cb8abf70e5dc2897f91d8
+  data.tar.gz: 3beac29ce82c267bfc2621e730a8b631f76a94f7426ac7d7901b31d204cda3320d1ab934e9c3e156ee52e5fca315728412ffd5a007bf4623dd2ac077d85399ba

data/egis.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = 'https://github.com/u2i/egis/blob/master/docs/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://u2i.github.io/egis/file.CHANGELOG.html'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

data/lib/egis.rb

@@ -40,13 +40,18 @@ require 'egis/s3_location_parser'
 #     config.aws_profile = 'my-profile'
 #   end
 #
+# @example Configure logger
+#   Egis.configure do |config|
+#     config.logger = Logger.new('athena.log', level: :debug)
+#   end
+#
 # @yield [Egis::Configuration]
 # @return [void]
 #
 module Egis
   class << self
-    def configure
-      yield(configuration)
+    def configure(&block)
+      configuration.configure(&block)
     end
 
     # @!visibility private
@@ -58,5 +63,10 @@ module Egis
     def mode
       @mode ||= Egis::StandardMode.new
     end
+
+    # @!visibility private
+    def logger
+      @configuration.logger
+    end
   end
 end

data/lib/egis/aws_client_provider.rb

@@ -6,25 +6,23 @@ require 'aws-sdk-athena'
 module Egis
   # @!visibility private
   class AwsClientProvider
-    def s3_client
-      Aws::S3::Client.new(client_config)
+    def s3_client(configuration)
+      Aws::S3::Client.new(client_config(configuration))
     end
 
-    def athena_client
-      Aws::Athena::Client.new(client_config)
+    def athena_client(configuration)
+      Aws::Athena::Client.new(client_config(configuration))
     end
 
     private
 
-    def client_config
-      configuration = Egis.configuration
-
-      config = {}
-      config[:region] = configuration.aws_region if configuration.aws_region
-      config[:access_key_id] = configuration.aws_access_key_id if configuration.aws_access_key_id
-      config[:secret_access_key] = configuration.aws_secret_access_key if configuration.aws_secret_access_key
-      config[:profile] = configuration.aws_profile if configuration.aws_profile
-      config
+    def client_config(configuration)
+      {
+        region: configuration.aws_region,
+        access_key_id: configuration.aws_access_key_id,
+        secret_access_key: configuration.aws_secret_access_key,
+        profile: configuration.aws_profile
+      }.compact
     end
   end
 end

data/lib/egis/client.rb

@@ -4,6 +4,9 @@ module Egis
   ##
   # The most fundamental {Egis} class. Provides an interface for executing Athena queries.
   #
+  # @yieldparam config [Egis::Configuration] Egis configuration block, if missing Egis will use global configuration
+  #   provided by {Egis.configure}
+  #
   # See configuration instructions {Egis.configure}.
   #
   # @see Egis.configure
@@ -33,14 +36,17 @@ module Egis
       'CANCELLED' => Egis::QueryStatus::CANCELLED
     }.freeze
 
-    DEFAULT_QUERY_STATUS_BACKOFF = ->(attempt) { 1.5**attempt - 1 }
+    private_constant :QUERY_STATUS_MAPPING
 
-    private_constant :QUERY_STATUS_MAPPING, :DEFAULT_QUERY_STATUS_BACKOFF
+    attr_reader :aws_s3_client
 
-    def initialize(aws_client_provider: Egis::AwsClientProvider.new, s3_location_parser: Egis::S3LocationParser.new)
-      @aws_athena_client = aws_client_provider.athena_client
+    def initialize(aws_client_provider: Egis::AwsClientProvider.new,
+                   s3_location_parser: Egis::S3LocationParser.new,
+                   &block)
+      @configuration = block_given? ? Egis.configuration.dup.configure(&block) : Egis.configuration
+      @aws_athena_client = aws_client_provider.athena_client(configuration)
+      @aws_s3_client = aws_client_provider.s3_client(configuration)
       @s3_location_parser = s3_location_parser
-      @query_status_backoff = Egis.configuration.query_status_backoff || DEFAULT_QUERY_STATUS_BACKOFF
     end
 
     ##
@@ -65,14 +71,16 @@ module Egis
     #   by workgroup.
     # @return [Egis::QueryStatus]
 
-    def execute_query(query, work_group: nil, database: nil, output_location: nil, async: true)
-      query_execution_id = aws_athena_client.start_query_execution(
+    def execute_query(query, work_group: nil, database: nil, output_location: nil, async: true, system_execution: false)
+      query_id = aws_athena_client.start_query_execution(
         query_execution_params(query, work_group, database, output_location)
       ).query_execution_id
 
-      return query_status(query_execution_id) if Egis.mode.async(async)
+      log_query_execution(query, query_id, system_execution)
+
+      return query_status(query_id) if Egis.mode.async(async)
 
-      query_status = wait_for_query_to_finish(query_execution_id)
+      query_status = wait_for_query_to_finish(query_id)
 
       raise Egis::Errors::QueryExecutionError, query_status.message unless query_status.finished?
 
@@ -89,21 +97,25 @@ module Egis
       resp = aws_athena_client.get_query_execution(query_execution_id: query_id)
 
       query_execution = resp.query_execution
+      query_status = query_execution.status.state
+
+      Egis.logger.debug { "Checking query status (#{query_id}): #{query_status}" }
 
       Egis::QueryStatus.new(
         query_execution.query_execution_id,
-        QUERY_STATUS_MAPPING.fetch(query_execution.status.state),
+        QUERY_STATUS_MAPPING.fetch(query_status),
         query_execution.status.state_change_reason,
-        parse_output_location(query_execution)
+        parse_output_location(query_execution),
+        client: self
       )
     end
 
     private
 
-    attr_reader :aws_athena_client, :s3_location_parser, :query_status_backoff
+    attr_reader :configuration, :aws_athena_client, :s3_location_parser
 
     def query_execution_params(query, work_group, database, output_location)
-      work_group_params = work_group || Egis.configuration.work_group
+      work_group_params = work_group || configuration.work_group
 
       params = {query_string: query}
       params[:work_group] = work_group_params if work_group_params
@@ -112,11 +124,20 @@ module Egis
       params
     end
 
-    def wait_for_query_to_finish(query_execution_id)
+    def log_query_execution(query, query_id, system_execution)
+      if system_execution
+        Egis.logger.debug { "Executing system query (#{query_id}): #{query.gsub(/\s+/, ' ')}" }
+      else
+        Egis.logger.info { "Executing query (#{query_id}): #{query.gsub(/\s+/, ' ')}" }
+      end
+    end
+
+    def wait_for_query_to_finish(query_id)
       attempt = 1
       loop do
-        sleep(query_status_backoff.call(attempt))
-        status = query_status(query_execution_id)
+        sleep(configuration.query_status_backoff.call(attempt))
+        status = query_status(query_id)
+
         return status unless status.queued? || status.running?
 
         attempt += 1

data/lib/egis/configuration.rb

@@ -4,6 +4,16 @@ module Egis
   # @!visibility private
   class Configuration
     attr_accessor :work_group, :aws_region, :aws_access_key_id, :aws_secret_access_key, :aws_profile,
-                  :query_status_backoff, :testing_s3_bucket
+                  :query_status_backoff, :testing_s3_bucket, :logger
+
+    def initialize
+      @logger = Logger.new(STDOUT, level: :info)
+      @query_status_backoff = ->(attempt) { 1.5**attempt - 1 }
+    end
+
+    def configure
+      yield(self)
+      self
+    end
   end
 end

data/lib/egis/database.rb

@@ -10,24 +10,30 @@ module Egis
   #
   # It is recommended to create database objects using {Egis::Client#database} method.
   #
+  # @!attribute [r] name
+  #   @return [String] Athena database name
+  #
   class Database
-    def initialize(database_name, client: Egis::Client.new, output_downloader: Egis::OutputDownloader.new)
+    def initialize(name, client: Egis::Client.new, output_downloader: Egis::OutputDownloader.new(client.aws_s3_client))
       @client = client
-      @database_name = database_name
       @output_downloader = output_downloader
+      @name = name
     end
 
+    attr_reader :name
+
     ##
     # Creates {Egis::Table} object. Executing it doesn't create Athena table yet.
     #
     # @param [String] table_name
     # @param [Egis::TableSchema] table_schema
     # @param [String] table_location S3 URL with table location (e.g. `s3://s3_bucket/table/location/`)
-    # @param [:tsv, :csv, :orc] format Table format (defaults to :tsv)
+    # @param [:tsv, :csv, :orc, {serde: 'SerdeClass', serde_properties: {property: value}}] format Table format
+    # (defaults to :tsv)
     # @return [Egis::Table]
 
     def table(table_name, table_schema, table_location, **options)
-      Table.new(self, table_name, table_schema, table_location, options: options)
+      Table.new(self, table_name, table_schema, table_location, client: client, options: options)
     end
 
     ##
@@ -36,7 +42,10 @@ module Egis
     # @return [void]
 
     def create
-      client.execute_query("CREATE DATABASE IF NOT EXISTS #{translate_name(database_name)};", async: false)
+      log_database_creation
+
+      client.execute_query("CREATE DATABASE IF NOT EXISTS #{translate_name(name)};", async: false,
+                                                                                     system_execution: true)
     end
 
     ##
@@ -45,7 +54,9 @@ module Egis
     # @return [void]
 
     def create!
-      client.execute_query("CREATE DATABASE #{translate_name(database_name)};", async: false)
+      log_database_creation
+
+      client.execute_query("CREATE DATABASE #{translate_name(name)};", async: false, system_execution: true)
     end
 
     ##
@@ -54,7 +65,10 @@ module Egis
     # @return [void]
 
     def drop
-      client.execute_query("DROP DATABASE IF EXISTS #{translate_name(database_name)} CASCADE;", async: false)
+      log_database_removal
+
+      client.execute_query("DROP DATABASE IF EXISTS #{translate_name(name)} CASCADE;", async: false,
+                                                                                       system_execution: true)
     end
 
     ##
@@ -63,14 +77,16 @@ module Egis
     # @return [void]
 
     def drop!
-      client.execute_query("DROP DATABASE #{translate_name(database_name)} CASCADE;", async: false)
+      log_database_removal
+
+      client.execute_query("DROP DATABASE #{translate_name(name)} CASCADE;", async: false, system_execution: true)
     end
 
     ##
     # (see Egis::Client#execute_query)
 
     def execute_query(query, **options)
-      client.execute_query(query, **{database: database_name, **options})
+      client.execute_query(query, **{database: name, **options})
     end
 
     ##
@@ -86,14 +102,22 @@ module Egis
     # @return [Boolean]
 
     def exists?
-      query_status = client.execute_query("SHOW DATABASES LIKE '#{database_name}';", async: false)
+      query_status = client.execute_query("SHOW DATABASES LIKE '#{name}';", async: false, system_execution: true)
       parsed_result = output_downloader.download(query_status.output_location)
-      parsed_result.flatten.include?(database_name)
+      parsed_result.flatten.include?(name)
     end
 
     private
 
-    attr_reader :client, :database_name, :output_downloader
+    attr_reader :client, :output_downloader
+
+    def log_database_creation
+      Egis.logger.info { "Creating database #{name}" }
+    end
+
+    def log_database_removal
+      Egis.logger.info { "Removing database #{name}" }
+    end
 
     def translate_name(name)
       Egis.mode.database_name(name)

data/lib/egis/output_downloader.rb

@@ -5,8 +5,8 @@ require 'csv'
 module Egis
   # @!visibility private
   class OutputDownloader
-    def initialize(aws_client_provider: Egis::AwsClientProvider.new)
-      @s3_client = aws_client_provider.s3_client
+    def initialize(aws_s3_client)
+      @s3_client = aws_s3_client
     end
 
     def download(output_location)

data/lib/egis/query_status.rb

@@ -23,7 +23,8 @@ module Egis
     attr_reader :id, :status, :message, :output_location
 
     def initialize(id, status, message, output_location,
-                   output_downloader: Egis::OutputDownloader.new,
+                   client: Egis::Client.new,
+                   output_downloader: Egis::OutputDownloader.new(client.aws_s3_client),
                    output_parser: Egis::OutputParser.new)
       raise ArgumentError, "Unsupported status #{status}" unless STATUSES.include?(status)
 
@@ -51,6 +52,10 @@ module Egis
       status == RUNNING
     end
 
+    def cancelled?
+      status == CANCELLED
+    end
+
     def in_progress?
       [RUNNING, QUEUED].include?(status)
     end

data/lib/egis/s3_cleaner.rb

@@ -3,8 +3,8 @@
 module Egis
   # @!visibility private
   class S3Cleaner
-    def initialize(aws_client_provider: Egis::AwsClientProvider.new)
-      @s3_client = aws_client_provider.s3_client
+    def initialize(aws_s3_client)
+      @s3_client = aws_s3_client
     end
 
     def delete(bucket, prefix)

data/lib/egis/table.rb

@@ -9,7 +9,7 @@ module Egis
   # @!attribute [r] database
   #   @return [Egis::Database]
   # @!attribute [r] name
-  #   @return [String] Athena database name
+  #   @return [String] Athena table name
   # @!attribute [r] schema
   #   @return [Egis::TableSchema] table's schema object
   #
@@ -17,11 +17,13 @@ module Egis
     DEFAULT_OPTIONS = {format: :tsv}.freeze
 
     def initialize(database, name, schema, location, options: {},
+                   client: Egis::Client.new,
                    partitions_generator: Egis::PartitionsGenerator.new,
                    table_ddl_generator: Egis::TableDDLGenerator.new,
-                   output_downloader: Egis::OutputDownloader.new,
+                   output_downloader: Egis::OutputDownloader.new(client.aws_s3_client),
                    output_parser: Egis::OutputParser.new,
-                   table_data_wiper: Egis::TableDataWiper.new)
+                   s3_cleaner: Egis::S3Cleaner.new(client.aws_s3_client),
+                   table_data_wiper: Egis::TableDataWiper.new(s3_cleaner: s3_cleaner))
       @database = database
       @name = name
       @schema = schema
@@ -42,8 +44,10 @@ module Egis
     # @return [void]
 
     def create
+      log_table_creation
+
       create_table_sql = table_ddl_generator.create_table_sql(self, permissive: true)
-      database.execute_query(create_table_sql, async: false)
+      database.execute_query(create_table_sql, async: false, system_execution: true)
     end
 
     ##
@@ -52,8 +56,10 @@ module Egis
     # @return [void]
 
     def create!
+      log_table_creation
+
       create_table_sql = table_ddl_generator.create_table_sql(self, permissive: false)
-      database.execute_query(create_table_sql, async: false)
+      database.execute_query(create_table_sql, async: false, system_execution: true)
     end
 
     ##
@@ -67,7 +73,7 @@ module Egis
 
     def add_partitions(partitions)
       load_partitions_query = partitions_generator.to_sql(name, partitions, permissive: true)
-      database.execute_query(load_partitions_query, async: false)
+      database.execute_query(load_partitions_query, async: false, system_execution: true)
     end
 
     ##
@@ -76,7 +82,7 @@ module Egis
 
     def add_partitions!(partitions)
       load_partitions_query = partitions_generator.to_sql(name, partitions, permissive: false)
-      database.execute_query(load_partitions_query, async: false)
+      database.execute_query(load_partitions_query, async: false, system_execution: true)
     end
 
     ##
@@ -87,18 +93,30 @@ module Egis
     # @return [void]
 
     def discover_partitions
-      database.execute_query("MSCK REPAIR TABLE #{name};", async: false)
+      database.execute_query("MSCK REPAIR TABLE #{name};", async: false, system_execution: true)
     end
 
     ##
     # Insert data into the table. Mostly useful for testing purposes.
     #
-    # @param [Array] rows Array of arrays with row values
+    # @example Insert with array of arrays
+    #   table.upload_data([
+    #       ['hello world', 'mx', 1],
+    #       ['hello again', 'us', 2]
+    #   ])
+    #
+    # @example Insert with array of hashes
+    #   table.upload_data([
+    #       {message: 'hello world', country: 'mx', type: 1},
+    #       {message: 'hello again', country: 'us', type: 2}
+    #   ])
+    #
+    # @param [Array] rows Array of arrays or hashes with row values
     # @return [void]
 
     def upload_data(rows)
       query = data_insert_query(rows)
-      database.execute_query(query, async: false)
+      database.execute_query(query, async: false, system_execution: true)
     end
 
     ##
@@ -107,7 +125,7 @@ module Egis
     # @return [Array] Array of arrays with row values.
 
     def download_data
-      result = database.execute_query("SELECT * FROM #{name};", async: false)
+      result = database.execute_query("SELECT * FROM #{name};", async: false, system_execution: true)
       content = output_downloader.download(result.output_location)
       output_parser.parse(content, column_types)
     end
@@ -141,23 +159,37 @@ module Egis
     attr_reader :options, :partitions_generator, :table_ddl_generator, :output_downloader, :output_parser,
                 :table_data_wiper
 
-    def column_serializers
-      @column_serializers ||= column_types.map { |type| Egis::Types.serializer(type) }
+    def log_table_creation
+      Egis.logger.info { "Creating table #{database.name}.#{name} located in #{location}" }
     end
 
     def column_types
-      (schema.columns + schema.partitions).map(&:type)
+      all_columns.map(&:type)
+    end
+
+    def all_columns
+      schema.columns + schema.partitions
     end
 
     def data_insert_query(rows)
+      insert_values = rows.map { |row| row_literal_values(row) }
+      row_clause = insert_values.map { |row| row_values_statement(row) }.join(",\n")
+
       <<~SQL
         INSERT INTO #{name} VALUES
-        #{rows.map { |row| row_values_statement(row) }.join(",\n")};
+        #{row_clause}
       SQL
     end
 
+    def row_literal_values(row)
+      all_columns.map.with_index do |column, index|
+        value = row.is_a?(Hash) ? row[column.name] : row[index]
+        Egis::Types.serializer(column.type).literal(value)
+      end
+    end
+
     def row_values_statement(row)
-      "(#{row.zip(column_serializers).map { |value, serializer| serializer.literal(value) }.join(', ')})"
+      "(#{row.join(', ')})"
     end
   end
 end

data/lib/egis/table_ddl_generator.rb

@@ -9,7 +9,7 @@ module Egis
           #{column_definition_sql(table.schema.columns)}
         )
         #{partition_statement(table.schema)}
-        #{format_statement(table.format)}
+        #{row_format_statement(table.format)}
         LOCATION '#{table.location}';
       SQL
     end
@@ -34,7 +34,30 @@ module Egis
       columns.map { |column| "`#{column.name}` #{column.type}" }.join(",\n")
     end
 
-    def format_statement(format)
+    def serde?(format)
+      format.is_a?(Hash) && format.key?(:serde)
+    end
+
+    def row_format_statement(format)
+      return serde_row_format_statement(format) if serde?(format)
+
+      delimited_row_format_statement(format)
+    end
+
+    def serde_row_format_statement(format)
+      row_format = "ROW FORMAT SERDE '#{format[:serde]}'"
+      return row_format unless format.key?(:serde_properties)
+
+      serde_properties = format[:serde_properties].map { |property, value| "'#{property}' = '#{value}'" }
+      <<-SQL
+        #{row_format}
+        WITH SERDEPROPERTIES (
+          #{serde_properties.join(",\n")}
+        )
+      SQL
+    end
+
+    def delimited_row_format_statement(format)
       case format
       when :csv
         "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','"

data/lib/egis/testing.rb

@@ -43,6 +43,6 @@ module Egis # rubocop:disable Style/Documentation
     yield
   ensure
     @mode = previous_mode
-    test_mode.cleanup
+    test_mode&.cleanup
   end
 end

data/lib/egis/testing/testing_mode.rb

@@ -6,7 +6,7 @@ module Egis
     class TestingMode
       def initialize(test_id, s3_bucket,
                      client: Egis::Client.new,
-                     output_downloader: Egis::OutputDownloader.new,
+                     output_downloader: Egis::OutputDownloader.new(client.aws_s3_client),
                      s3_location_parser: Egis::S3LocationParser.new)
         @test_id = test_id
         @s3_bucket = s3_bucket

data/lib/egis/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Egis
-  VERSION = '1.1.1'
+  VERSION = '1.5.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: egis
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Agnieszka Czereba
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-18 00:00:00.000000000 Z
+date: 2021-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-athena
@@ -83,7 +83,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/u2i/egis
   source_code_uri: https://github.com/u2i/egis
-  changelog_uri: https://github.com/u2i/egis/blob/master/docs/CHANGELOG.md
+  changelog_uri: https://u2i.github.io/egis/file.CHANGELOG.html
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -99,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: A handy wrapper for AWS Athena Ruby SDK.