egis 1.2.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1ca49c2a770af01b7ff35956785bd77ca5df10446e229620f7cb21892b65e1f
-  data.tar.gz: 450b5de96addf904a43f7b14bc5b55c3f876c982977e6b8ae51a6e58a0466a04
+  metadata.gz: ca7464c31cc32115edb77470e2d5f2abe99ab9f4a64eef2ae0820ed8d5b982b7
+  data.tar.gz: 8002e611e1c59635365a3ca2ff54191e1a9671f4b5498c940438c85b7ea45d28
 SHA512:
-  metadata.gz: f542064733b5d9805a27dd40b4628527e3faf932788f96b7911db0726f729f8f89835c06ba767d5532cab603633838146113ae46dfd8b1089d70ba3288806312
-  data.tar.gz: d833d875b4cee9b4d435aa0394908b1466ff1588bb9620fe237c72b95e16965ebf34299a8e5a63d1ce70bf1642ebfd053f6089688ffd83320dadcbb10cc2ca8d
+  metadata.gz: 5971ac98ab6bfdcbcac23a4b9e0e2b8b31b14cb06987ea6f3891fc0c1787087ebfca85eae026c8cb1c2952f11e4659cf36827492b8d9fdf7d43d3a6943d3bd7f
+  data.tar.gz: 98d379a3ddb1b3796b08e9c5a4cc584826324fa03e24d64ddb604405ec57a6f0d75a3d2138cf9123f154e275b5202c592814c4cea6a00049d75eda6ec5d2de8b

data/lib/egis/aws_client_provider.rb

@@ -6,25 +6,23 @@ require 'aws-sdk-athena'
 module Egis
   # @!visibility private
   class AwsClientProvider
-    def s3_client
-      Aws::S3::Client.new(client_config)
+    def s3_client(configuration)
+      Aws::S3::Client.new(client_config(configuration))
     end
 
-    def athena_client
-      Aws::Athena::Client.new(client_config)
+    def athena_client(configuration)
+      Aws::Athena::Client.new(client_config(configuration))
     end
 
     private
 
-    def client_config
-      configuration = Egis.configuration
-
-      config = {}
-      config[:region] = configuration.aws_region if configuration.aws_region
-      config[:access_key_id] = configuration.aws_access_key_id if configuration.aws_access_key_id
-      config[:secret_access_key] = configuration.aws_secret_access_key if configuration.aws_secret_access_key
-      config[:profile] = configuration.aws_profile if configuration.aws_profile
-      config
+    def client_config(configuration)
+      {
+        region: configuration.aws_region,
+        access_key_id: configuration.aws_access_key_id,
+        secret_access_key: configuration.aws_secret_access_key,
+        profile: configuration.aws_profile
+      }.compact
     end
   end
 end

data/lib/egis/client.rb

@@ -4,6 +4,9 @@ module Egis
   ##
   # The most fundamental {Egis} class. Provides an interface for executing Athena queries.
   #
+  # @yieldparam config [Egis::Configuration] Egis configuration block, if missing Egis will use global configuration
+  #   provided by {Egis.configure}
+  #
   # See configuration instructions {Egis.configure}.
   #
   # @see Egis.configure
@@ -33,14 +36,17 @@ module Egis
       'CANCELLED' => Egis::QueryStatus::CANCELLED
     }.freeze
 
-    DEFAULT_QUERY_STATUS_BACKOFF = ->(attempt) { 1.5**attempt - 1 }
+    private_constant :QUERY_STATUS_MAPPING
 
-    private_constant :QUERY_STATUS_MAPPING, :DEFAULT_QUERY_STATUS_BACKOFF
+    attr_reader :aws_s3_client
 
-    def initialize(aws_client_provider: Egis::AwsClientProvider.new, s3_location_parser: Egis::S3LocationParser.new)
-      @aws_athena_client = aws_client_provider.athena_client
+    def initialize(aws_client_provider: Egis::AwsClientProvider.new,
+                   s3_location_parser: Egis::S3LocationParser.new,
+                   &block)
+      @configuration = block_given? ? Egis.configuration.dup.configure(&block) : Egis.configuration
+      @aws_athena_client = aws_client_provider.athena_client(configuration)
+      @aws_s3_client = aws_client_provider.s3_client(configuration)
       @s3_location_parser = s3_location_parser
-      @query_status_backoff = Egis.configuration.query_status_backoff || DEFAULT_QUERY_STATUS_BACKOFF
     end
 
     ##
@@ -99,16 +105,17 @@ module Egis
         query_execution.query_execution_id,
         QUERY_STATUS_MAPPING.fetch(query_status),
         query_execution.status.state_change_reason,
-        parse_output_location(query_execution)
+        parse_output_location(query_execution),
+        client: self
       )
     end
 
     private
 
-    attr_reader :aws_athena_client, :s3_location_parser, :query_status_backoff
+    attr_reader :configuration, :aws_athena_client, :s3_location_parser
 
     def query_execution_params(query, work_group, database, output_location)
-      work_group_params = work_group || Egis.configuration.work_group
+      work_group_params = work_group || configuration.work_group
 
       params = {query_string: query}
       params[:work_group] = work_group_params if work_group_params
@@ -128,7 +135,7 @@ module Egis
     def wait_for_query_to_finish(query_id)
       attempt = 1
       loop do
-        sleep(query_status_backoff.call(attempt))
+        sleep(configuration.query_status_backoff.call(attempt))
         status = query_status(query_id)
 
         return status unless status.queued? || status.running?

data/lib/egis/configuration.rb

@@ -8,6 +8,12 @@ module Egis
 
     def initialize
       @logger = Logger.new(STDOUT, level: :info)
+      @query_status_backoff = ->(attempt) { 1.5**attempt - 1 }
+    end
+
+    def configure
+      yield(self)
+      self
     end
   end
 end

data/lib/egis/database.rb

@@ -14,10 +14,10 @@ module Egis
   #   @return [String] Athena database name
   #
   class Database
-    def initialize(name, client: Egis::Client.new, output_downloader: Egis::OutputDownloader.new)
+    def initialize(name, client: Egis::Client.new, output_downloader: Egis::OutputDownloader.new(client.aws_s3_client))
       @client = client
-      @name = name
       @output_downloader = output_downloader
+      @name = name
     end
 
     attr_reader :name
@@ -28,11 +28,11 @@ module Egis
     # @param [String] table_name
     # @param [Egis::TableSchema] table_schema
     # @param [String] table_location S3 URL with table location (e.g. `s3://s3_bucket/table/location/`)
-    # @param [:tsv, :csv, :orc] format Table format (defaults to :tsv)
+    # @param [:tsv, :csv, :orc, :orc_index_access, :json, String] format Table Format (defaults to :tsv)
     # @return [Egis::Table]
 
     def table(table_name, table_schema, table_location, **options)
-      Table.new(self, table_name, table_schema, table_location, options: options)
+      Table.new(self, table_name, table_schema, table_location, client: client, options: options)
     end
 
     ##

data/lib/egis/output_downloader.rb

@@ -5,8 +5,8 @@ require 'csv'
 module Egis
   # @!visibility private
   class OutputDownloader
-    def initialize(aws_client_provider: Egis::AwsClientProvider.new)
-      @s3_client = aws_client_provider.s3_client
+    def initialize(aws_s3_client)
+      @s3_client = aws_s3_client
     end
 
     def download(output_location)

data/lib/egis/query_status.rb

@@ -23,7 +23,8 @@ module Egis
     attr_reader :id, :status, :message, :output_location
 
     def initialize(id, status, message, output_location,
-                   output_downloader: Egis::OutputDownloader.new,
+                   client: Egis::Client.new,
+                   output_downloader: Egis::OutputDownloader.new(client.aws_s3_client),
                    output_parser: Egis::OutputParser.new)
       raise ArgumentError, "Unsupported status #{status}" unless STATUSES.include?(status)
 
@@ -51,6 +52,10 @@ module Egis
       status == RUNNING
     end
 
+    def cancelled?
+      status == CANCELLED
+    end
+
     def in_progress?
       [RUNNING, QUEUED].include?(status)
     end

data/lib/egis/s3_cleaner.rb

@@ -3,8 +3,8 @@
 module Egis
   # @!visibility private
   class S3Cleaner
-    def initialize(aws_client_provider: Egis::AwsClientProvider.new)
-      @s3_client = aws_client_provider.s3_client
+    def initialize(aws_s3_client)
+      @s3_client = aws_s3_client
     end
 
     def delete(bucket, prefix)

data/lib/egis/table.rb

@@ -17,11 +17,13 @@ module Egis
     DEFAULT_OPTIONS = {format: :tsv}.freeze
 
     def initialize(database, name, schema, location, options: {},
+                   client: Egis::Client.new,
                    partitions_generator: Egis::PartitionsGenerator.new,
                    table_ddl_generator: Egis::TableDDLGenerator.new,
-                   output_downloader: Egis::OutputDownloader.new,
+                   output_downloader: Egis::OutputDownloader.new(client.aws_s3_client),
                    output_parser: Egis::OutputParser.new,
-                   table_data_wiper: Egis::TableDataWiper.new)
+                   s3_cleaner: Egis::S3Cleaner.new(client.aws_s3_client),
+                   table_data_wiper: Egis::TableDataWiper.new(s3_cleaner: s3_cleaner))
       @database = database
       @name = name
       @schema = schema
@@ -97,7 +99,19 @@ module Egis
     ##
     # Insert data into the table. Mostly useful for testing purposes.
     #
-    # @param [Array] rows Array of arrays with row values
+    # @example Insert with array of arrays
+    #   table.upload_data([
+    #       ['hello world', 'mx', 1],
+    #       ['hello again', 'us', 2]
+    #   ])
+    #
+    # @example Insert with array of hashes
+    #   table.upload_data([
+    #       {message: 'hello world', country: 'mx', type: 1},
+    #       {message: 'hello again', country: 'us', type: 2}
+    #   ])
+    #
+    # @param [Array] rows Array of arrays or hashes with row values
     # @return [void]
 
     def upload_data(rows)
@@ -128,7 +142,6 @@ module Egis
 
     ##
     # @return Table data format
-
     def format
       options.fetch(:format)
     end
@@ -149,23 +162,33 @@ module Egis
       Egis.logger.info { "Creating table #{database.name}.#{name} located in #{location}" }
     end
 
-    def column_serializers
-      @column_serializers ||= column_types.map { |type| Egis::Types.serializer(type) }
+    def column_types
+      all_columns.map(&:type)
     end
 
-    def column_types
-      (schema.columns + schema.partitions).map(&:type)
+    def all_columns
+      schema.columns + schema.partitions
     end
 
     def data_insert_query(rows)
+      insert_values = rows.map { |row| row_literal_values(row) }
+      row_clause = insert_values.map { |row| row_values_statement(row) }.join(",\n")
+
       <<~SQL
         INSERT INTO #{name} VALUES
-        #{rows.map { |row| row_values_statement(row) }.join(",\n")};
+        #{row_clause}
       SQL
     end
 
+    def row_literal_values(row)
+      all_columns.map.with_index do |column, index|
+        value = row.is_a?(Hash) ? row[column.name] : row[index]
+        Egis::Types.serializer(column.type).literal(value)
+      end
+    end
+
     def row_values_statement(row)
-      "(#{row.zip(column_serializers).map { |value, serializer| serializer.literal(value) }.join(', ')})"
+      "(#{row.join(', ')})"
     end
   end
 end

data/lib/egis/table_ddl_generator.rb

@@ -35,13 +35,30 @@ module Egis
     end
 
     def format_statement(format)
+      return format if format.is_a?(String)
+
+      format_preset(format)
+    end
+
+    def format_preset(format) # rubocop:disable Metrics/MethodLength
       case format
       when :csv
         "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','"
       when :tsv
         "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t'"
       when :orc
+        <<~SQL
+          ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
+          WITH SERDEPROPERTIES (
+            'orc.column.index.access' = 'false'
+          )
+          STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
+          OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
+        SQL
+      when :orc_index_access
         'STORED AS ORC'
+      when :json
+        "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'"
       else
         raise Errors::UnsupportedTableFormat, format.to_s
       end

data/lib/egis/testing/testing_mode.rb

@@ -6,7 +6,7 @@ module Egis
     class TestingMode
       def initialize(test_id, s3_bucket,
                      client: Egis::Client.new,
-                     output_downloader: Egis::OutputDownloader.new,
+                     output_downloader: Egis::OutputDownloader.new(client.aws_s3_client),
                      s3_location_parser: Egis::S3LocationParser.new)
         @test_id = test_id
         @s3_bucket = s3_bucket

data/lib/egis/testing.rb

@@ -43,6 +43,6 @@ module Egis # rubocop:disable Style/Documentation
     yield
   ensure
     @mode = previous_mode
-    test_mode.cleanup if test_mode
+    test_mode&.cleanup
   end
 end

data/lib/egis/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Egis
-  VERSION = '1.2.1'
+  VERSION = '2.0.0'
 end

data/lib/egis.rb

@@ -50,8 +50,8 @@ require 'egis/s3_location_parser'
 #
 module Egis
   class << self
-    def configure
-      yield(configuration)
+    def configure(&block)
+      configuration.configure(&block)
     end
 
     # @!visibility private

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: egis
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Agnieszka Czereba
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-29 00:00:00.000000000 Z
+date: 2021-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-athena
@@ -99,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: A handy wrapper for AWS Athena Ruby SDK.