multiwoven-integrations 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4c9d23f9f2c64aa4e25911e6e399b2e316237902d742bb09cdc2fd4cb538666
-  data.tar.gz: 3257d08bcdf074497cd26dca09b3d8367039cda5b1c6922dcc8d943a5e0d293d
+  metadata.gz: e8263bcb86ca81c8262642cd939c8079e671e7cb03a29a905c24f564888038c5
+  data.tar.gz: cf5ba4ee0ad1f25967592e72369b3e0452d09715a036b70685d48ee4f29568de
 SHA512:
-  metadata.gz: 29ba8c2cc87689a8bf51ac6ea0e0ab439e83197cd1aaa81b390928cf088651a827de5ce66361f13a3cada6c8252fbe38b4ec5cd5ebd8f73e6d1ec44f7f2e16a0
-  data.tar.gz: c4361564c16167cc5f2c59b24c9ff644c59ea5fd6c09979d17dbcb11adc63f526e82cbc00783c5f23621ce5339e9a8f82f023c516234bea4f98f0bb3dcbbd7a1
+  metadata.gz: a0c976f1fd891d8474dbc113bce838c0a566e0127e49829c61c37042b479b3303185193beb66a204a752a87f8a7ec82b512d51fba5437e5239e1fc8a889ddcb9
+  data.tar.gz: 0ecd0c38c46db77c5e6c9ea6d8ef94cfd3325f7cb9e5f1a2d86ae6d22c307305595cac210d5d6710843806d8c8891c1b04bc7cf663efcb7e5b42cd3bbe45e39f

data/README.md

@@ -43,6 +43,23 @@ Multiwoven integrations is the collection of connectors built on top of [Multiwo
 Multiwoven protocol is an open source standard for moving data between data sources to any third-part destinations.
 Anyone can build a connetor with basic ruby knowledge using the protocol.
 
+## Prerequisites
+
+Before you begin the installation, ensure you have the following dependencies installed:
+
+- **MySQL Client**
+  - Command: `brew install mysql-client`
+  - Description: Required for database interactions.
+
+- **Zstandard (zstd)**
+  - Command: `brew install zstd`
+  - Description: Needed for data compression and decompression.
+
+- **OpenSSL 3**
+  - Command: `brew install openssl@3`
+  - Description: Essential for secure communication.
+
+
 ### Installation
 
 Install the gem and add to the application's Gemfile by executing:

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.3.2"
+    VERSION = "0.3.3"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/amazon_s3/client.rb

@@ -4,13 +4,13 @@ module Multiwoven::Integrations::Source
   module AmazonS3
     include Multiwoven::Integrations::Core
     class Client < SourceConnector
-      DISCOVER_QUERY = "SELECT * FROM S3Object LIMIT 1;"
-
+      @session_name = ""
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
-        auth_data = get_auth_data(connection_config)
-        client = config_aws(auth_data, connection_config[:region])
-        client.get_bucket_location({ bucket: connection_config[:bucket] })
+        @session_name = "connection-#{connection_config[:region]}-#{connection_config[:bucket]}"
+        conn = create_connection(connection_config)
+        path = build_path(connection_config)
+        get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
         ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
       rescue StandardError => e
         ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
@@ -18,17 +18,13 @@ module Multiwoven::Integrations::Source
 
       def discover(connection_config)
         connection_config = connection_config.with_indifferent_access
-        auth_data = get_auth_data(connection_config)
-        connection_config[:access_id] = auth_data.credentials.access_key_id
-        connection_config[:secret_access] = auth_data.credentials.secret_access_key
-        connection_config[:session_token] = auth_data.credentials.session_token
+        @session_name = "discover-#{connection_config[:region]}-#{connection_config[:bucket]}"
         conn = create_connection(connection_config)
         # If pulling from multiple files, all files must have the same schema
-        path = build_path(connection_config[:path])
-        full_path = "s3://#{connection_config[:bucket]}/#{path}*.#{connection_config[:file_type]}"
-        records = get_results(conn, "DESCRIBE SELECT * FROM '#{full_path}';")
+        path = build_path(connection_config)
+        records = get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
         columns = build_discover_columns(records)
-        streams = [Multiwoven::Integrations::Protocol::Stream.new(name: full_path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
+        streams = [Multiwoven::Integrations::Protocol::Stream.new(name: path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
         catalog = Catalog.new(streams: streams)
         catalog.to_multiwoven_message
       rescue StandardError => e
@@ -37,10 +33,7 @@ module Multiwoven::Integrations::Source
 
       def read(sync_config)
         connection_config = sync_config.source.connection_specification.with_indifferent_access
-        auth_data = get_auth_data(connection_config)
-        connection_config[:access_id] = auth_data.credentials.access_key_id
-        connection_config[:secret_access] = auth_data.credentials.secret_access_key
-        connection_config[:session_token] = auth_data.credentials.session_token
+        @session_name = "#{sync_config.sync_id}-#{sync_config.source.name}-#{sync_config.destination.name}"
         conn = create_connection(connection_config)
         query = sync_config.model.query
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
@@ -57,38 +50,47 @@ module Multiwoven::Integrations::Source
       private
 
       def get_auth_data(connection_config)
+        session = @session_name
+        @session_name = ""
         if connection_config[:auth_type] == "user"
           Aws::Credentials.new(connection_config[:access_id], connection_config[:secret_access])
         elsif connection_config[:auth_type] == "role"
           sts_client = Aws::STS::Client.new(region: connection_config[:region])
-          session_name = "s3-check-connection"
-          sts_client.assume_role({
-                                   role_arn: connection_config[:arn],
-                                   role_session_name: session_name
-                                 })
+          resp = sts_client.assume_role({
+                                          role_arn: connection_config[:arn],
+                                          role_session_name: session
+                                        })
+          Aws::Credentials.new(
+            resp.credentials.access_key_id,
+            resp.credentials.secret_access_key,
+            resp.credentials.session_token
+          )
         end
       end
 
-      # DuckDB
       def create_connection(connection_config)
+        # In the case when previewing a query
+        @session_name = "preview-#{connection_config[:region]}-#{connection_config[:bucket]}" if @session_name.to_s.empty?
+        auth_data = get_auth_data(connection_config)
         conn = DuckDB::Database.open.connect
         # Set up S3 configuration
         secret_query = "
               CREATE SECRET amazons3_source (
               TYPE S3,
-              KEY_ID '#{connection_config[:access_id]}',
-              SECRET '#{connection_config[:secret_access]}',
+              KEY_ID '#{auth_data.credentials.access_key_id}',
+              SECRET '#{auth_data.credentials.secret_access_key}',
               REGION '#{connection_config[:region]}',
-              SESSION_TOKEN '#{connection_config[:session_token]}'
+              SESSION_TOKEN '#{auth_data.credentials.session_token}'
           );
         "
         get_results(conn, secret_query)
         conn
       end
 
-      def build_path(path)
-        path = "#{path}/" if !path.to_s.strip.empty? && path[-1] != "/"
-        path
+      def build_path(connection_config)
+        path = connection_config[:path]
+        path = "#{path}/" if path.to_s.strip.empty? || path[-1] != "/"
+        "s3://#{connection_config[:bucket]}#{path}*.#{connection_config[:file_type]}"
       end
 
       def get_results(conn, query)
@@ -132,41 +134,6 @@ module Multiwoven::Integrations::Source
           "boolean"
         end
       end
-
-      # AWS SDK
-      def config_aws(config, region)
-        Aws.config.update({
-                            region: region,
-                            credentials: config
-                          })
-        Aws::S3::Client.new
-      end
-
-      def build_select_content_options(config, query)
-        config = config.with_indifferent_access
-        bucket_name = config[:bucket]
-        file_key = config[:file_key]
-        file_type = config[:file_type]
-        options = {
-          bucket: bucket_name,
-          key: file_key,
-          expression_type: "SQL",
-          expression: query,
-          output_serialization: {
-            json: {}
-          }
-        }
-        if file_type == "parquet"
-          options[:input_serialization] = {
-            parquet: {}
-          }
-        elsif file_type == "csv"
-          options[:input_serialization] = {
-            csv: { file_header_info: "USE" }
-          }
-        end
-        options
-      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-06-27 00:00:00.000000000 Z
+date: 2024-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport