multiwoven-integrations 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9a7e28f8f3f1a559e5bb4feaec6a030201d447e1f1c7e90e8e8ced256274d5c
-  data.tar.gz: a16b584bbc112998e54b9a51785498411e98cb8156116f66e082afd55c1c4916
+  metadata.gz: e8263bcb86ca81c8262642cd939c8079e671e7cb03a29a905c24f564888038c5
+  data.tar.gz: cf5ba4ee0ad1f25967592e72369b3e0452d09715a036b70685d48ee4f29568de
 SHA512:
-  metadata.gz: f2e3b44f30db035ac8b87f1cc2ffbecc34aef53630ff7c5757193affaebc0749c827368c72cdb04444a00c0abc800c6c2b0cef97aa2484b532662292e8b0ccd0
-  data.tar.gz: 0f2650b28d03b4118dcb86d3533cadf6fec2bcb54c0b54f349e79aa7f276b50c9a574e6b3ef87c2b19dda810b18cefcf00dfb470706753e49f5e35c0a5d858a4
+  metadata.gz: a0c976f1fd891d8474dbc113bce838c0a566e0127e49829c61c37042b479b3303185193beb66a204a752a87f8a7ec82b512d51fba5437e5239e1fc8a889ddcb9
+  data.tar.gz: 0ecd0c38c46db77c5e6c9ea6d8ef94cfd3325f7cb9e5f1a2d86ae6d22c307305595cac210d5d6710843806d8c8891c1b04bc7cf663efcb7e5b42cd3bbe45e39f

data/README.md

@@ -43,6 +43,23 @@ Multiwoven integrations is the collection of connectors built on top of [Multiwo
 Multiwoven protocol is an open source standard for moving data between data sources to any third-part destinations.
 Anyone can build a connetor with basic ruby knowledge using the protocol.
 
+## Prerequisites
+
+Before you begin the installation, ensure you have the following dependencies installed:
+
+- **MySQL Client**
+  - Command: `brew install mysql-client`
+  - Description: Required for database interactions.
+
+- **Zstandard (zstd)**
+  - Command: `brew install zstd`
+  - Description: Needed for data compression and decompression.
+
+- **OpenSSL 3**
+  - Command: `brew install openssl@3`
+  - Description: Essential for secure communication.
+
+
 ### Installation
 
 Install the gem and add to the application's Gemfile by executing:

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.3.1"
+    VERSION = "0.3.3"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/amazon_s3/client.rb

@@ -4,12 +4,13 @@ module Multiwoven::Integrations::Source
   module AmazonS3
     include Multiwoven::Integrations::Core
     class Client < SourceConnector
-      DISCOVER_QUERY = "SELECT * FROM S3Object LIMIT 1;"
-
+      @session_name = ""
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
-        client = config_aws(connection_config)
-        client.get_bucket_policy_status({ bucket: connection_config[:bucket] })
+        @session_name = "connection-#{connection_config[:region]}-#{connection_config[:bucket]}"
+        conn = create_connection(connection_config)
+        path = build_path(connection_config)
+        get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
         ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
       rescue StandardError => e
         ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
@@ -17,13 +18,13 @@ module Multiwoven::Integrations::Source
 
       def discover(connection_config)
         connection_config = connection_config.with_indifferent_access
+        @session_name = "discover-#{connection_config[:region]}-#{connection_config[:bucket]}"
         conn = create_connection(connection_config)
         # If pulling from multiple files, all files must have the same schema
-        path = build_path(connection_config[:path])
-        full_path = "s3://#{connection_config[:bucket]}/#{path}*.#{connection_config[:file_type]}"
-        records = get_results(conn, "DESCRIBE SELECT * FROM '#{full_path}';")
+        path = build_path(connection_config)
+        records = get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
         columns = build_discover_columns(records)
-        streams = [Multiwoven::Integrations::Protocol::Stream.new(name: full_path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
+        streams = [Multiwoven::Integrations::Protocol::Stream.new(name: path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
         catalog = Catalog.new(streams: streams)
         catalog.to_multiwoven_message
       rescue StandardError => e
@@ -32,6 +33,7 @@ module Multiwoven::Integrations::Source
 
       def read(sync_config)
         connection_config = sync_config.source.connection_specification.with_indifferent_access
+        @session_name = "#{sync_config.sync_id}-#{sync_config.source.name}-#{sync_config.destination.name}"
         conn = create_connection(connection_config)
         query = sync_config.model.query
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
@@ -47,25 +49,48 @@ module Multiwoven::Integrations::Source
 
       private
 
-      # DuckDB
+      def get_auth_data(connection_config)
+        session = @session_name
+        @session_name = ""
+        if connection_config[:auth_type] == "user"
+          Aws::Credentials.new(connection_config[:access_id], connection_config[:secret_access])
+        elsif connection_config[:auth_type] == "role"
+          sts_client = Aws::STS::Client.new(region: connection_config[:region])
+          resp = sts_client.assume_role({
+                                          role_arn: connection_config[:arn],
+                                          role_session_name: session
+                                        })
+          Aws::Credentials.new(
+            resp.credentials.access_key_id,
+            resp.credentials.secret_access_key,
+            resp.credentials.session_token
+          )
+        end
+      end
+
       def create_connection(connection_config)
+        # In the case when previewing a query
+        @session_name = "preview-#{connection_config[:region]}-#{connection_config[:bucket]}" if @session_name.to_s.empty?
+        auth_data = get_auth_data(connection_config)
         conn = DuckDB::Database.open.connect
         # Set up S3 configuration
         secret_query = "
               CREATE SECRET amazons3_source (
               TYPE S3,
-              KEY_ID '#{connection_config[:access_id]}',
-              SECRET '#{connection_config[:secret_access]}',
-              REGION '#{connection_config[:region]}'
+              KEY_ID '#{auth_data.credentials.access_key_id}',
+              SECRET '#{auth_data.credentials.secret_access_key}',
+              REGION '#{connection_config[:region]}',
+              SESSION_TOKEN '#{auth_data.credentials.session_token}'
           );
         "
         get_results(conn, secret_query)
         conn
       end
 
-      def build_path(path)
-        path = "#{path}/" if !path.to_s.strip.empty? && path[-1] != "/"
-        path
+      def build_path(connection_config)
+        path = connection_config[:path]
+        path = "#{path}/" if path.to_s.strip.empty? || path[-1] != "/"
+        "s3://#{connection_config[:bucket]}#{path}*.#{connection_config[:file_type]}"
       end
 
       def get_results(conn, query)
@@ -109,43 +134,6 @@ module Multiwoven::Integrations::Source
           "boolean"
         end
       end
-
-      # AWS SDK
-      def config_aws(config)
-        config = config.with_indifferent_access
-        Aws.config.update({
-                            region: config[:region],
-                            credentials: Aws::Credentials.new(config[:access_id], config[:secret_access])
-                          })
-        config.with_indifferent_access
-        Aws::S3::Client.new
-      end
-
-      def build_select_content_options(config, query)
-        config = config.with_indifferent_access
-        bucket_name = config[:bucket]
-        file_key = config[:file_key]
-        file_type = config[:file_type]
-        options = {
-          bucket: bucket_name,
-          key: file_key,
-          expression_type: "SQL",
-          expression: query,
-          output_serialization: {
-            json: {}
-          }
-        }
-        if file_type == "parquet"
-          options[:input_serialization] = {
-            parquet: {}
-          }
-        elsif file_type == "csv"
-          options[:input_serialization] = {
-            csv: { file_header_info: "USE" }
-          }
-        end
-        options
-      end
     end
   end
 end

data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json

@@ -6,13 +6,51 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "AmazonS3",
     "type": "object",
-    "required": ["region", "bucket", "access_id", "secret_access", "file_type"],
+    "if": {
+      "properties": {
+        "auth_type": {
+          "enum": ["user"]
+        }
+      }
+    },
+    "then": {
+      "required": [
+        "auth_type",
+        "region",
+        "bucket",
+        "access_id",
+        "secret_access",
+        "file_type"
+      ]
+    },
+    "else": {
+      "required": [
+        "auth_type",
+        "region",
+        "bucket",
+        "arn",
+        "file_type"
+      ]
+    },
     "properties": {
-      "region": {
-        "description": "AWS region",
-        "examples": ["us-east-2"],
+      "auth_type": {
+        "title": "Authentication type",
         "type": "string",
-        "title": "Region",
+        "default": "user",
+        "description": "Authenticate either by using an IAM User (Access Key ID & Secret Access Key) or an IAM Role (ARN)",
+        "enum": [
+          "user",
+          "role"
+        ],
+        "enumNames": [
+          "IAM User",
+          "IAM Role"
+        ],
+        "order": 0
+      },
+      "arn": {
+        "type": "string",
+        "title": "IAM Role ARN",
         "order": 1
       },
       "access_id": {
@@ -26,26 +64,40 @@
         "multiwoven_secret": true,
         "order": 3
       },
+      "region": {
+        "description": "AWS region",
+        "examples": [
+          "us-east-2"
+        ],
+        "type": "string",
+        "title": "Region",
+        "order": 4
+      },
       "bucket": {
         "description": "Bucket Name",
         "type": "string",
         "title": "Bucket",
-        "order": 4
+        "order": 5
       },
       "path": {
         "description": "Path to csv or parquet files",
-        "examples": ["/path/to/files"],
+        "examples": [
+          "/path/to/files"
+        ],
         "type": "string",
         "title": "Path",
-        "order": 5
+        "order": 6
       },
       "file_type": {
         "description": "The type of file to read",
         "type": "string",
         "title": "File Type",
-        "enum": ["csv", "parquet"],
-        "order": 6
+        "enum": [
+          "csv",
+          "parquet"
+        ],
+        "order": 7
       }
     }
   }
-}
+}

data/lib/multiwoven/integrations.rb

@@ -30,6 +30,7 @@ require "base64"
 require "aws-sdk-s3"
 require "duckdb"
 require "iterable-api-client"
+require "aws-sdk-sts"
 
 # Service
 require_relative "integrations/config"

data/multiwoven-integrations.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "async-websocket"
   spec.add_runtime_dependency "aws-sdk-athena"
   spec.add_runtime_dependency "aws-sdk-s3"
+  spec.add_runtime_dependency "aws-sdk-sts"
   spec.add_runtime_dependency "csv"
   spec.add_runtime_dependency "dry-schema"
   spec.add_runtime_dependency "dry-struct"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-06-26 00:00:00.000000000 Z
+date: 2024-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sts
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement