multiwoven-integrations 0.3.2 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4c9d23f9f2c64aa4e25911e6e399b2e316237902d742bb09cdc2fd4cb538666
-  data.tar.gz: 3257d08bcdf074497cd26dca09b3d8367039cda5b1c6922dcc8d943a5e0d293d
+  metadata.gz: cb9f117bf574599b93ccfd2238f1629166c473184837ec48a16a447650b1dc54
+  data.tar.gz: efa39262c596d2b50bbd04d2d8a9d98889b708a9e2859e6ec3801b2c6eb79169
 SHA512:
-  metadata.gz: 29ba8c2cc87689a8bf51ac6ea0e0ab439e83197cd1aaa81b390928cf088651a827de5ce66361f13a3cada6c8252fbe38b4ec5cd5ebd8f73e6d1ec44f7f2e16a0
-  data.tar.gz: c4361564c16167cc5f2c59b24c9ff644c59ea5fd6c09979d17dbcb11adc63f526e82cbc00783c5f23621ce5339e9a8f82f023c516234bea4f98f0bb3dcbbd7a1
+  metadata.gz: 959b53319cb4581fd8aeb2731891f967031a23096c42e1e855fa79802bf81e799a746ba62c1c08f20484c3a443418d2175e1d9218abd28939eceaf89b6f87964
+  data.tar.gz: 22c706c8d47d9803cbdee5fa91a8f6862f7d0efb11c13466e2b0b0e07753a5e784e43443487a939a05f0cee0457bf310d12ce274e24ad322e5aa2cf8767c05ea

data/README.md

@@ -43,6 +43,23 @@ Multiwoven integrations is the collection of connectors built on top of [Multiwo
 Multiwoven protocol is an open source standard for moving data between data sources to any third-part destinations.
 Anyone can build a connetor with basic ruby knowledge using the protocol.
 
+## Prerequisites
+
+Before you begin the installation, ensure you have the following dependencies installed:
+
+- **MySQL Client**
+  - Command: `brew install mysql-client`
+  - Description: Required for database interactions.
+
+- **Zstandard (zstd)**
+  - Command: `brew install zstd`
+  - Description: Needed for data compression and decompression.
+
+- **OpenSSL 3**
+  - Command: `brew install openssl@3`
+  - Description: Essential for secure communication.
+
+
 ### Installation
 
 Install the gem and add to the application's Gemfile by executing:

data/lib/multiwoven/integrations/core/constants.rb

@@ -34,6 +34,9 @@ module Multiwoven
       AIRTABLE_BASES_ENDPOINT = "https://api.airtable.com/v0/meta/bases"
       AIRTABLE_GET_BASE_SCHEMA_ENDPOINT = "https://api.airtable.com/v0/meta/bases/{baseId}/tables"
 
+      AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
+      AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
+
       # HTTP
       HTTP_GET = "GET"
       HTTP_POST = "POST"

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.3.2"
+    VERSION = "0.3.4"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/amazon_s3/client.rb

@@ -4,13 +4,13 @@ module Multiwoven::Integrations::Source
   module AmazonS3
     include Multiwoven::Integrations::Core
     class Client < SourceConnector
-      DISCOVER_QUERY = "SELECT * FROM S3Object LIMIT 1;"
-
+      @session_name = ""
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
-        auth_data = get_auth_data(connection_config)
-        client = config_aws(auth_data, connection_config[:region])
-        client.get_bucket_location({ bucket: connection_config[:bucket] })
+        @session_name = "connection-#{connection_config[:region]}-#{connection_config[:bucket]}"
+        conn = create_connection(connection_config)
+        path = build_path(connection_config)
+        get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
         ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
       rescue StandardError => e
         ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
@@ -18,17 +18,13 @@ module Multiwoven::Integrations::Source
 
       def discover(connection_config)
         connection_config = connection_config.with_indifferent_access
-        auth_data = get_auth_data(connection_config)
-        connection_config[:access_id] = auth_data.credentials.access_key_id
-        connection_config[:secret_access] = auth_data.credentials.secret_access_key
-        connection_config[:session_token] = auth_data.credentials.session_token
+        @session_name = "discover-#{connection_config[:region]}-#{connection_config[:bucket]}"
         conn = create_connection(connection_config)
         # If pulling from multiple files, all files must have the same schema
-        path = build_path(connection_config[:path])
-        full_path = "s3://#{connection_config[:bucket]}/#{path}*.#{connection_config[:file_type]}"
-        records = get_results(conn, "DESCRIBE SELECT * FROM '#{full_path}';")
+        path = build_path(connection_config)
+        records = get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
         columns = build_discover_columns(records)
-        streams = [Multiwoven::Integrations::Protocol::Stream.new(name: full_path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
+        streams = [Multiwoven::Integrations::Protocol::Stream.new(name: path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
         catalog = Catalog.new(streams: streams)
         catalog.to_multiwoven_message
       rescue StandardError => e
@@ -37,10 +33,7 @@ module Multiwoven::Integrations::Source
 
       def read(sync_config)
         connection_config = sync_config.source.connection_specification.with_indifferent_access
-        auth_data = get_auth_data(connection_config)
-        connection_config[:access_id] = auth_data.credentials.access_key_id
-        connection_config[:secret_access] = auth_data.credentials.secret_access_key
-        connection_config[:session_token] = auth_data.credentials.session_token
+        @session_name = "#{sync_config.sync_id}-#{sync_config.source.name}-#{sync_config.destination.name}"
         conn = create_connection(connection_config)
         query = sync_config.model.query
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
@@ -57,38 +50,49 @@ module Multiwoven::Integrations::Source
       private
 
       def get_auth_data(connection_config)
+        session = @session_name.gsub(/\s+/, "-")
+        @session_name = ""
         if connection_config[:auth_type] == "user"
           Aws::Credentials.new(connection_config[:access_id], connection_config[:secret_access])
         elsif connection_config[:auth_type] == "role"
-          sts_client = Aws::STS::Client.new(region: connection_config[:region])
-          session_name = "s3-check-connection"
-          sts_client.assume_role({
-                                   role_arn: connection_config[:arn],
-                                   role_session_name: session_name
-                                 })
+          credentials = Aws::Credentials.new(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+          sts_client = Aws::STS::Client.new(region: connection_config[:region], credentials: credentials)
+          resp = sts_client.assume_role({
+                                          role_arn: connection_config[:arn],
+                                          role_session_name: session,
+                                          external_id: connection_config[:external_id]
+                                        })
+          Aws::Credentials.new(
+            resp.credentials.access_key_id,
+            resp.credentials.secret_access_key,
+            resp.credentials.session_token
+          )
         end
       end
 
-      # DuckDB
       def create_connection(connection_config)
+        # In the case when previewing a query
+        @session_name = "preview-#{connection_config[:region]}-#{connection_config[:bucket]}" if @session_name.to_s.empty?
+        auth_data = get_auth_data(connection_config)
         conn = DuckDB::Database.open.connect
         # Set up S3 configuration
         secret_query = "
               CREATE SECRET amazons3_source (
               TYPE S3,
-              KEY_ID '#{connection_config[:access_id]}',
-              SECRET '#{connection_config[:secret_access]}',
+              KEY_ID '#{auth_data.credentials.access_key_id}',
+              SECRET '#{auth_data.credentials.secret_access_key}',
               REGION '#{connection_config[:region]}',
-              SESSION_TOKEN '#{connection_config[:session_token]}'
+              SESSION_TOKEN '#{auth_data.credentials.session_token}'
           );
         "
         get_results(conn, secret_query)
         conn
       end
 
-      def build_path(path)
-        path = "#{path}/" if !path.to_s.strip.empty? && path[-1] != "/"
-        path
+      def build_path(connection_config)
+        path = connection_config[:path]
+        path = "#{path}/" if path.to_s.strip.empty? || path[-1] != "/"
+        "s3://#{connection_config[:bucket]}#{path}*.#{connection_config[:file_type]}"
       end
 
       def get_results(conn, query)
@@ -132,41 +136,6 @@ module Multiwoven::Integrations::Source
           "boolean"
         end
       end
-
-      # AWS SDK
-      def config_aws(config, region)
-        Aws.config.update({
-                            region: region,
-                            credentials: config
-                          })
-        Aws::S3::Client.new
-      end
-
-      def build_select_content_options(config, query)
-        config = config.with_indifferent_access
-        bucket_name = config[:bucket]
-        file_key = config[:file_key]
-        file_type = config[:file_type]
-        options = {
-          bucket: bucket_name,
-          key: file_key,
-          expression_type: "SQL",
-          expression: query,
-          output_serialization: {
-            json: {}
-          }
-        }
-        if file_type == "parquet"
-          options[:input_serialization] = {
-            parquet: {}
-          }
-        elsif file_type == "csv"
-          options[:input_serialization] = {
-            csv: { file_header_info: "USE" }
-          }
-        end
-        options
-      end
     end
   end
 end

data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json

@@ -29,6 +29,7 @@
         "region",
         "bucket",
         "arn",
+        "external_id",
         "file_type"
       ]
     },
@@ -53,16 +54,22 @@
         "title": "IAM Role ARN",
         "order": 1
       },
+      "external_id": {
+        "type": "string",
+        "title": "External Id",
+        "description": "Unique ID that allows handshake between AWS accounts.",
+        "order": 2
+      },
       "access_id": {
         "type": "string",
         "title": "Access Id",
-        "order": 2
+        "order": 3
       },
       "secret_access": {
         "type": "string",
         "title": "Secret Access",
         "multiwoven_secret": true,
-        "order": 3
+        "order": 4
       },
       "region": {
         "description": "AWS region",
@@ -71,13 +78,13 @@
         ],
         "type": "string",
         "title": "Region",
-        "order": 4
+        "order": 5
       },
       "bucket": {
         "description": "Bucket Name",
         "type": "string",
         "title": "Bucket",
-        "order": 5
+        "order": 6
       },
       "path": {
         "description": "Path to csv or parquet files",
@@ -86,7 +93,7 @@
         ],
         "type": "string",
         "title": "Path",
-        "order": 6
+        "order": 7
       },
       "file_type": {
         "description": "The type of file to read",
@@ -96,7 +103,7 @@
           "csv",
           "parquet"
         ],
-        "order": 7
+        "order": 8
       }
     }
   }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.4
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-06-27 00:00:00.000000000 Z
+date: 2024-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport