multiwoven-integrations 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: febb4cb77450c3c979b96a081e455bd9bcc7e108cd2947f3fc234d967a4c6be5
-  data.tar.gz: e2c4cfec3ffb70f51ba93ba1ce5e0f65183af0473db3ac7530178249ad87cb4b
+  metadata.gz: b4c9d23f9f2c64aa4e25911e6e399b2e316237902d742bb09cdc2fd4cb538666
+  data.tar.gz: 3257d08bcdf074497cd26dca09b3d8367039cda5b1c6922dcc8d943a5e0d293d
 SHA512:
-  metadata.gz: 367011ca22c74f7792ffc576041c25527b5890f07e0cad6e49f64abfb444e06656f415fbb601a7743ad8f47f12eebc5382a301a237ec180db259d6bed8c72709
-  data.tar.gz: 056e9d7df88a2444e4d9cc500af8b87f0376c4bde9e1236848b7ee1d633bd243f1275d72f251c8354e0eb208308d29b5727717623f99d248b3e7596c25dd3ce0
+  metadata.gz: 29ba8c2cc87689a8bf51ac6ea0e0ab439e83197cd1aaa81b390928cf088651a827de5ce66361f13a3cada6c8252fbe38b4ec5cd5ebd8f73e6d1ec44f7f2e16a0
+  data.tar.gz: c4361564c16167cc5f2c59b24c9ff644c59ea5fd6c09979d17dbcb11adc63f526e82cbc00783c5f23621ce5339e9a8f82f023c516234bea4f98f0bb3dcbbd7a1

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -11,7 +11,7 @@ module Multiwoven
     DestinationSyncMode = Types::String.enum("insert", "upsert")
     ConnectorType = Types::String.enum("source", "destination")
     ConnectorQueryType = Types::String.enum("raw_sql", "soql")
-    ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql")
+    ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector")
     ConnectionStatusType = Types::String.enum("succeeded", "failed")
     StreamType = Types::String.enum("static", "dynamic")
     StreamAction = Types::String.enum("fetch", "create", "update", "delete")

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.3.0"
+    VERSION = "0.3.2"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/amazon_s3/client.rb

@@ -8,8 +8,9 @@ module Multiwoven::Integrations::Source
 
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
-        client = config_aws(connection_config)
-        client.get_bucket_policy_status({ bucket: connection_config[:bucket] })
+        auth_data = get_auth_data(connection_config)
+        client = config_aws(auth_data, connection_config[:region])
+        client.get_bucket_location({ bucket: connection_config[:bucket] })
         ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
       rescue StandardError => e
         ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
@@ -17,6 +18,10 @@ module Multiwoven::Integrations::Source
 
       def discover(connection_config)
         connection_config = connection_config.with_indifferent_access
+        auth_data = get_auth_data(connection_config)
+        connection_config[:access_id] = auth_data.credentials.access_key_id
+        connection_config[:secret_access] = auth_data.credentials.secret_access_key
+        connection_config[:session_token] = auth_data.credentials.session_token
         conn = create_connection(connection_config)
         # If pulling from multiple files, all files must have the same schema
         path = build_path(connection_config[:path])
@@ -32,6 +37,10 @@ module Multiwoven::Integrations::Source
 
       def read(sync_config)
         connection_config = sync_config.source.connection_specification.with_indifferent_access
+        auth_data = get_auth_data(connection_config)
+        connection_config[:access_id] = auth_data.credentials.access_key_id
+        connection_config[:secret_access] = auth_data.credentials.secret_access_key
+        connection_config[:session_token] = auth_data.credentials.session_token
         conn = create_connection(connection_config)
         query = sync_config.model.query
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
@@ -47,6 +56,19 @@ module Multiwoven::Integrations::Source
 
       private
 
+      def get_auth_data(connection_config)
+        if connection_config[:auth_type] == "user"
+          Aws::Credentials.new(connection_config[:access_id], connection_config[:secret_access])
+        elsif connection_config[:auth_type] == "role"
+          sts_client = Aws::STS::Client.new(region: connection_config[:region])
+          session_name = "s3-check-connection"
+          sts_client.assume_role({
+                                   role_arn: connection_config[:arn],
+                                   role_session_name: session_name
+                                 })
+        end
+      end
+
       # DuckDB
       def create_connection(connection_config)
         conn = DuckDB::Database.open.connect
@@ -56,7 +78,8 @@ module Multiwoven::Integrations::Source
               TYPE S3,
               KEY_ID '#{connection_config[:access_id]}',
               SECRET '#{connection_config[:secret_access]}',
-              REGION '#{connection_config[:region]}'
+              REGION '#{connection_config[:region]}',
+              SESSION_TOKEN '#{connection_config[:session_token]}'
           );
         "
         get_results(conn, secret_query)
@@ -111,13 +134,11 @@ module Multiwoven::Integrations::Source
       end
 
       # AWS SDK
-      def config_aws(config)
-        config = config.with_indifferent_access
+      def config_aws(config, region)
         Aws.config.update({
-                            region: config[:region],
-                            credentials: Aws::Credentials.new(config[:access_id], config[:secret_access])
+                            region: region,
+                            credentials: config
                           })
-        config.with_indifferent_access
         Aws::S3::Client.new
       end
 

data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json

@@ -6,13 +6,51 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "AmazonS3",
     "type": "object",
-    "required": ["region", "bucket", "access_id", "secret_access", "file_type"],
+    "if": {
+      "properties": {
+        "auth_type": {
+          "enum": ["user"]
+        }
+      }
+    },
+    "then": {
+      "required": [
+        "auth_type",
+        "region",
+        "bucket",
+        "access_id",
+        "secret_access",
+        "file_type"
+      ]
+    },
+    "else": {
+      "required": [
+        "auth_type",
+        "region",
+        "bucket",
+        "arn",
+        "file_type"
+      ]
+    },
     "properties": {
-      "region": {
-        "description": "AWS region",
-        "examples": ["us-east-2"],
+      "auth_type": {
+        "title": "Authentication type",
         "type": "string",
-        "title": "Region",
+        "default": "user",
+        "description": "Authenticate either by using an IAM User (Access Key ID & Secret Access Key) or an IAM Role (ARN)",
+        "enum": [
+          "user",
+          "role"
+        ],
+        "enumNames": [
+          "IAM User",
+          "IAM Role"
+        ],
+        "order": 0
+      },
+      "arn": {
+        "type": "string",
+        "title": "IAM Role ARN",
         "order": 1
       },
       "access_id": {
@@ -26,26 +64,40 @@
         "multiwoven_secret": true,
         "order": 3
       },
+      "region": {
+        "description": "AWS region",
+        "examples": [
+          "us-east-2"
+        ],
+        "type": "string",
+        "title": "Region",
+        "order": 4
+      },
       "bucket": {
         "description": "Bucket Name",
         "type": "string",
         "title": "Bucket",
-        "order": 4
+        "order": 5
       },
       "path": {
         "description": "Path to csv or parquet files",
-        "examples": ["/path/to/files"],
+        "examples": [
+          "/path/to/files"
+        ],
         "type": "string",
         "title": "Path",
-        "order": 5
+        "order": 6
       },
       "file_type": {
         "description": "The type of file to read",
         "type": "string",
         "title": "File Type",
-        "enum": ["csv", "parquet"],
-        "order": 6
+        "enum": [
+          "csv",
+          "parquet"
+        ],
+        "order": 7
       }
     }
   }
-}
+}

data/lib/multiwoven/integrations.rb

@@ -30,6 +30,7 @@ require "base64"
 require "aws-sdk-s3"
 require "duckdb"
 require "iterable-api-client"
+require "aws-sdk-sts"
 
 # Service
 require_relative "integrations/config"

data/multiwoven-integrations.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "async-websocket"
   spec.add_runtime_dependency "aws-sdk-athena"
   spec.add_runtime_dependency "aws-sdk-s3"
+  spec.add_runtime_dependency "aws-sdk-sts"
   spec.add_runtime_dependency "csv"
   spec.add_runtime_dependency "dry-schema"
   spec.add_runtime_dependency "dry-struct"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-06-21 00:00:00.000000000 Z
+date: 2024-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sts
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement