multiwoven-integrations 0.34.14 → 0.34.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af3f6a21444f5aa85047e82e0174fa7971c3999a9191641fc4c856eada85a8a8
-  data.tar.gz: df145b18f1f9693eea22d24ff683c19f5aab138a0120691451eb18929f89cc27
+  metadata.gz: b0bc47bb18e9eb776b3e1b72ad711dddba187f2fb5d092deb0f8261dc17c250a
+  data.tar.gz: a49ddccf93fba66d26359af93f66b4a60f211e759e83f4da32eb9f555e359be7
 SHA512:
-  metadata.gz: a9b2487bf3e5108110f35794faf4ea49f4d1769aec463d552170347191cdac7856a5bf42d116dbaf38a9ede5f7fff569ac56edaae92acf0da0b53f6231f4a514
-  data.tar.gz: 3130acccdbe8f24751d263a7126c5c9394eadca7e731f7c1e8e97c5f127227058af5741a5ff728bdfef90254e4fb8a31b363505a0a39b99fd0d6388a1e8ac834
+  metadata.gz: 4445bb6e997771cdfe4f9113ca2a1fe444ac6825aad43ea013a1a74b86abc853738c851cf86d47dc6d975cb15e0bb96b7be9010bf00f76719e821cc7bcc8b65f
+  data.tar.gz: bd04c9011ab5eafd34bf9bc2019b03b6948fe6dfc31302b19d05a98dd97f7138e04162a6ca0bd394ee8888a9d8e1f34b61e9f360eadfefe179fbc3ba6cbf6302

data/lib/multiwoven/integrations/destination/amazon_s3/client.rb

@@ -43,11 +43,18 @@ module Multiwoven::Integrations::Destination
       private
 
       def create_connection(connection_config)
-        Aws::S3::Client.new(
+        connection_config = connection_config.with_indifferent_access
+        s3_options = {
           region: connection_config[:region],
           access_key_id: connection_config[:access_key_id],
           secret_access_key: connection_config[:secret_access_key]
-        )
+        }
+        endpoint = connection_config[:endpoint].to_s.strip
+        if endpoint.present?
+          s3_options[:endpoint] = endpoint
+          s3_options[:force_path_style] = connection_config[:path_style] == true
+        end
+        Aws::S3::Client.new(**s3_options)
       end
 
       def upload_csv_content(sync_config, records)

data/lib/multiwoven/integrations/destination/amazon_s3/config/spec.json

@@ -50,6 +50,20 @@
         "order": 6,
         "enum": ["csv"],
         "default": "csv"
+      },
+      "endpoint": {
+        "description": "Custom S3-compatible endpoint URL (e.g. http://localhost:9000 for MinIO). Leave empty for AWS S3.",
+        "type": "string",
+        "title": "Custom Endpoint",
+        "examples": ["http://localhost:9000"],
+        "order": 7
+      },
+      "path_style": {
+        "description": "Use path-style bucket URLs. Required for MinIO and some S3-compatible stores when using a custom endpoint.",
+        "type": "boolean",
+        "title": "Path Style",
+        "default": false,
+        "order": 8
       }
     }
   }

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.34.14"
+    VERSION = "0.34.15"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/amazon_s3/client.rb

@@ -9,7 +9,6 @@ module Multiwoven::Integrations::Source
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
         @session_name = "connection-#{connection_config[:region]}-#{connection_config[:bucket]}"
-
         if unstructured_data?(connection_config)
           create_s3_connection(connection_config)
           @s3_resource.bucket(connection_config[:bucket]).objects.limit(1).first
@@ -90,11 +89,18 @@ module Multiwoven::Integrations::Source
         # Get authentication credentials
         auth_data = get_auth_data(connection_config)
 
-        # Create S3 resource for easier operations
-        @s3_resource = Aws::S3::Resource.new(
+        # Build client options; use custom endpoint when provided (e.g. MinIO)
+        s3_options = {
           region: connection_config[:region],
           credentials: auth_data
-        )
+        }
+        endpoint = connection_config[:endpoint].to_s.strip
+        if endpoint.present?
+          s3_options[:endpoint] = endpoint
+          s3_options[:force_path_style] = connection_config[:path_style] == true
+        end
+
+        @s3_resource = Aws::S3::Resource.new(**s3_options)
       end
 
       def create_connection(connection_config)
@@ -104,17 +110,30 @@ module Multiwoven::Integrations::Source
         conn = DuckDB::Database.open.connect
         # Install and/or Load the HTTPFS extension
         conn.execute(INSTALL_HTTPFS_QUERY)
-        # Set up S3 configuration
-        secret_query = "
-              CREATE SECRET amazons3_source (
-              TYPE S3,
-              KEY_ID '#{auth_data.credentials.access_key_id}',
-              SECRET '#{auth_data.credentials.secret_access_key}',
-              REGION '#{connection_config[:region]}',
-              SESSION_TOKEN '#{auth_data.credentials.session_token}'
-          );
-        "
+        # Set up S3 configuration (optional custom endpoint for MinIO / S3-compatible stores)
+        secret_parts = [
+          "TYPE S3",
+          "KEY_ID '#{auth_data.credentials.access_key_id.gsub("'", "''")}'",
+          "SECRET '#{auth_data.credentials.secret_access_key.gsub("'", "''")}'",
+          "REGION '#{connection_config[:region]}'"
+        ]
+        secret_parts << "SESSION_TOKEN '#{auth_data.credentials.session_token.gsub("'", "''")}'" if auth_data.credentials.session_token.present?
+        endpoint = connection_config[:endpoint].to_s.strip
+        if endpoint.present?
+          uri = URI.parse(endpoint)
+
+          # DuckDB expects host:port only
+          duckdb_endpoint = uri.host
+          duckdb_endpoint += ":#{uri.port}" if uri.port
+
+          secret_parts << "ENDPOINT '#{duckdb_endpoint}'"
+
+          # Disable SSL if http
+          secret_parts << "USE_SSL false" if uri.scheme == "http"
+        end
+        secret_query = "CREATE SECRET amazons3_source (#{secret_parts.join(", ")});"
         get_results(conn, secret_query)
+        conn.execute("SET s3_url_style = 'path';") if connection_config[:path_style].present?
         conn
       end
 

data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json

@@ -113,6 +113,20 @@
           "parquet"
         ],
         "order": 8
+      },
+      "endpoint": {
+        "description": "Custom S3-compatible endpoint URL (e.g. http://localhost:9000 for MinIO). Leave empty for AWS S3.",
+        "type": "string",
+        "title": "Custom Endpoint",
+        "examples": ["http://localhost:9000"],
+        "order": 10
+      },
+      "path_style": {
+        "description": "Use path-style bucket URLs. Required for MinIO and some S3-compatible stores when using a custom endpoint.",
+        "type": "boolean",
+        "title": "Path Style",
+        "default": false,
+        "order": 11
       }
     }
   }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.34.14
+  version: 0.34.15
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-12-15 00:00:00.000000000 Z
+date: 2026-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport