multiwoven-integrations 0.34.15 → 0.34.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0bc47bb18e9eb776b3e1b72ad711dddba187f2fb5d092deb0f8261dc17c250a
-  data.tar.gz: a49ddccf93fba66d26359af93f66b4a60f211e759e83f4da32eb9f555e359be7
+  metadata.gz: b6b99b3873653f5b2e7e7854a933ad303133548fe02d3c226e6cdb5416b3a2fa
+  data.tar.gz: b4e0f73a8cf353a66884917ba79d3f9e1535bb9807acaeddb810ceb50b00fbd3
 SHA512:
-  metadata.gz: 4445bb6e997771cdfe4f9113ca2a1fe444ac6825aad43ea013a1a74b86abc853738c851cf86d47dc6d975cb15e0bb96b7be9010bf00f76719e821cc7bcc8b65f
-  data.tar.gz: bd04c9011ab5eafd34bf9bc2019b03b6948fe6dfc31302b19d05a98dd97f7138e04162a6ca0bd394ee8888a9d8e1f34b61e9f360eadfefe179fbc3ba6cbf6302
+  metadata.gz: 66cb050fc40ed6385371b2f4577f77f2902c79f29e36a3fa4a5388314653cacff8b66427eb39deeec3af9773227f5568b5fb45b3cbe2a5c103567c0bee43bcb4
+  data.tar.gz: 37a520ae02b487111059022dc1d27b7f177a5b52436df77dba9d3d0adce32355d909d75d0057eb60a2dc9b7428d73befdd3b75bf38c56df32352cc26d3861dad

data/lib/multiwoven/integrations/destination/amazon_s3/client.rb

@@ -13,9 +13,12 @@ module Multiwoven::Integrations::Destination
         ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
       end
 
-      def discover(_connection_config = nil)
-        catalog_json = read_json(CATALOG_SPEC_PATH)
-        catalog = build_catalog(catalog_json)
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        conn = create_connection(connection_config)
+        records = discover_columns_from_s3(conn, connection_config)
+        grouped = group_by_table(records, connection_config[:file_name])
+        catalog = Catalog.new(streams: create_streams(grouped))
         catalog.to_multiwoven_message
       rescue StandardError => e
         handle_exception(e, {
@@ -94,6 +97,60 @@ module Multiwoven::Integrations::Destination
         timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
         "#{connection_config[:file_name]}_#{timestamp}.#{connection_config[:format_type]}"
       end
+
+      def build_discover_prefix(connection_config)
+        file_path = connection_config[:file_path].to_s.strip
+        file_path = "#{file_path}/" if file_path.present? && file_path[-1] != "/"
+        format_type = connection_config[:format_type].to_s.downcase
+        "#{file_path}#{connection_config[:file_name]}.#{format_type}"
+      end
+
+      def discover_columns_from_s3(s3_client, connection_config)
+        bucket = connection_config[:bucket_name]
+        prefix = build_discover_prefix(connection_config)
+        format_type = connection_config[:format_type].to_s.downcase
+
+        response = s3_client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys: 100)
+        raise StandardError, "No files found in the bucket" if response.contents.empty?
+
+        key = response.contents&.find { |obj| obj.key.end_with?(".#{format_type}") }&.key
+        raise StandardError, "No files found in the bucket" if key.nil?
+
+        read_csv_headers(s3_client, bucket, key)
+      end
+
+      def read_csv_headers(s3_client, bucket, key)
+        obj = s3_client.get_object(bucket: bucket, key: key)
+        first_line = obj.body.read.to_s.lines.first
+        return [] if first_line.nil? || first_line.strip.empty?
+
+        CSV.parse_line(first_line.strip)
+      end
+
+      def group_by_table(records, file_name)
+        result = {}
+        records.each do |entry|
+          table_name = file_name
+          column_data = {
+            column_name: entry,
+            type: "string",
+            optional: true
+          }
+          result[table_name] ||= { tablename: table_name, columns: [] }
+          result[table_name][:columns] << column_data
+        end
+        result
+      end
+
+      def create_streams(tables)
+        tables.values.map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(
+            name: r[:tablename],
+            action: StreamAction["create"],
+            json_schema: convert_to_json_schema(r[:columns])
+          )
+        end
+      end
     end
   end
 end

data/lib/multiwoven/integrations/destination/amazon_s3/config/spec.json

@@ -4,7 +4,7 @@
   "connection_specification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "AmazonS3",
-    "required": ["access_key_id", "secret_access_key", "region", "bucket_name", "file_path", "file_name", "format_type" ],
+    "required": ["access_key_id", "secret_access_key", "region", "bucket_name", "file_name", "format_type" ],
     "properties": {
       "access_key_id": {
         "description": "The AWS Access Key ID to use for authentication.",

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.34.15"
+    VERSION = "0.34.16"
 
     ENABLED_SOURCES = %w[
       Snowflake

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.34.15
+  version: 0.34.16
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-02-12 00:00:00.000000000 Z
+date: 2026-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -599,7 +599,6 @@ files:
 - lib/multiwoven/integrations/destination/ais_data_store/config/spec.json
 - lib/multiwoven/integrations/destination/ais_data_store/icon.svg
 - lib/multiwoven/integrations/destination/amazon_s3/client.rb
-- lib/multiwoven/integrations/destination/amazon_s3/config/catalog.json
 - lib/multiwoven/integrations/destination/amazon_s3/config/meta.json
 - lib/multiwoven/integrations/destination/amazon_s3/config/spec.json
 - lib/multiwoven/integrations/destination/amazon_s3/icon.svg

data/lib/multiwoven/integrations/destination/amazon_s3/config/catalog.json

@@ -1,16 +0,0 @@
-{
-  "request_rate_limit": 600,
-  "request_rate_limit_unit": "minute",
-  "request_rate_concurrency": 10,
-  "schema_mode": "schemaless",
-  "streams": [
-    {
-      "name": "create",
-      "batch_support": true,
-      "batch_size": 100000,
-      "action": "create",
-      "json_schema": {},
-      "supported_sync_modes": ["full_refresh","incremental"]
-    }
-  ]
-}