multiwoven-integrations 0.22.1 → 0.22.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc3297301450525e523cde59f46f1371bd67fa92a31a0a3135ae9818563a9b31
-  data.tar.gz: 3168abc5dcd112f1cfd86b4daeb19d7501aba221261e5db75d1b10976bd95e6b
+  metadata.gz: 2352e4287b9b68a5f417a413da77bb2776c5e17e0330373f49ec182e3021aab3
+  data.tar.gz: cdae111a370a6e5a637141463b001278569538b71fac1f229c4a8ee5cc4b0725
 SHA512:
-  metadata.gz: e141fddcc210447d6dcf88cae35e051050965d0d3548e7a30f8a4b164e9a8df5fdad9e07d3b44d0d5f5edc714f6e3f52d457b3959a74c8363e88414802198d59
-  data.tar.gz: e32081015f5afeeeb76c393d8770b6e87901b4146681c2f9419ce75e2a8674c7d306566da0e4ca9d9cb43294019aa986540b094010dfb4a7dcefec4c5cc78558
+  metadata.gz: 3b7ca0805127b15270285a29fa13a45a847d259c7a74ff1fa1f6462bb3822ccd09237dc7d6a6345eb846e294233b94397fe150ab602e2df332e85aafc0d9af70
+  data.tar.gz: 422b9ff1ea7baad83d1a79066a09cdfcebbb509aff3e9768e20b0120b83043865e634dacf66750239a2cd2ef1e7309e7483baca0c698bb30c81f81ffa41ee07d

data/lib/multiwoven/integrations/core/unstructured_source_connector.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations::Core
+    class UnstructuredSourceConnector < SourceConnector
+      UNSTRUCTURED_SCHEMA = {
+        type: "object",
+        properties: {
+          element_id: { type: "string" },
+          text: { type: "string" },
+          created_date: { type: "string" },
+          modified_date: { type: "string" },
+          filename: { type: "string" },
+          filetype: { type: "string" }
+        },
+        required: %w[
+          element_id
+          text
+          created_date
+          modified_date
+          filename
+          filetype
+        ]
+      }.freeze
+
+      UNSTRUCTURED_STREAM_CONFIG = {
+        supported_sync_modes: ["incremental"],
+        source_defined_cursor: true,
+        default_cursor_field: ["modified_date"],
+        source_defined_primary_key: [["element_id"]]
+      }.freeze
+
+      # Commands for unstructured data operations
+      UNSTRUCTURED = "unstructured"
+      LIST_FILES_CMD = "list_files"
+      DOWNLOAD_FILE_CMD = "download_file"
+
+      def unstructured_data?(connection_config)
+        connection_config["data_type"] == UNSTRUCTURED
+      end
+
+      def create_unstructured_stream
+        Multiwoven::Integrations::Protocol::Stream.new(
+          name: UNSTRUCTURED,
+          action: StreamAction["fetch"],
+          json_schema: UNSTRUCTURED_SCHEMA,
+          **UNSTRUCTURED_STREAM_CONFIG
+        )
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -10,7 +10,7 @@ module Multiwoven
     SyncStatus = Types::String.enum("started", "running", "complete", "incomplete")
     DestinationSyncMode = Types::String.enum("insert", "upsert")
     ConnectorType = Types::String.enum("source", "destination")
-    ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml")
+    ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml", "unstructured")
     ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml", "dynamic_sql")
     ConnectionStatusType = Types::String.enum("succeeded", "failed")
     StreamType = Types::String.enum("static", "dynamic", "user_defined")

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.22.1"
+    VERSION = "0.22.4"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/amazon_s3/client.rb

@@ -3,14 +3,21 @@
 module Multiwoven::Integrations::Source
   module AmazonS3
     include Multiwoven::Integrations::Core
-    class Client < SourceConnector
+    class Client < UnstructuredSourceConnector
       @session_name = ""
+
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
         @session_name = "connection-#{connection_config[:region]}-#{connection_config[:bucket]}"
-        conn = create_connection(connection_config)
-        path = build_path(connection_config)
-        get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
+
+        if unstructured_data?(connection_config)
+          create_s3_connection(connection_config)
+          @s3_resource.bucket(connection_config[:bucket]).objects.limit(1).first
+        else
+          conn = create_connection(connection_config)
+          path = build_path(connection_config)
+          get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
+        end
         ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
       rescue StandardError => e
         ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
@@ -19,12 +26,17 @@ module Multiwoven::Integrations::Source
       def discover(connection_config)
         connection_config = connection_config.with_indifferent_access
         @session_name = "discover-#{connection_config[:region]}-#{connection_config[:bucket]}"
-        conn = create_connection(connection_config)
-        # If pulling from multiple files, all files must have the same schema
-        path = build_path(connection_config)
-        records = get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
-        columns = build_discover_columns(records)
-        streams = [Multiwoven::Integrations::Protocol::Stream.new(name: path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
+
+        streams = if unstructured_data?(connection_config)
+                    [create_unstructured_stream]
+                  else
+                    conn = create_connection(connection_config)
+                    # If pulling from multiple files, all files must have the same schema
+                    path = build_path(connection_config)
+                    records = get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
+                    columns = build_discover_columns(records)
+                    [Multiwoven::Integrations::Protocol::Stream.new(name: path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
+                  end
         catalog = Catalog.new(streams: streams)
         catalog.to_multiwoven_message
       rescue StandardError => e
@@ -34,6 +46,9 @@ module Multiwoven::Integrations::Source
       def read(sync_config)
         connection_config = sync_config.source.connection_specification.with_indifferent_access
         @session_name = "#{sync_config.sync_id}-#{sync_config.source.name}-#{sync_config.destination.name}"
+
+        return handle_unstructured_data(sync_config) if unstructured_data?(connection_config)
+
         conn = create_connection(connection_config)
         query = sync_config.model.query
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
@@ -69,6 +84,19 @@ module Multiwoven::Integrations::Source
         end
       end
 
+      def create_s3_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+
+        # Get authentication credentials
+        auth_data = get_auth_data(connection_config)
+
+        # Create S3 resource for easier operations
+        @s3_resource = Aws::S3::Resource.new(
+          region: connection_config[:region],
+          credentials: auth_data
+        )
+      end
+
       def create_connection(connection_config)
         # In the case when previewing a query
         @session_name = "preview-#{connection_config[:region]}-#{connection_config[:bucket]}" if @session_name.to_s.empty?
@@ -137,6 +165,71 @@ module Multiwoven::Integrations::Source
           "boolean"
         end
       end
+
+      def handle_unstructured_data(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        bucket_name = connection_config[:bucket]
+        command = sync_config.model.query.strip
+        create_s3_connection(connection_config)
+
+        case command
+        when LIST_FILES_CMD
+          list_files_in_folder(bucket_name, connection_config[:path] || "")
+        when /^#{DOWNLOAD_FILE_CMD}\s+(.+)$/
+          # Extract the file path and remove surrounding quotes if present
+          file_path = ::Regexp.last_match(1).strip
+          file_path = file_path.gsub(/^["']|["']$/, "") # Remove leading/trailing quotes
+          download_file_to_local(bucket_name, file_path, sync_config.sync_id)
+        else
+          raise "Invalid command. Supported commands: #{LIST_FILES_CMD}, #{DOWNLOAD_FILE_CMD} <file_path>"
+        end
+      end
+
+      def list_files_in_folder(bucket_name, folder_path)
+        folder_path = folder_path.end_with?("/") ? folder_path : "#{folder_path}/"
+        bucket = @s3_resource.bucket(bucket_name)
+
+        bucket.objects(prefix: folder_path).reject { |object| object.key == folder_path }.map do |object|
+          RecordMessage.new(
+            data: {
+              file_name: File.basename(object.key),
+              file_path: object.key,
+              size: object.content_length,
+              file_type: File.extname(object.key).sub(".", ""),
+              created_date: object.last_modified.to_s,
+              modified_date: object.last_modified.to_s
+            },
+            emitted_at: Time.now.to_i
+          ).to_multiwoven_message
+        end
+      end
+
+      def download_file_to_local(bucket_name, file_path, sync_id)
+        download_path = ENV["FILE_DOWNLOAD_PATH"]
+        file = if download_path
+                 File.join(download_path, "syncs", sync_id, File.basename(file_path))
+               else
+                 Tempfile.new(["s3_file", "syncs", sync_id, File.extname(file_path)]).path
+               end
+
+        object = @s3_resource.bucket(bucket_name).object(file_path)
+        object.get(response_target: file)
+
+        [RecordMessage.new(
+          data: {
+            local_path: file,
+            file_name: File.basename(file_path),
+            file_path: file_path,
+            size: object.content_length,
+            file_type: File.extname(file_path).sub(".", ""),
+            modified_date: object.last_modified.to_s,
+            created_date: object.last_modified.to_s
+          },
+          emitted_at: Time.now.to_i
+        ).to_multiwoven_message]
+      rescue Aws::S3::Errors::NoSuchKey
+        raise "File not found: #{file_path}"
+      end
     end
   end
 end

data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json

@@ -19,8 +19,7 @@
         "region",
         "bucket",
         "access_id",
-        "secret_access",
-        "file_type"
+        "secret_access"
       ]
     },
     "else": {
@@ -29,11 +28,21 @@
         "region",
         "bucket",
         "arn",
-        "external_id",
-        "file_type"
+        "external_id"
       ]
     },
     "properties": {
+      "data_type": {
+        "description": "Type of data in the files",
+        "type": "string",
+        "title": "Data Type",
+        "enum": [
+          "structured",
+          "unstructured"
+        ],
+        "default": "structured",
+        "order": 9
+      },
       "auth_type": {
         "title": "Authentication type",
         "type": "string",

data/lib/multiwoven/integrations/source/sftp/client.rb

@@ -87,7 +87,11 @@ module Multiwoven::Integrations::Source
 
       def query(conn, query)
         @sftp.download!(@remote_file_path, @tempfile.path)
-        query = query.gsub(/FROM\s+\S+/i, "FROM read_csv_auto('#{@tempfile.path}')") if query.match?(/\bFROM\b/i)
+        if query.gsub(/FROM\s+\S+/i).count > 1
+          query = query.gsub("FROM #{@remote_file_path}", "FROM read_csv_auto('#{@tempfile.path}')")
+        elsif query.match?(/\bFROM\b/i)
+          query = query.gsub(/FROM\s+\S+/i, "FROM read_csv_auto('#{@tempfile.path}')")
+        end
         records = get_results(conn, query)
         records.map do |row|
           RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message

data/lib/multiwoven/integrations.rb

@@ -56,6 +56,7 @@ require_relative "integrations/core/http_helper"
 require_relative "integrations/core/http_client"
 require_relative "integrations/core/streaming_http_client"
 require_relative "integrations/core/query_builder"
+require_relative "integrations/core/unstructured_source_connector"
 
 # Source
 require_relative "integrations/source/snowflake/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.22.1
+  version: 0.22.4
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-04-08 00:00:00.000000000 Z
+date: 2025-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -585,6 +585,7 @@ files:
 - lib/multiwoven/integrations/core/rate_limiter.rb
 - lib/multiwoven/integrations/core/source_connector.rb
 - lib/multiwoven/integrations/core/streaming_http_client.rb
+- lib/multiwoven/integrations/core/unstructured_source_connector.rb
 - lib/multiwoven/integrations/core/utils.rb
 - lib/multiwoven/integrations/destination/airtable/client.rb
 - lib/multiwoven/integrations/destination/airtable/config/catalog.json