multiwoven-integrations 0.16.2 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b110db8b3c603525e09b967f773f5f7e3e8bc771c5204178b7058dac8f25bf9f
-  data.tar.gz: e29d8c55f595c6810a052b6e47d73958413ac4ae7d6dfc1904103059716f0c12
+  metadata.gz: a182e10fe6151cdfbffef960e34aaf8b14be263a2f03a29c71e247d9125938a8
+  data.tar.gz: 79e70b4fd2f81c406c2ca436bc45d422ad63ccd6489bf5c012a7b8ff601ddf21
 SHA512:
-  metadata.gz: 0f25aaf49865562a122c28d54cb4c4ed6f6619aa6054d84e7616a8211d8e4887bb8b69665aac7702c92768149e78649de4a2fcf51c0a63dc5c398745e20c37fd
-  data.tar.gz: 8dbf1e05b6ec0154654653696f922ff7f1765179ca73909bfd3a55d923e6569ff2430c705d202ad5727cb73c8dc70f5d31090a5bae9507ea60d495a1f8d36081
+  metadata.gz: 3808200a5d1f51b873748399963d285053cd6830bcbeb1beb20f08d0eca91bea535a5c811092b767923282aa66ccc4adf6a754cf329b312ab5248475d6a9f042
+  data.tar.gz: 436bee5b004c3b32b7232e28f4e75eefeeacd40d524c6d6b5bf4d4b7e2475638eba06e9bc96820c227c36c6ff33b624172f81cb7a8ffe78ad27040ff7ffbdad1

data/lib/multiwoven/integrations/destination/amazon_s3/client.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Destination
+  module AmazonS3
+    include Multiwoven::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        conn = create_connection(connection_config)
+        conn.head_bucket(bucket: connection_config[:bucket_name])
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AMAZONS3:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def write(sync_config, records, _action = "destination_insert")
+        records_size = records.size
+        log_message_array = []
+        write_success = upload_csv_content(sync_config, records)
+        write_failure = records_size - write_success
+        log_message_array << log_request_response("info", @args, @response)
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AMAZONS3:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        Aws::S3::Client.new(
+          region: connection_config[:region],
+          access_key_id: connection_config[:access_key_id],
+          secret_access_key: connection_config[:secret_access_key]
+        )
+      end
+
+      def upload_csv_content(sync_config, records)
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        conn = create_connection(connection_config)
+        file_name = generate_local_file_name(connection_config)
+        csv_content = generate_csv_content(records)
+        begin
+          @args = ["create", connection_config[:bucket_name], "#{connection_config[:file_path]}#{file_name}", csv_content]
+          @response = conn.put_object(
+            bucket: connection_config[:bucket_name],
+            key: "#{connection_config[:file_path]}#{file_name}",
+            body: csv_content
+          )
+          write_success = records.size
+        rescue StandardError => e
+          handle_exception(e, {
+                             context: "AMAZONS3:RECORD:WRITE:EXCEPTION",
+                             type: "error",
+                             sync_id: sync_config.sync_id,
+                             sync_run_id: sync_config.sync_run_id
+                           })
+          write_success = 0
+        end
+        write_success
+      end
+
+      def generate_csv_content(records)
+        CSV.generate do |csv|
+          headers = records.first.keys
+          csv << headers
+          records.each { |record| csv << record.values_at(*headers) }
+        end
+      end
+
+      def generate_local_file_name(connection_config)
+        timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
+        "#{connection_config[:file_name]}_#{timestamp}.#{connection_config[:format_type]}"
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/amazon_s3/config/catalog.json

@@ -0,0 +1,16 @@
+{
+  "request_rate_limit": 600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "schema_mode": "schemaless",
+  "streams": [
+    {
+      "name": "create",
+      "batch_support": true,
+      "batch_size": 100000,
+      "action": "create",
+      "json_schema": {},
+      "supported_sync_modes": ["full_refresh","incremental"]
+    }
+  ]
+}

data/lib/multiwoven/integrations/destination/amazon_s3/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "AmazonS3",
+    "title": "Amazon S3",
+    "connector_type": "source",
+    "category": "Data Lake",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-amazons3",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/destination/amazon_s3/config/spec.json

@@ -0,0 +1,56 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/destination/amazons3",
+  "stream_type": "static",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "AmazonS3",
+    "required": ["access_key_id", "secret_access_key", "region", "bucket_name", "file_path", "file_name", "format_type" ],
+    "properties": {
+      "access_key_id": {
+        "description": "The AWS Access Key ID to use for authentication.",
+        "type": "string",
+        "title": "Personal Access Key",
+        "order": 0
+      },
+      "secret_access_key": {
+        "description": "The AWS Secret Access Key to use for authentication.",
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Secret Access Key",
+        "order": 1
+      },
+      "region": {
+        "description": "AWS region.",
+        "type": "string",
+        "title": "Region",
+        "order": 2
+      },
+      "bucket_name": {
+        "title": "Bucket Name",
+        "description": "Amazon S3 bucket name.",
+        "type": "string",
+        "order": 3
+      },
+      "file_path": {
+        "title": "File Path",
+        "type": "string",
+        "description": "Path to the directory where files will be written.",
+        "order": 4
+      },
+      "file_name": {
+        "title": "File Name",
+        "type": "string",
+        "description": "Name of the file to be written.",
+        "order": 5
+      },
+      "format_type": {
+        "title": "File Format Type",
+        "type": "string",
+        "description": "Format of the data output.",
+        "order": 6,
+        "enum": ["csv"],
+        "default": "csv"
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/destination/amazon_s3/icon.svg

@@ -0,0 +1,34 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="428" height="512" viewBox="0 0 428 512">
+  <defs>
+    <style>
+      .cls-1 {
+        fill: #e25444;
+      }
+
+      .cls-1, .cls-2, .cls-3 {
+        fill-rule: evenodd;
+      }
+
+      .cls-2 {
+        fill: #7b1d13;
+      }
+
+      .cls-3 {
+        fill: #58150d;
+      }
+    </style>
+  </defs>
+  <path class="cls-1" d="M378,99L295,257l83,158,34-19V118Z"/>
+  <path class="cls-2" d="M378,99L212,118,127.5,257,212,396l166,19V99Z"/>
+  <path class="cls-3" d="M43,99L16,111V403l27,12L212,257Z"/>
+  <path class="cls-1" d="M42.637,98.667l169.587,47.111V372.444L42.637,415.111V98.667Z"/>
+  <path class="cls-3" d="M212.313,170.667l-72.008-11.556,72.008-81.778,71.83,81.778Z"/>
+  <path class="cls-3" d="M284.143,159.111l-71.919,11.733-71.919-11.733V77.333"/>
+  <path class="cls-3" d="M212.313,342.222l-72.008,13.334,72.008,70.222,71.83-70.222Z"/>
+  <path class="cls-2" d="M212,16L140,54V159l72.224-20.333Z"/>
+  <path class="cls-2" d="M212.224,196.444l-71.919,7.823V309.105l71.919,8.228V196.444Z"/>
+  <path class="cls-2" d="M212.224,373.333L140.305,355.3V458.363L212.224,496V373.333Z"/>
+  <path class="cls-1" d="M284.143,355.3l-71.919,18.038V496l71.919-37.637V355.3Z"/>
+  <path class="cls-1" d="M212.224,196.444l71.919,7.823V309.105l-71.919,8.228V196.444Z"/>
+  <path class="cls-1" d="M212,16l72,38V159l-72-20V16Z"/>
+</svg>

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.16.2"
+    VERSION = "0.18.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -21,6 +21,7 @@ module Multiwoven
       VertexModel
       HttpModel
       OpenAI
+      Sftp
     ].freeze
 
     ENABLED_DESTINATIONS = %w[
@@ -45,6 +46,7 @@ module Multiwoven
       MicrosoftSql
       Mailchimp
       AISDataStore
+      AmazonS3
     ].freeze
   end
 end

data/lib/multiwoven/integrations/source/sftp/client.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module Sftp
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        if @sftp.stat!(@remote_file_path)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SFTP:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        db = create_connection(connection_config)
+        @sftp.download!(@remote_file_path, @tempfile.path)
+        query = "SELECT * FROM read_csv_auto('#{@tempfile.path}')"
+        records = db.query(query).columns
+        catalog = Catalog.new(streams: create_streams(records.map(&:name)))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SFTP:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        @tempfile&.close!
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        conn = create_connection(connection_config)
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        query(conn, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SFTP:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        initialize_file_path(connection_config)
+        @sftp = with_sftp_client(connection_config)
+        conn = DuckDB::Database.open.connect
+        conn.execute(INSTALL_HTTPFS_QUERY)
+        conn
+      end
+
+      def initialize_file_path(connection_config)
+        @remote_file_path = File.join(
+          connection_config[:file_path],
+          "#{connection_config[:file_name]}.#{connection_config[:format_type]}"
+        )
+        @tempfile = Tempfile.new(File.basename(@remote_file_path))
+      end
+
+      def with_sftp_client(connection_config, &block)
+        Net::SFTP.start(
+          connection_config[:host],
+          connection_config[:username],
+          password: connection_config[:password],
+          port: connection_config.fetch(:port, 22), &block
+        )
+      end
+
+      def get_results(conn, query)
+        results = conn.query(query)
+        hash_array_values(results)
+      end
+
+      def query(conn, query)
+        query_regex = /\ASELECT\s+(?<columns>[\w,\s]+)\s+FROM\s+\w+\s*(?:LIMIT\s+(?<limit>\d+))?\s*(?:OFFSET\s+(?<offset>\d+))?\z/i
+        match = query.match(query_regex)
+        columns = match[:columns] || "*"
+        offset = match[:offset].to_i || 0
+        limit = match[:limit]&.to_i || nil
+        @sftp.download!(@remote_file_path, @tempfile.path)
+        adjusted_query = "SELECT #{columns} FROM read_csv_auto('#{@tempfile.path}') OFFSET #{offset}"
+        adjusted_query += " LIMIT #{limit}" if limit
+        records = get_results(conn, adjusted_query)
+        records.map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+
+      def hash_array_values(describe)
+        keys = describe.columns.map(&:name)
+        describe.map do |row|
+          Hash[keys.zip(row)]
+        end
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |column, index|
+          table_name = @remote_file_path
+          column_data = {
+            column_name: column,
+            type: "string",
+            optional: true
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/sftp/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Sftp",
+    "title": "SFTP",
+    "connector_type": "source",
+    "category": "File Storage",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-sftp",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/sftp/config/spec.json

@@ -0,0 +1,59 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/sftp",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "SFTP",
+    "required": ["host", "username", "password", "file_path", "format_type" ],
+    "properties": {
+      "host": {
+        "title": "Host",
+        "description": "Hostname of the SFTP server.",
+        "type": "string",
+        "order": 0
+      },
+      "port": {
+        "title": "Port",
+        "description": "Port of the SFTP server.",
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 65536,
+        "default": 22,
+        "order": 1
+      },
+      "username": {
+        "title": "User Name",
+        "description": "Username to use to access the SFTP server.",
+        "type": "string",
+        "order": 2
+      },
+      "password": {
+        "title": "Password",
+        "description": "Password associated with the username.",
+        "type": "string",
+        "multiwoven_secret": true,
+        "order": 3
+      },
+      "file_path": {
+        "title": "File path",
+        "type": "string",
+        "description": "Path to the directory where file is stored.",
+        "order": 4
+      },
+      "file_name": {
+        "title": "File Name",
+        "type": "string",
+        "description": "Name of the file to be written.",
+        "order": 5
+      },
+      "format_type": {
+        "title": "File Format Type",
+        "type": "string",
+        "description": "Format of the data output.",
+        "enum": ["csv"],
+        "default": "csv",
+        "order": 6
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/sftp/icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="250" height="250" fill="none"><g clip-path="url(#a)"><path fill="#615EFF" d="M200.745 218.706a12.051 12.051 0 0 1-3.483 8.475 11.837 11.837 0 0 1-8.4 3.514H62.072c-3.15 0-6.173-1.261-8.403-3.507a12.053 12.053 0 0 1-3.49-8.472V31.485a12.08 12.08 0 0 1 .903-4.593 11.996 11.996 0 0 1 2.578-3.894 11.875 11.875 0 0 1 3.86-2.6 11.791 11.791 0 0 1 4.552-.912h81.371l57.302 58.96v140.26Z"/><path fill="#fff" d="M83.868 156.832c-.142-1.429-.75-2.539-1.822-3.33-1.074-.792-2.53-1.187-4.37-1.187-1.25 0-2.305.177-3.166.531-.861.343-1.522.821-1.981 1.435a3.457 3.457 0 0 0-.673 2.09c-.023.65.112 1.217.407 1.701.307.484.725.903 1.256 1.258.53.342 1.144.643 1.84.903a19.02 19.02 0 0 0 2.229.638l3.255.779c1.58.354 3.03.827 4.352 1.417 1.32.591 2.464 1.317 3.431 2.179a9.057 9.057 0 0 1 2.247 3.047c.542 1.169.82 2.509.831 4.021-.011 2.22-.578 4.145-1.698 5.775-1.109 1.617-2.712 2.875-4.811 3.773-2.088.885-4.606 1.328-7.554 1.328-2.925 0-5.472-.449-7.642-1.346-2.158-.898-3.845-2.226-5.06-3.986-1.202-1.771-1.833-3.962-1.892-6.572h7.412c.082 1.217.43 2.232 1.044 3.047.625.803 1.456 1.411 2.494 1.825 1.05.401 2.235.602 3.555.602 1.298 0 2.424-.189 3.38-.567.966-.378 1.715-.903 2.246-1.576.53-.673.796-1.447.796-2.321 0-.815-.242-1.5-.725-2.055-.472-.555-1.168-1.027-2.088-1.417-.908-.39-2.022-.744-3.343-1.063l-3.945-.992c-3.054-.744-5.466-1.907-7.235-3.489-1.77-1.583-2.648-3.714-2.636-6.395-.012-2.196.572-4.116 1.751-5.757 1.192-1.641 2.825-2.923 4.9-3.844 2.076-.921 4.435-1.382 7.076-1.382 2.69 0 5.036.461 7.04 1.382 2.018.921 3.586 2.203 4.706 3.844 1.12 1.641 1.699 3.543 1.734 5.704h-7.341Zm12.988 25.844v-36.278h23.988v6.324h-16.328v8.645h14.736v6.324h-14.736v14.985h-7.66Zm27.76-29.954v-6.324h29.754v6.324h-11.091v29.954h-7.571v-29.954h-11.092Zm34.654 29.954v-36.278h14.294c2.748 0 5.089.526 7.023 1.577 1.934 1.039 3.408 2.486 4.422 4.34 1.026 1.842 1.539 3.968 1.539 6.377s-.519 4.535-1.557 6.377c-1.037 1.842-2.541 3.277-4.51 4.304-1.958 1.028-4.329 1.541-7.112 1.541h-9.11v-6.146h7.872c1.474 0 2.689-.254 3.644-.762.967-.52 1.686-1.234 2.158-2.143.484-.921.725-1.978.725-3.171 0-1.205-.241-2.256-.725-3.153-.472-.91-1.191-1.612-2.158-2.108-.967-.508-2.194-.762-3.679-.762h-5.166v30.007h-7.66Z"/><path fill="#1A194D" d="m149.75 76.907 50.996 49.177V78.101h-45.173a11.33 11.33 0 0 1-5.643-1.433l-.18.24Z" opacity=".3"/><path fill="#C5C4FF" d="M200.746 78.35h-45.297a11.836 11.836 0 0 1-8.399-3.514 12.055 12.055 0 0 1-3.484-8.475v-47.2l57.18 59.189Z"/></g><defs><clipPath id="a"><path fill="#fff" d="M50 19h151v212H50z"/></clipPath></defs></svg>

data/lib/multiwoven/integrations.rb

@@ -74,6 +74,7 @@ require_relative "integrations/source/aws_sagemaker_model/client"
 require_relative "integrations/source/google_vertex_model/client"
 require_relative "integrations/source/http_model/client"
 require_relative "integrations/source/open_ai/client"
+require_relative "integrations/source/sftp/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"
@@ -97,6 +98,7 @@ require_relative "integrations/destination/microsoft_excel/client"
 require_relative "integrations/destination/microsoft_sql/client"
 require_relative "integrations/destination/mailchimp/client"
 require_relative "integrations/destination/ais_data_store/client"
+require_relative "integrations/destination/amazon_s3/client"
 
 module Multiwoven
   module Integrations

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.16.2
+  version: 0.18.0
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-12-24 00:00:00.000000000 Z
+date: 2025-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -596,6 +596,11 @@ files:
 - lib/multiwoven/integrations/destination/ais_data_store/config/meta.json
 - lib/multiwoven/integrations/destination/ais_data_store/config/spec.json
 - lib/multiwoven/integrations/destination/ais_data_store/icon.svg
+- lib/multiwoven/integrations/destination/amazon_s3/client.rb
+- lib/multiwoven/integrations/destination/amazon_s3/config/catalog.json
+- lib/multiwoven/integrations/destination/amazon_s3/config/meta.json
+- lib/multiwoven/integrations/destination/amazon_s3/config/spec.json
+- lib/multiwoven/integrations/destination/amazon_s3/icon.svg
 - lib/multiwoven/integrations/destination/databricks_lakehouse/client.rb
 - lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json
 - lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json
@@ -758,6 +763,10 @@ files:
 - lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json
 - lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/icon.svg
 - lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb
+- lib/multiwoven/integrations/source/sftp/client.rb
+- lib/multiwoven/integrations/source/sftp/config/meta.json
+- lib/multiwoven/integrations/source/sftp/config/spec.json
+- lib/multiwoven/integrations/source/sftp/icon.svg
 - lib/multiwoven/integrations/source/snowflake/client.rb
 - lib/multiwoven/integrations/source/snowflake/config/meta.json
 - lib/multiwoven/integrations/source/snowflake/config/spec.json