multiwoven-integrations 0.1.75 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae5784806bc5de9222274f6a03be3883720e308f1e0922a76a588f5e582112fb
-  data.tar.gz: 065a346c9beebf746e1e8d6d4a7b49d76c01185005269df442e58719933ca9f4
+  metadata.gz: 91cf72711d231942521cb83dd735b0a4b9067bd30d2aaeecf2be27d69d163da2
+  data.tar.gz: 5f2cc42a72e86cb0d8d1ed590c6fc337aa4539892305fc4f0502c74c7bb5dfa7
 SHA512:
-  metadata.gz: 00fa3449d0395073e4b5277d105b5331c19e4290e9a8972baa80bc80aeee6ef63effcf94bc0484ced2ea79bb33900467bd2a68ee85b8a8d2796af317a9ef9340
-  data.tar.gz: 263e855471164f79df4acafb32116b752cd3945d2439fc29eea4540ee2c6360d80d472caf2996475a4b3398a68d0361d907e2f5a8844ca4f2cd123da02706155
+  metadata.gz: 392ac958aeb012d4e1bf48daadbe168cf8e0cabd095fd79d155ed8fa4cc14cd8967974d9ee0085e4cf36ac9b78111d09da6e631fe21f97fd5be2136200fd56af
+  data.tar.gz: e3e76a87549ad98f9bed153746035fc1c6672ec30754eb1943a54d43cf2dbe253cbc3bb4235c4b11c9b98544856ff432cbd067af36bada6a12422d835426acca

data/lib/multiwoven/integrations/core/source_connector.rb

@@ -13,6 +13,12 @@ module Multiwoven
 
       private
 
+      # This needs to be implemented as private method
+      # In every source connector. This will be used for model preview
+      def create_connection(connector_config)
+        # return a connection to the client's source
+      end
+
       # This needs to be implemented as private method
       # In every source connector. This will be used for model preview
       def query(connection, query)

data/lib/multiwoven/integrations/destination/iterable/client.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module Iterable
+        include Multiwoven::Integrations::Core
+        class Client < DestinationConnector
+          MAX_CHUNK_SIZE = 10
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            initialize_client(connection_config)
+            channels = ::Iterable::Channels.new
+            response = channels.all
+            if response.success?
+              success_status
+            else
+              failure_status(nil)
+            end
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "ITERABLE:CHECK_CONNECTION:EXCEPTION",
+                               type: "error"
+                             })
+            failure_status(e)
+          end
+
+          def discover(_connection_config = nil)
+            catalog = build_catalog(load_catalog)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception(
+              "ITERABLE:DISCOVER:EXCEPTION",
+              "error",
+              e
+            )
+          end
+
+          def write(sync_config, records, action = "create")
+            @action = sync_config.stream.action || action
+            connection_config = sync_config.destination.connection_specification.with_indifferent_access
+            initialize_client(connection_config)
+            process_records(records, sync_config.stream)
+          rescue StandardError => e
+            handle_exception("ITERABLE:WRITE:EXCEPTION", "error", e)
+          end
+
+          private
+
+          def initialize_client(connection_config)
+            ::Iterable.configure do |config|
+              config.token = connection_config[:api_key]
+            end
+          end
+
+          def process_records(records, stream)
+            write_success = 0
+            write_failure = 0
+            records.each do |record_object|
+              record = extract_data(record_object, stream.json_schema[:properties])
+              response = process_stream(record, stream)
+              if response.success?
+                write_success += 1
+              else
+                write_failure += 1
+              end
+            rescue StandardError => e
+              handle_exception("ITERABLE:WRITE:EXCEPTION", "error", e)
+              write_failure += 1
+            end
+            tracking_message(write_success, write_failure)
+          end
+
+          def process_stream(record, stream)
+            klass = ::Iterable.const_get(stream.name).new(*initialize_params(stream, record))
+            item_attrs = initialize_attribute(stream, record)
+            if stream.name == "CatalogItems"
+              klass.send(@action, item_attrs)
+            else
+              klass.send(@action)
+            end
+          end
+
+          def initialize_params(stream, record)
+            if stream.name == "CatalogItems"
+              [record[:catalog_name], record[:item_id]]
+            else
+              [record[:catalog]]
+            end
+          end
+
+          def initialize_attribute(stream, record)
+            if stream.name == "CatalogItems"
+              JSON.parse(record[:item_attribute])
+            else
+              {}
+            end
+          end
+
+          def tracking_message(success, failure)
+            Multiwoven::Integrations::Protocol::TrackingMessage.new(
+              success: success, failed: failure
+            ).to_multiwoven_message
+          end
+
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/iterable/config/catalog.json

@@ -0,0 +1,47 @@
+{
+  "request_rate_limit": 6000,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": [
+    {
+      "name": "Catalogs",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "properties": {
+          "catalog": {
+            "type": "string"
+          }
+        }
+      },
+      "response_type": "application/json",
+      "supported_sync_modes": ["incremental"],
+      "source_defined_cursor": true
+    },
+    {
+      "name": "CatalogItems",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "required": ["catalog_name", "item_id", "item_attribute"],
+        "properties": {
+          "catalog_name": {
+            "type": "string"
+          },
+          "item_id": {
+            "type": "string"
+          },
+          "item_attribute": {
+            "type": "string",
+            "description": "String must be in json format"
+          }
+        }
+      },
+      "response_type": "application/json",
+      "supported_sync_modes": ["incremental"],
+      "source_defined_cursor": true,
+      "default_cursor_field": ["updated"],
+      "source_defined_primary_key": [["catalog_name"]]
+    }
+  ]
+}

data/lib/multiwoven/integrations/destination/iterable/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Iterable",
+    "title": "Iterable",
+    "connector_type": "destination",
+    "category": "Marketing Automation",
+    "documentation_url": "https://docs.multiwoven.com/destinations/iterable",
+    "github_issue_label": "destination-iterable",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/destination/iterable/config/spec.json

@@ -0,0 +1,19 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/destination/iterable",
+  "stream_type": "static",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Iterable",
+    "type": "object",
+    "required": ["api_key"],
+    "properties": {
+      "api_key": {
+        "description": "The Iterable API key.",
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "api_key",
+        "order": 0
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/destination/iterable/icon.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 25.4.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.0" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+     viewBox="0 0 292 193.6" style="enable-background:new 0 0 292 193.6;" xml:space="preserve">
+<style type="text/css">
+	.st0{opacity:0.25;fill:#59C1A7;enable-background:new    ;}
+	.st1{opacity:0.25;fill:#36C3F2;enable-background:new    ;}
+	.st2{opacity:0.15;fill:#EF3D55;enable-background:new    ;}
+	.st3{opacity:0.15;fill:#6A266D;enable-background:new    ;}
+	.st4{fill:#6A266D;}
+	.st5{fill:#36C3F2;}
+	.st6{fill:#59C1A7;}
+	.st7{fill:#EF3D55;}
+	.st8{fill:#333332;}
+	.st9{fill:none;}
+</style>
+    <g id="Lines">
+
+		<rect x="140.4" y="74.4" transform="matrix(0.7071 -0.7071 0.7071 0.7071 -12.7722 138.3672)" class="st0" width="40.4" height="20.3"/>
+
+        <rect x="150.3" y="35.6" transform="matrix(0.7071 -0.7071 0.7071 0.7071 7.5244 129.821)" class="st1" width="20.3" height="40.4"/>
+
+        <rect x="121.3" y="64.4" transform="matrix(0.7071 -0.7071 0.7071 0.7071 -21.341 117.7495)" class="st2" width="20.3" height="40.4"/>
+
+        <rect x="111.9" y="45" transform="matrix(0.7071 -0.7071 0.7071 0.7071 -0.2944 109.5877)" class="st3" width="40.4" height="20.4"/>
+</g>
+    <g id="Nodes">
+	<circle class="st4" cx="146" cy="41.3" r="10.2"/>
+        <circle class="st5" cx="174.9" cy="70.4" r="10.2"/>
+        <circle class="st6" cx="146" cy="99.3" r="10.2"/>
+        <circle class="st7" cx="117.1" cy="70.3" r="10.2"/>
+</g>
+    <g id="Iterable">
+	<path class="st8" d="M116.7,139.6h-12.5c-0.2,0-0.4,0.2-0.4,0.4v22c0,0.2,0.2,0.4,0.4,0.4h12.5c0.2,0,0.4-0.2,0.4-0.4v-3.5
+		c0-0.2-0.2-0.4-0.4-0.4h-7.6c-0.4,0-0.8-0.3-0.8-0.8c0,0,0,0,0,0v-3.6c0-0.4,0.3-0.8,0.8-0.8c0,0,0,0,0,0h7c0.2,0,0.4-0.2,0.4-0.4
+		v-3.1c0-0.2-0.2-0.4-0.4-0.4h-7c-0.4,0-0.8-0.3-0.8-0.8c0,0,0,0,0,0v-3.4c0-0.4,0.3-0.8,0.8-0.8c0,0,0,0,0,0h7.6
+		c0.2,0,0.4-0.2,0.4-0.4v-3.5C117.1,139.8,116.9,139.6,116.7,139.6C116.7,139.6,116.7,139.6,116.7,139.6z"/>
+        <path class="st8" d="M168.6,162.4c0.1,0,0.3-0.1,0.3-0.2c0.1-0.1,0.1-0.3,0-0.4l-8.8-22c-0.1-0.2-0.2-0.3-0.4-0.3h-2.2
+		c-0.2,0-0.3,0.1-0.4,0.3l-8.8,22c-0.1,0.1-0.1,0.3,0,0.4c0.1,0.1,0.2,0.2,0.3,0.2h4c0.2,0,0.3-0.1,0.4-0.3l1.1-3
+		c0.1-0.3,0.4-0.5,0.8-0.5h7.4c0.3,0,0.6,0.2,0.8,0.5l1.1,3c0.1,0.2,0.2,0.3,0.4,0.3H168.6z M161,154.3c-0.1,0.2-0.4,0.3-0.6,0.3
+		h-3.5c-0.3,0-0.5-0.1-0.6-0.3c-0.1-0.2-0.2-0.5-0.1-0.7l1.7-4.8c0.1-0.4,0.6-0.6,1-0.5c0.2,0.1,0.4,0.3,0.5,0.5l1.7,4.8
+		C161.2,153.9,161.1,154.1,161,154.3z"/>
+        <path class="st8" d="M212.1,162.4c0.2,0,0.4-0.2,0.4-0.4v-3.5c0-0.2-0.2-0.4-0.4-0.4h-6.5c-0.4,0-0.8-0.3-0.8-0.8c0,0,0,0,0,0V140
+		c0-0.2-0.2-0.4-0.4-0.4h-3.6c-0.2,0-0.4,0.2-0.4,0.4v22c0,0.2,0.2,0.4,0.4,0.4L212.1,162.4z"/>
+        <path class="st8" d="M176.7,162.4h8.6c1.3,0,2.7-0.3,3.8-0.9c2-1.1,3.3-3.2,3.3-5.6c0.1-2.4-1.3-4.6-3.4-5.6
+		c-0.1,0-0.1-0.1-0.1-0.2V150l0.1-0.1c1.3-1.1,2-2.7,2-4.3c0-1-0.3-2.1-0.8-3c-0.5-0.9-1.3-1.7-2.3-2.2c-1.2-0.6-2.5-0.9-3.8-0.8
+		h-7.3c-0.2,0-0.4,0.2-0.5,0.4c0,0,0,0,0,0V162C176.3,162.2,176.4,162.4,176.7,162.4C176.7,162.4,176.7,162.4,176.7,162.4z
+		 M180.7,144.6c0-0.4,0.3-0.7,0.7-0.7c0,0,0,0,0,0h2c0.8,0,1.5,0.2,2.1,0.7c0.4,0.4,0.7,0.9,0.8,1.4c0.1,0.8-0.2,1.6-0.8,2.1
+		c-0.6,0.5-1.3,0.8-2.1,0.8h-2c-0.4,0-0.7-0.3-0.7-0.7c0,0,0,0,0,0L180.7,144.6z M180.7,153.4c0-0.4,0.3-0.7,0.7-0.7c0,0,0,0,0,0h3
+		c0.8,0,1.6,0.2,2.2,0.8c0.6,0.5,0.9,1.2,0.9,1.9c0,0.8-0.3,1.5-0.8,2c-0.6,0.6-1.4,0.8-2.2,0.8h-3c-0.4,0-0.7-0.3-0.7-0.7l0,0
+		L180.7,153.4z"/>
+        <path class="st8" d="M137.7,154.5c-0.2-0.4-0.1-0.9,0.3-1.1c0,0,0,0,0,0c1-0.6,1.8-1.4,2.4-2.4c0.7-1.2,1.1-2.6,1-4
+		c0-1.3-0.4-2.6-1-3.7c-0.6-1.1-1.6-2.1-2.7-2.8c-1.2-0.7-2.7-1.1-4.1-1.1h-7c-0.2,0-0.4,0.2-0.4,0.4v22c0,0.2,0.2,0.4,0.4,0.4h3.6
+		c0.2,0,0.4-0.2,0.4-0.4v-6.6c0-0.4,0.3-0.8,0.8-0.8c0,0,0,0,0,0h1.1c0.3,0,0.6,0.2,0.7,0.4l3.8,7.2c0.1,0.1,0.2,0.2,0.4,0.2h4.2
+		c0.2,0,0.4-0.2,0.4-0.4c0-0.1,0-0.1,0-0.2L137.7,154.5z M135.8,149.4c-0.7,0.6-1.6,0.9-2.5,0.9h-1.8c-0.4,0-0.8-0.3-0.8-0.8
+		c0,0,0,0,0,0v-4.7c0-0.4,0.3-0.8,0.8-0.8c0,0,0,0,0,0h1.8c0.9,0,1.8,0.3,2.5,0.9C137.1,146.1,137.1,148.1,135.8,149.4
+		C135.9,149.4,135.8,149.4,135.8,149.4L135.8,149.4z"/>
+        <path class="st8" d="M95.4,139.6H81c-0.2,0-0.4,0.2-0.4,0.4l0,0v3.5c0,0.2,0.2,0.4,0.4,0.4h4.2c0.4,0,0.8,0.3,0.8,0.8c0,0,0,0,0,0
+		l0,0v17.3c0,0.2,0.2,0.4,0.4,0.4h3.7c0.2,0,0.4-0.2,0.4-0.4l0,0v-17.4c0-0.4,0.3-0.8,0.8-0.8c0,0,0,0,0,0h4.1
+		c0.2,0,0.4-0.2,0.4-0.4V140C95.8,139.8,95.6,139.6,95.4,139.6L95.4,139.6z"/>
+        <path class="st8" d="M233.4,158.1h-7.6c-0.4,0-0.8-0.3-0.8-0.8c0,0,0,0,0,0v-3.6c0-0.4,0.3-0.8,0.8-0.8c0,0,0,0,0,0h7
+		c0.2,0,0.4-0.2,0.4-0.4v-3.1c0-0.2-0.2-0.4-0.4-0.4h-7c-0.4,0-0.8-0.3-0.8-0.7c0,0,0,0,0,0v-3.4c0-0.4,0.3-0.8,0.8-0.8c0,0,0,0,0,0
+		h7.6c0.2,0,0.4-0.2,0.4-0.4v-3.5c0-0.2-0.2-0.4-0.4-0.4h-12.5c-0.2,0-0.4,0.2-0.4,0.4v22c0,0.2,0.2,0.4,0.4,0.4h12.5
+		c0.2,0,0.4-0.2,0.4-0.4v-3.5C233.8,158.3,233.6,158.1,233.4,158.1z"/>
+        <path class="st8" d="M72.1,158.1h-3.6c-0.4,0-0.8-0.3-0.8-0.8c0,0,0,0,0,0l0,0v-12.6c0-0.4,0.3-0.8,0.8-0.8c0,0,0,0,0,0h3.6
+		c0.2,0,0.4-0.2,0.4-0.4l0,0V140c0-0.2-0.2-0.4-0.4-0.4H58.6c-0.2,0-0.4,0.2-0.4,0.4l0,0v3.5c0,0.2,0.2,0.4,0.4,0.4h3.8
+		c0.4,0,0.8,0.3,0.8,0.8c0,0,0,0,0,0v12.6c0,0.4-0.4,0.8-0.8,0.8h-3.8c-0.2,0-0.4,0.2-0.4,0.4l0,0v3.5c0,0.2,0.2,0.4,0.4,0.4h4.9
+		c0,0,0.1,0,0.1,0h3.7c0,0,0.1,0,0.1,0h4.8c0.2,0,0.4-0.2,0.4-0.4l0,0v-3.5C72.6,158.3,72.4,158.1,72.1,158.1L72.1,158.1z"/>
+</g>
+    <rect x="50" y="25.3" class="st9" width="192" height="143"/>
+</svg>

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.75"
+    VERSION = "0.2.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -13,6 +13,7 @@ module Multiwoven
       SalesforceConsumerGoodsCloud
       AwsAthena
       Clickhouse
+      AmazonS3
     ].freeze
 
     ENABLED_DESTINATIONS = %w[
@@ -29,6 +30,7 @@ module Multiwoven
       Postgresql
       Http
       Zendesk
+      Iterable
     ].freeze
   end
 end

data/lib/multiwoven/integrations/source/amazon_s3/client.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module AmazonS3
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      DISCOVER_QUERY = "SELECT * FROM S3Object LIMIT 1;"
+
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        client = config_aws(connection_config)
+        client.get_bucket_policy_status({ bucket: connection_config[:bucket] })
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        conn = create_connection(connection_config)
+        # If pulling from multiple files, all files must have the same schema
+        path = build_path(connection_config[:path])
+        full_path = "s3://#{connection_config[:bucket]}/#{path}*.#{connection_config[:file_type]}"
+        records = get_results(conn, "DESCRIBE SELECT * FROM '#{full_path}';")
+        columns = build_discover_columns(records)
+        streams = [Multiwoven::Integrations::Protocol::Stream.new(name: full_path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
+        catalog = Catalog.new(streams: streams)
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, { context: "AMAZONS3:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        conn = create_connection(connection_config)
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        query(conn, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AMAZONS3:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      # DuckDB
+      def create_connection(connection_config)
+        conn = DuckDB::Database.open.connect
+        # Set up S3 configuration
+        secret_query = "
+              CREATE SECRET amazons3_source (
+              TYPE S3,
+              KEY_ID '#{connection_config[:access_id]}',
+              SECRET '#{connection_config[:secret_access]}',
+              REGION '#{connection_config[:region]}'
+          );
+        "
+        get_results(conn, secret_query)
+        conn
+      end
+
+      def build_path(path)
+        path = "#{path}/" if !path.to_s.strip.empty? && path[-1] != "/"
+        path
+      end
+
+      def get_results(conn, query)
+        results = conn.query(query)
+        hash_array_values(results)
+      end
+
+      def query(conn, query)
+        records = get_results(conn, query)
+        records.map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+
+      def hash_array_values(describe)
+        keys = describe.columns.map(&:name)
+        describe.map do |row|
+          Hash[keys.zip(row)]
+        end
+      end
+
+      def build_discover_columns(describe_results)
+        describe_results.map do |row|
+          type = column_schema_helper(row["column_type"])
+          {
+            column_name: row["column_name"],
+            type: type
+          }
+        end
+      end
+
+      def column_schema_helper(column_type)
+        case column_type
+        when "VARCHAR", "BIT", "DATE", "TIME", "TIMESTAMP", "UUID"
+          "string"
+        when "DOUBLE"
+          "number"
+        when "BIGINT", "HUGEINT", "INTEGER", "SMALLINT"
+          "integer"
+        when "BOOLEAN"
+          "boolean"
+        end
+      end
+
+      # AWS SDK
+      def config_aws(config)
+        config = config.with_indifferent_access
+        Aws.config.update({
+                            region: config[:region],
+                            credentials: Aws::Credentials.new(config[:access_id], config[:secret_access])
+                          })
+        config.with_indifferent_access
+        Aws::S3::Client.new
+      end
+
+      def build_select_content_options(config, query)
+        config = config.with_indifferent_access
+        bucket_name = config[:bucket]
+        file_key = config[:file_key]
+        file_type = config[:file_type]
+        options = {
+          bucket: bucket_name,
+          key: file_key,
+          expression_type: "SQL",
+          expression: query,
+          output_serialization: {
+            json: {}
+          }
+        }
+        if file_type == "parquet"
+          options[:input_serialization] = {
+            parquet: {}
+          }
+        elsif file_type == "csv"
+          options[:input_serialization] = {
+            csv: { file_header_info: "USE" }
+          }
+        end
+        options
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/amazon_s3/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "AmazonS3",
+    "title": "Amazon S3",
+    "connector_type": "source",
+    "category": "Data Lake",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-amazons3",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json

@@ -0,0 +1,51 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/amazons3",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "AmazonS3",
+    "type": "object",
+    "required": ["region", "bucket", "access_id", "secret_access", "file_type"],
+    "properties": {
+      "region": {
+        "description": "AWS region",
+        "examples": ["us-east-2"],
+        "type": "string",
+        "title": "Region",
+        "order": 1
+      },
+      "access_id": {
+        "type": "string",
+        "title": "Access Id",
+        "order": 2
+      },
+      "secret_access": {
+        "type": "string",
+        "title": "Secret Access",
+        "multiwoven_secret": true,
+        "order": 3
+      },
+      "bucket": {
+        "description": "Bucket Name",
+        "type": "string",
+        "title": "Bucket",
+        "order": 4
+      },
+      "path": {
+        "description": "Path to csv or parquet files",
+        "examples": ["/path/to/files"],
+        "type": "string",
+        "title": "Path",
+        "order": 5
+      },
+      "file_type": {
+        "description": "The type of file to read",
+        "type": "string",
+        "title": "File Type",
+        "enum": ["csv", "parquet"],
+        "order": 6
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/amazon_s3/icon.svg

@@ -0,0 +1,34 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="428" height="512" viewBox="0 0 428 512">
+  <defs>
+    <style>
+      .cls-1 {
+        fill: #e25444;
+      }
+
+      .cls-1, .cls-2, .cls-3 {
+        fill-rule: evenodd;
+      }
+
+      .cls-2 {
+        fill: #7b1d13;
+      }
+
+      .cls-3 {
+        fill: #58150d;
+      }
+    </style>
+  </defs>
+  <path class="cls-1" d="M378,99L295,257l83,158,34-19V118Z"/>
+  <path class="cls-2" d="M378,99L212,118,127.5,257,212,396l166,19V99Z"/>
+  <path class="cls-3" d="M43,99L16,111V403l27,12L212,257Z"/>
+  <path class="cls-1" d="M42.637,98.667l169.587,47.111V372.444L42.637,415.111V98.667Z"/>
+  <path class="cls-3" d="M212.313,170.667l-72.008-11.556,72.008-81.778,71.83,81.778Z"/>
+  <path class="cls-3" d="M284.143,159.111l-71.919,11.733-71.919-11.733V77.333"/>
+  <path class="cls-3" d="M212.313,342.222l-72.008,13.334,72.008,70.222,71.83-70.222Z"/>
+  <path class="cls-2" d="M212,16L140,54V159l72.224-20.333Z"/>
+  <path class="cls-2" d="M212.224,196.444l-71.919,7.823V309.105l71.919,8.228V196.444Z"/>
+  <path class="cls-2" d="M212.224,373.333L140.305,355.3V458.363L212.224,496V373.333Z"/>
+  <path class="cls-1" d="M284.143,355.3l-71.919,18.038V496l71.919-37.637V355.3Z"/>
+  <path class="cls-1" d="M212.224,196.444l71.919,7.823V309.105l-71.919,8.228V196.444Z"/>
+  <path class="cls-1" d="M212,16l72,38V159l-72-20V16Z"/>
+</svg>

data/lib/multiwoven/integrations.rb

@@ -27,6 +27,9 @@ require "zip"
 require "zendesk_api"
 require "faraday"
 require "base64"
+require "aws-sdk-s3"
+require "duckdb"
+require "iterable-api-client"
 
 # Service
 require_relative "integrations/config"
@@ -54,6 +57,7 @@ require_relative "integrations/source/databricks/client"
 require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
 require_relative "integrations/source/aws_athena/client"
 require_relative "integrations/source/clickhouse/client"
+require_relative "integrations/source/amazon_s3/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"
@@ -69,6 +73,7 @@ require_relative "integrations/destination/sftp/client"
 require_relative "integrations/destination/postgresql/client"
 require_relative "integrations/destination/http/client"
 require_relative "integrations/destination/zendesk/client"
+require_relative "integrations/destination/iterable/client"
 
 module Multiwoven
   module Integrations

data/multiwoven-integrations.gemspec

@@ -36,14 +36,17 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "activesupport"
   spec.add_runtime_dependency "async-websocket"
   spec.add_runtime_dependency "aws-sdk-athena"
+  spec.add_runtime_dependency "aws-sdk-s3"
   spec.add_runtime_dependency "csv"
   spec.add_runtime_dependency "dry-schema"
   spec.add_runtime_dependency "dry-struct"
   spec.add_runtime_dependency "dry-types"
+  spec.add_runtime_dependency "duckdb"
   spec.add_runtime_dependency "git"
   spec.add_runtime_dependency "google-apis-sheets_v4"
   spec.add_runtime_dependency "google-cloud-bigquery"
   spec.add_runtime_dependency "hubspot-api-client"
+  spec.add_runtime_dependency "iterable-api-client"
   spec.add_runtime_dependency "net-sftp"
   spec.add_runtime_dependency "pg"
   spec.add_runtime_dependency "rake"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.75
+  version: 0.2.0
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-06-06 00:00:00.000000000 Z
+date: 2024-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: duckdb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +192,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: iterable-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: net-sftp
   requirement: !ruby/object:Gem::Requirement
@@ -456,6 +498,11 @@ files:
 - lib/multiwoven/integrations/destination/hubspot/config/meta.json
 - lib/multiwoven/integrations/destination/hubspot/config/spec.json
 - lib/multiwoven/integrations/destination/hubspot/icon.svg
+- lib/multiwoven/integrations/destination/iterable/client.rb
+- lib/multiwoven/integrations/destination/iterable/config/catalog.json
+- lib/multiwoven/integrations/destination/iterable/config/meta.json
+- lib/multiwoven/integrations/destination/iterable/config/spec.json
+- lib/multiwoven/integrations/destination/iterable/icon.svg
 - lib/multiwoven/integrations/destination/klaviyo/client.rb
 - lib/multiwoven/integrations/destination/klaviyo/config/catalog.json
 - lib/multiwoven/integrations/destination/klaviyo/config/meta.json
@@ -500,6 +547,10 @@ files:
 - lib/multiwoven/integrations/protocol/protocol.rb
 - lib/multiwoven/integrations/rollout.rb
 - lib/multiwoven/integrations/service.rb
+- lib/multiwoven/integrations/source/amazon_s3/client.rb
+- lib/multiwoven/integrations/source/amazon_s3/config/meta.json
+- lib/multiwoven/integrations/source/amazon_s3/config/spec.json
+- lib/multiwoven/integrations/source/amazon_s3/icon.svg
 - lib/multiwoven/integrations/source/aws_athena/client.rb
 - lib/multiwoven/integrations/source/aws_athena/config/meta.json
 - lib/multiwoven/integrations/source/aws_athena/config/spec.json