multiwoven-integrations 0.1.66 → 0.1.68

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d41f10c87a8d9a6b6d5b3f04952afcc4c906266d79a2c0935b4468d6ccb4592e
-  data.tar.gz: 5a09773b98f94d2ecf401ee8c46affe0684704b65a5d0dd73838418dbcf2d78d
+  metadata.gz: 77816f54f13495e303382ef4c1e2e932472475d31e0f486d3dcc0535e561f36d
+  data.tar.gz: 854a6871e65a3ad4764548ea14793a04c02d42cde0ed31712c252aff960134ab
 SHA512:
-  metadata.gz: 3cff2743efecd6b06a1d7470d11ef3410adea1bd81ee5f028468d964a6630f8f3bafb4f5b7729aaa514afbb4608cacd0ee44723cb27863facc88dd0c5143c773
-  data.tar.gz: bbfc548d9a8e0df2efcf466a728fd3d13e6eb3a8fe49ed7844d7c55249764b515e57f5c1a915601ec3e570195835b8a3162d5f3b9bd08b092c652c0990e8e015
+  metadata.gz: 0de8475fa0b27194c1a61b58d56f11fe3a5df8058c6065664a2c67c7af5c5d4c709aab352c0be4c7f53b9f62fda1031a47f1590844755b54897b38a652173306
+  data.tar.gz: dce5ca510e72bccf871075a1a5dd523db56961454b8d4ba987e6037b219f07683a79b1dabc03e51e9b533b18af52ce048ad60684fb2f9ee6e619c9fac272316b

data/lib/multiwoven/integrations/config.rb

@@ -3,10 +3,11 @@
 module Multiwoven
   module Integrations
     class Config
-      attr_accessor :logger
+      attr_accessor :logger, :exception_reporter
 
       def initialize(params = {})
         @logger = params[:logger]
+        @exception_reporter = params[:exception_reporter]
       end
     end
   end

data/lib/multiwoven/integrations/core/utils.rb

@@ -48,6 +48,11 @@ module Multiwoven
         Integrations::Service.logger
       end
 
+      def report_exception(exception)
+        reporter = Integrations::Service.exception_reporter
+        reporter&.report(exception)
+      end
+
       def create_log_message(context, type, exception)
         Integrations::Protocol::LogMessage.new(
           name: context,
@@ -60,7 +65,7 @@ module Multiwoven
         logger.error(
           "#{context}: #{exception.message}"
         )
-
+        report_exception(exception)
         create_log_message(context, type, exception)
       end
 

data/lib/multiwoven/integrations/destination/zendesk/config/catalog.json

@@ -1,110 +1,110 @@
 {
-    "request_rate_limit": 700,
-    "request_rate_limit_unit": "minute",
-    "request_rate_concurrency": 10,
-    "streams": [
-        {
-            "name": "tickets",
-            "action": "create",
-            "json_schema": {
-                "type": "object",
-                "additionalProperties": true,
-                "properties": {
-                    "id": {
-                        "type": "integer"
-                    },
-                    "subject": {
-                        "type": "string"
-                    },
-                    "description": {
-                        "type": "string"
-                    },
-                    "status": {
-                        "type": "string"
-                    },
-                    "priority": {
-                        "type": "string"
-                    },
-                    "requester_id": {
-                        "type": "integer"
-                    },
-                    "assignee_id": {
-                        "type": "integer"
-                    },
-                    "tags": {
-                        "type": "array",
-                        "items": {
-                            "type": "string"
-                        }
-                    },
-                    "created_at": {
-                        "type": "string",
-                        "format": "date-time"
-                    },
-                    "updated_at": {
-                        "type": "string",
-                        "format": "date-time"
-                    }
-                }
-            },
-            "supported_sync_modes": [
-                "incremental"
-            ],
-            "source_defined_cursor": true,
-            "default_cursor_field": [
-                "updated_at"
-            ],
-            "source_defined_primary_key": [
-                [
-                    "id"
-                ]
-            ]
-        },
-        {
-            "name": "users",
-            "action": "create",
-            "json_schema": {
-                "type": "object",
-                "additionalProperties": true,
-                "properties": {
-                    "id": {
-                        "type": "integer"
-                    },
-                    "name": {
-                        "type": "string"
-                    },
-                    "email": {
-                        "type": "string"
-                    },
-                    "role": {
-                        "type": "string"
-                    },
-                    "last_login_at": {
-                        "type": "string",
-                        "format": "date-time"
-                    },
-                    "created_at": {
-                        "type": "string",
-                        "format": "date-time"
-                    },
-                    "updated_at": {
-                        "type": "string",
-                        "format": "date-time"
-                    }
-                }
-            },
-            "supported_sync_modes": [
-                "incremental"
-            ],
-            "source_defined_cursor": true,
-            "default_cursor_field": [
-                "updated_at"
-            ],
-            "source_defined_primary_key": [
-                [
-                    "id"
-                ]
-            ]
+  "request_rate_limit": 700,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": [
+    {
+      "name": "tickets",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "properties": {
+          "id": {
+            "type": "integer"
+          },
+          "subject": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          },
+          "status": {
+            "type": "string"
+          },
+          "priority": {
+            "type": "string"
+          },
+          "requester_id": {
+            "type": "integer"
+          },
+          "assignee_id": {
+            "type": "integer"
+          },
+          "tags": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "created_at": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "updated_at": {
+            "type": "string",
+            "format": "date-time"
+          }
         }
-    ]
+      },
+      "supported_sync_modes": [
+        "incremental"
+      ],
+      "source_defined_cursor": true,
+      "default_cursor_field": [
+        "updated_at"
+      ],
+      "source_defined_primary_key": [
+        [
+          "id"
+        ]
+      ]
+    },
+    {
+      "name": "users",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "properties": {
+          "id": {
+            "type": "integer"
+          },
+          "name": {
+            "type": "string"
+          },
+          "email": {
+            "type": "string"
+          },
+          "role": {
+            "type": "string"
+          },
+          "last_login_at": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "created_at": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "updated_at": {
+            "type": "string",
+            "format": "date-time"
+          }
+        }
+      },
+      "supported_sync_modes": [
+        "incremental"
+      ],
+      "source_defined_cursor": true,
+      "default_cursor_field": [
+        "updated_at"
+      ],
+      "source_defined_primary_key": [
+        [
+          "id"
+        ]
+      ]
+    }
+  ]
 }

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.66"
+    VERSION = "0.1.68"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -11,6 +11,7 @@ module Multiwoven
       Postgresql
       Databricks
       SalesforceConsumerGoodsCloud
+      AwsAthena
     ].freeze
 
     ENABLED_DESTINATIONS = %w[
@@ -25,8 +26,8 @@ module Multiwoven
       SalesforceConsumerGoodsCloud
       Sftp
       Postgresql
-      Zendesk
       Http
+      Zendesk
     ].freeze
   end
 end

data/lib/multiwoven/integrations/service.rb

@@ -28,6 +28,10 @@ module Multiwoven
           config.logger || default_logger
         end
 
+        def exception_reporter
+          config.exception_reporter
+        end
+
         def config
           @config ||= Config.new
         end

data/lib/multiwoven/integrations/source/aws_athena/client.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module AwsAthena
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        athena_client = create_connection(connection_config)
+        athena_client.list_work_groups
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:schema]}' ORDER BY table_name, ordinal_position;"
+        db = create_connection(connection_config)
+        results = query_execution(db, query)
+        catalog = Catalog.new(streams: create_streams(results))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "AWS:ATHENA:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        db = create_connection(connection_config)
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(
+          "AWS:ATHENA:READ:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      private
+
+      def create_connection(connection_config)
+        Aws.config.update({ credentials: Aws::Credentials.new(connection_config[:access_key], connection_config[:secret_access_key]), region: connection_config[:region] })
+        @database = connection_config[:schema]
+        @output_location = connection_config[:output_location]
+        Aws::Athena::Client.new
+      end
+
+      def query_execution(db, query)
+        response = db.start_query_execution(
+          query_string: query,
+          query_execution_context: { database: @database },
+          result_configuration: { output_location: @output_location }
+        )
+        query_execution_id = response[:query_execution_id]
+        loop do
+          response = db.get_query_execution(query_execution_id: query_execution_id)
+          status = response.query_execution.status.state
+          break if %w[SUCCEEDED FAILED CANCELLED].include?(status)
+        end
+        transform_results(db.get_query_results(query_execution_id: query_execution_id))
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def transform_results(results)
+        columns = results.result_set.result_set_metadata.column_info.map(&:name)
+        rows = results.result_set.rows.map do |row|
+          row.data.map(&:var_char_value)
+        end
+        rows.map { |row| columns.zip(row).to_h }
+      end
+
+      def query(db, query)
+        records = []
+        query_execution(db, query).map do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry["table_name"]
+          column_data = {
+            column_name: entry["column_name"],
+            data_type: entry["data_type"],
+            is_nullable: entry["is_nullable"] == "YES"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/aws_athena/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "AwsAthena",
+    "title": "AWS Athena",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-aws-athena",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/aws_athena/config/spec.json

@@ -0,0 +1,63 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/athena",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "AWS Athena",
+    "type": "object",
+    "required": ["access_key", "secret_access_key", "region", "workgroup", "catalog", "output_location"],
+    "properties": {
+      "access_key": {
+        "description": "The AWS Access Key ID to use for authentication.",
+        "examples": ["AWSATHENAACCESS"],
+        "type": "string",
+        "title": "Personal Access Key",
+        "order": 0
+      },
+      "secret_access_key": {
+        "description": "The AWS Secret Access Key to use for authentication.",
+        "examples": ["AWSATHENASECRET"],
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Secret Access Key",
+        "order": 1
+      },
+      "region": {
+        "description": "AWS region where Athena is located.",
+        "examples": ["ATHENA_REGION"],
+        "type": "string",
+        "title": "Secret Access Key",
+        "order": 2
+      },
+      "workgroup": {
+        "description": "The Athena workgroup you previously set up in AWS.",
+        "examples": ["ATHENA_WORKGROUP"],
+        "type": "string",
+        "title": "Workgroup",
+        "order": 3
+      },
+      "catalog": {
+        "description": "The Data catalog name within Athena.",
+        "examples": ["ATHENA_CATALOG"],
+        "type": "string",
+        "title": "Catalog",
+        "order": 4
+      },
+      "schema": {
+        "description": "The specific Athena database/schema to connect to.",
+        "examples": ["ATHENA_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 5
+      },
+      "output_location": {
+        "description": "S3 path for query output.",
+        "examples": ["s3://example-bucket-name/query-results/"],
+        "type": "string",
+        "title": "Query",
+        "order": 6
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/aws_athena/icon.svg

@@ -0,0 +1,22 @@
+<svg height="100" viewBox="0 0 100 100" width="100" xmlns="http://www.w3.org/2000/svg">
+    <path d="m84 64.68-34-4.19-34 4.19 34 26.62z" fill="#fcbf92"/>
+    <path d="m16 64.68 34 10.17v16.45l-34-17.05z" fill="#9d5025"/>
+    <path d="m84 64.68-34 10.17v16.45l34-17.05z" fill="#f58534"/>
+    <path d="m20.39 50.22h-4.39v9.67l4.39.8 4.88-5.18z" fill="#9d5025"/>
+    <path d="m25.27 60.29-4.88.4v-10.47h4.88z" fill="#f58534"/>
+    <path d="m28.56 38.16-5.78 1.09v21.94l5.78 1.2 5.29-12.17z" fill="#9d5025"/>
+    <path d="m33.85 61.79-5.29.6v-24.23l5.29.6z" fill="#f58534"/>
+    <path d="m39.83 43.04-7.98.8v19.24l7.98 1.6 8.47-10.87z" fill="#9d5025"/>
+    <path d="m45.21 63.78-5.38.9v-21.64l5.38.4z" fill="#f58534"/>
+    <g fill="#9d5025">
+        <path d="m50 25.59-5.38 1.7v38.29l5.38 1.09 5.38-20.54z"/>
+        <path d="m54.98 63.78 5.19.9 7.98-14.46-7.98-14.36-5.19.8z"/>
+        <path d="m68.15 50.22-7.98-14.36-5.19.8"/>
+        <path d="m66.05 61.79 5.39.6 5.78-18.25-5.78-18.15-5.39 1.3z"/>
+        <path d="m74.73 60.29 4.88.4 4.39-18.05-4.39-18.64-4.88 1.2z"/>
+    </g>
+    <path d="m50 25.59 5.38 1.7v38.29l-5.38 1.09z" fill="#f58534"/>
+    <path d="m68.15 37.46-7.98-1.6v28.82l7.98-1.6z" fill="#f58534"/>
+    <path d="m77.22 28.29-5.78-2.3v36.4l5.78-1.2z" fill="#f58534"/>
+    <path d="m84 26.19-4.39-2.19v36.69l4.39-.8z" fill="#f58534"/>
+</svg>

data/lib/multiwoven/integrations.rb

@@ -22,6 +22,7 @@ require "stripe"
 require "net/sftp"
 require "csv"
 require "securerandom"
+require "aws-sdk-athena"
 require "zip"
 require "zendesk_api"
 
@@ -49,6 +50,7 @@ require_relative "integrations/source/bigquery/client"
 require_relative "integrations/source/postgresql/client"
 require_relative "integrations/source/databricks/client"
 require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
+require_relative "integrations/source/aws_athena/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"
@@ -62,8 +64,8 @@ require_relative "integrations/destination/stripe/client"
 require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
 require_relative "integrations/destination/sftp/client"
 require_relative "integrations/destination/postgresql/client"
-require_relative "integrations/destination/zendesk/client"
 require_relative "integrations/destination/http/client"
+require_relative "integrations/destination/zendesk/client"
 
 module Multiwoven
   module Integrations

data/multiwoven-integrations.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "activesupport"
   spec.add_runtime_dependency "async-websocket"
+  spec.add_runtime_dependency "aws-sdk-athena"
   spec.add_runtime_dependency "csv"
   spec.add_runtime_dependency "dry-schema"
   spec.add_runtime_dependency "dry-struct"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.66
+  version: 0.1.68
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-05-14 00:00:00.000000000 Z
+date: 2024-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-athena
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -486,6 +500,10 @@ files:
 - lib/multiwoven/integrations/protocol/protocol.rb
 - lib/multiwoven/integrations/rollout.rb
 - lib/multiwoven/integrations/service.rb
+- lib/multiwoven/integrations/source/aws_athena/client.rb
+- lib/multiwoven/integrations/source/aws_athena/config/meta.json
+- lib/multiwoven/integrations/source/aws_athena/config/spec.json
+- lib/multiwoven/integrations/source/aws_athena/icon.svg
 - lib/multiwoven/integrations/source/bigquery/client.rb
 - lib/multiwoven/integrations/source/bigquery/config/meta.json
 - lib/multiwoven/integrations/source/bigquery/config/spec.json