multiwoven-integrations 0.9.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd6418b9c76b3b111b921a6a4bb9da3f2b332179e583e8f5080e5151518408bf
-  data.tar.gz: 5c153b30244e1d6235c076b39c38ad50cd1e996a8f060b304a01c25b9f841756
+  metadata.gz: 34da17012dfaee9c1275aad7968ac32378d822f53819f71d421cf29323fca12a
+  data.tar.gz: 9a5816f59fa89720d52ec80f8d206b6eb9e303206a9858f80f999c74095fdc46
 SHA512:
-  metadata.gz: f287bf95ad5156f21473c99ff9f711869628b7115cc5c27e1b3b2da24dc4ea1060a19d104dc83e72fdf6ecc30fa429a2f03920ecb977afecc00a1fcd9d77de08
-  data.tar.gz: a5ab7543a804567ba9813e24fe73574d4f11eb910d98519fbc847431a1b28ce37eb58597464714b021d86d98252898c59365d87416b859ed46fc8e59fbd161e9
+  metadata.gz: b29fad3dabb2793292a2f101eb2136081051e23bb0b63013bb4d1ee6c7b76e4f8b00e8dc2db05a8ec807fffc8b27dc8034613409872ffe9f5ddfad41fec2c831
+  data.tar.gz: 7de72831e5fe9d5075abf1a5dae5219fcf3732a85f75b105a6b173c2ad0b56ae21e750d45ef8f8f691378a1843634496bc12cd947a550a789f0db81133004774

data/lib/multiwoven/integrations/core/constants.rb

@@ -50,9 +50,6 @@ module Multiwoven
       DATABRICKS_HEALTH_URL  = "https://%<databricks_host>s/api/2.0/serving-endpoints/%<endpoint_name>s"
       DATABRICKS_SERVING_URL = "https://%<databricks_host>s/serving-endpoints/%<endpoint_name>s/invocations"
 
-      AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
-      AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
-
       # HTTP
       HTTP_GET = "GET"
       HTTP_POST = "POST"

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.9.2"
+    VERSION = "0.10.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -17,6 +17,7 @@ module Multiwoven
       MariaDB
       Oracle
       DatabricksModel
+      AwsSagemakerModel
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module AwsSagemakerModel
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        response = @client.describe_endpoint(endpoint_name: connection_config[:endpoint_name])
+        if response.endpoint_status == "InService"
+          success_status
+        else
+          failure_status
+        end
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(_connection_config)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AWS:SAGEMAKER MODEL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        payload = sync_config.model.query
+        create_connection(connection_config)
+        run_model(connection_config, payload)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AWS:SAGEMAKER MODEL:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        @client = Aws::SageMaker::Client.new(
+          region: connection_config[:region],
+          access_key_id: connection_config[:access_key],
+          secret_access_key: connection_config[:secret_access_key]
+        )
+
+        @client_runtime = Aws::SageMakerRuntime::Client.new(
+          region: connection_config[:region],
+          access_key_id: connection_config[:access_key],
+          secret_access_key: connection_config[:secret_access_key]
+        )
+      end
+
+      def run_model(connection_config, payload)
+        response = @client_runtime.invoke_endpoint(
+          endpoint_name: connection_config[:endpoint_name],
+          content_type: "application/json",
+          body: payload
+        )
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, context: "AWS:SAGEMAKER MODEL:RUN_MODEL:EXCEPTION", type: "error")
+      end
+
+      def process_response(response)
+        data = JSON.parse(response.body.read)
+        [RecordMessage.new(data: { response: data }, emitted_at: Time.now.to_i).to_multiwoven_message]
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "AwsSagemakerModel",
+    "title": "AWS Sagemaker Model",
+    "connector_type": "source",
+    "category": "AI Model",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-aws-sagemaker-model",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json

@@ -0,0 +1,50 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/aws_sagemaker-model",
+  "stream_type": "user_defined",
+  "connector_query_type": "ai_ml",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "AWS Sagemaker Model",
+    "type": "object",
+    "required": ["access_key", "secret_access_key", "region", "endpoint_name", "request_format", "response_format"],
+    "properties": {
+      "access_key": {
+        "description": "The AWS Access Key ID to use for authentication",
+        "type": "string",
+        "title": "Personal Access Key",
+        "order": 0
+      },
+      "secret_access_key": {
+        "description": "The AWS Secret Access Key to use for authentication",
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Secret Access Key",
+        "order": 1
+      },
+      "region": {
+        "description": "AWS region",
+        "type": "string",
+        "title": "Region",
+        "order": 2
+      },
+      "endpoint_name": {
+        "description": "Endpoint name for AWS Sagemaker",
+        "type": "string",
+        "title": "Endpoint name",
+        "order": 3
+      },
+      "request_format": {
+        "description": "Sample Request Format",
+        "type": "string",
+        "title": "Request Format",
+        "order": 4
+      },
+      "response_format": {
+        "description": "Sample Response Format",
+        "type": "string",
+        "title": "Response Format",
+        "order": 5
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="80px" height="80px" viewBox="0 0 80 80" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 64 (93537) - https://sketch.com -->
+    <title>Icon-Architecture/64/Arch_AWS-SageMaker_64</title>
+    <desc>Created with Sketch.</desc>
+    <defs>
+        <linearGradient x1="0%" y1="100%" x2="100%" y2="0%" id="linearGradient-1">
+            <stop stop-color="#055F4E" offset="0%"></stop>
+            <stop stop-color="#56C0A7" offset="100%"></stop>
+        </linearGradient>
+    </defs>
+    <g id="Icon-Architecture/64/Arch_AWS-SageMaker_64" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Icon-Architecture-BG/64/Machine-Learning" fill="url(#linearGradient-1)">
+            <rect id="Rectangle" x="0" y="0" width="80" height="80"></rect>
+        </g>
+        <path d="M54.034,26.0335731 C54.034,26.5935654 53.578,27.0495592 53.017,27.0495592 C52.458,27.0495592 52.002,26.5935654 52.002,26.0335731 C52.002,25.4735807 52.458,25.0175869 53.017,25.0175869 C53.578,25.0175869 54.034,25.4735807 54.034,26.0335731 L54.034,26.0335731 Z M48.002,36.000437 C48.002,35.4494445 48.45,35.0004506 49.002,35.0004506 C49.554,35.0004506 50.002,35.4494445 50.002,36.000437 C50.002,36.5514294 49.554,37.0004233 49.002,37.0004233 C48.45,37.0004233 48.002,36.5514294 48.002,36.000437 L48.002,36.000437 Z M48.002,55.0001775 C48.002,54.449185 48.45,54.0001912 49.002,54.0001912 C49.554,54.0001912 50.002,54.449185 50.002,55.0001775 C50.002,55.55117 49.554,56.0001639 49.002,56.0001639 C48.45,56.0001639 48.002,55.55117 48.002,55.0001775 L48.002,55.0001775 Z M58.002,42.000355 C58.002,42.5513475 57.554,43.0003414 57.002,43.0003414 C56.45,43.0003414 56.002,42.5513475 56.002,42.000355 C56.002,41.4493626 56.45,41.0003687 57.002,41.0003687 C57.554,41.0003687 58.002,41.4493626 58.002,42.000355 L58.002,42.000355 Z M65,45.2723104 L59.963,42.3823498 C59.979,42.2563515 60.002,42.1313532 60.002,42.000355 C60.002,40.3463776 58.656,39.000396 57.002,39.000396 C55.347,39.000396 54.002,40.3463776 54.002,42.000355 C54.002,43.6543324 55.347,45.0003141 57.002,45.0003141 C57.801,45.0003141 58.523,44.6813184 59.061,44.1713254 L63.886,46.9392876 L59.555,49.105258 C59.216,49.2752557 59.002,49.621251 59.002,50.0002458 L59.002,58.4411305 L46.983,65.8370295 L41.003,62.4200762 L41.003,56.0001639 L46.186,56.0001639 C46.6,57.161148 47.7,58.0001366 49.002,58.0001366 C50.656,58.0001366 52.002,56.6541549 52.002,55.0001775 C52.002,53.3452001 50.656,52.0002185 49.002,52.0002185 C47.7,52.0002185 46.6,52.838207 46.186,54.0001912 L41.003,54.0001912 L41.003,40.0003823 C41.003,39.6493871 40.818,39.3233916 40.517,39.1423941 L35.516,36.142435 L34.487,37.8574116 L39.003,40.5663746 L39.003,43.5073345 L33.002,48.1232714 L33.002,44.0003277 C33.002,43.6963319 32.864,43.4083358 32.627,43.2193384 L28.002,39.5193889 L28.002,34.535457 L33.556,30.8325075 C33.835,30.6465101 34.002,30.3345143 34.002,30.0005189 L34.002,24.0006008 L32.002,24.0006008 L32.002,29.4655262 L27.013,32.7904808 L22.002,29.4635262 L22.002,21.5746339 L27.002,18.6586738 L27.002,27.0005599 L29.002,27.0005599 L29.002,17.4916897 L33.005,15.1567216 L39.001,18.6156744 L39.002,31.0005052 C39.002,31.3595003 39.194,31.6904958 39.506,31.8684934 L46.042,35.6034424 C46.024,35.7344406 46.002,35.8644388 46.002,36.000437 C46.002,37.6544144 47.347,39.000396 49.002,39.000396 C50.656,39.000396 52.002,37.6544144 52.002,36.000437 C52.002,34.3464595 50.656,33.0004779 49.002,33.0004779 C48.208,33.0004779 47.49,33.3154736 46.953,33.8204667 L41.002,30.4195132 L41.001,18.6176743 L46.964,15.1767213 L58.002,22.5356208 L58.002,25.0005872 L55.851,25.0005872 C55.429,23.844603 54.318,23.0176142 53.017,23.0176142 C51.354,23.0176142 50.002,24.3705958 50.002,26.0335731 C50.002,27.6965504 51.354,29.0495319 53.017,29.0495319 C54.343,29.0495319 55.471,28.1905436 55.875,27.0005599 L58.002,27.0005599 L58.002,30.0005189 C58.002,30.359514 58.194,30.6905095 58.506,30.868507 L65,34.5804564 L65,45.2723104 Z M33.02,65.8370295 L29.867,63.897056 L35.583,59.8141118 L34.421,58.186134 L28.018,62.7590716 L21.002,58.4411305 L21.002,50.5662381 L25.516,47.8572751 L24.487,46.1422985 L19.958,48.8602614 L15.002,46.3822952 L15.001,40.6173739 L20.449,37.8944111 L19.555,36.1054355 L15.001,38.3814044 L15.002,34.5804564 L20.963,31.1745029 L26.002,34.5194572 L26.002,39.4803894 L20.449,43.1673391 L21.555,44.8333163 L26.958,41.2453653 L31.002,44.4803212 L31.002,49.6622504 L26.392,53.207202 L27.611,54.7921804 L39.003,46.0303 L39.003,62.4190762 L33.02,65.8370295 Z M66.496,33.1324761 L60.002,29.4205268 L60.002,22.0006281 C60.002,21.6656327 59.835,21.353637 59.556,21.1686395 L47.556,13.1687487 C47.24,12.9587516 46.832,12.9447518 46.502,13.1347492 L40.004,16.884698 L33.502,13.1347492 C33.19,12.9547517 32.807,12.9547517 32.498,13.1367492 L20.498,20.1366536 C20.19,20.3156511 20.002,20.6446466 20.002,21.0006418 L20.002,29.4205268 L13.506,33.1324761 C13.194,33.3094737 13.002,33.6414692 13.002,34.0004643 L13.002,34.4174586 C13.001,34.4384583 13,34.458458 13,34.4794577 L13,45.3633091 C13,45.3833088 13.001,45.4033086 13.002,45.4223083 L13.002,47.0002868 C13.002,47.3792816 13.216,47.7252769 13.555,47.8942745 L19.002,50.6182374 L19.002,59.0001229 C19.002,59.3471182 19.181,59.6691138 19.477,59.8511113 L32.477,67.851002 C32.638,67.9500007 32.82,68 33.002,68 C33.173,68 33.344,67.9560006 33.498,67.8680018 L40.003,64.1520525 L46.506,67.8680018 C46.821,68.0489993 47.213,68.0419994 47.526,67.851002 L60.526,59.8511113 C60.822,59.6691138 61.002,59.3471182 61.002,59.0001229 L61.002,50.6182374 L66.447,47.8942745 C66.786,47.7252769 67,47.3792816 67,47.0002868 L67,34.0004643 C67,33.6414692 66.807,33.3104737 66.496,33.1324761 L66.496,33.1324761 Z" id="AWS-SageMaker_Icon_64_Squid" fill="#FFFFFF"></path>
+    </g>
+</svg>

data/lib/multiwoven/integrations.rb

@@ -32,6 +32,8 @@ require "duckdb"
 require "iterable-api-client"
 require "aws-sdk-sts"
 require "ruby-oci8"
+require "aws-sdk-sagemaker"
+require "aws-sdk-sagemakerruntime"
 
 # Service
 require_relative "integrations/config"
@@ -63,6 +65,7 @@ require_relative "integrations/source/amazon_s3/client"
 require_relative "integrations/source/maria_db/client"
 require_relative "integrations/source/oracle_db/client"
 require_relative "integrations/source/databrics_model/client"
+require_relative "integrations/source/aws_sagemaker_model/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

data/multiwoven-integrations.gemspec

@@ -36,8 +36,10 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "activesupport"
   spec.add_runtime_dependency "async-websocket"
   spec.add_runtime_dependency "aws-sdk-athena"
+  spec.add_runtime_dependency "aws-sdk-cloudwatchlogs"
   spec.add_runtime_dependency "aws-sdk-s3"
   spec.add_runtime_dependency "aws-sdk-sts"
+  spec.add_runtime_dependency "aws-sigv4"
   spec.add_runtime_dependency "csv"
   spec.add_runtime_dependency "dry-schema"
   spec.add_runtime_dependency "dry-struct"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.0
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-27 00:00:00.000000000 Z
+date: 2024-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudwatchlogs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sigv4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -600,6 +628,11 @@ files:
 - lib/multiwoven/integrations/source/aws_athena/config/meta.json
 - lib/multiwoven/integrations/source/aws_athena/config/spec.json
 - lib/multiwoven/integrations/source/aws_athena/icon.svg
+- lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb
+- lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json
+- lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json
+- lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json
+- lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg
 - lib/multiwoven/integrations/source/bigquery/client.rb
 - lib/multiwoven/integrations/source/bigquery/config/meta.json
 - lib/multiwoven/integrations/source/bigquery/config/spec.json