multiwoven-integrations 0.9.2 → 0.10.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fd6418b9c76b3b111b921a6a4bb9da3f2b332179e583e8f5080e5151518408bf
4
- data.tar.gz: 5c153b30244e1d6235c076b39c38ad50cd1e996a8f060b304a01c25b9f841756
3
+ metadata.gz: 34da17012dfaee9c1275aad7968ac32378d822f53819f71d421cf29323fca12a
4
+ data.tar.gz: 9a5816f59fa89720d52ec80f8d206b6eb9e303206a9858f80f999c74095fdc46
5
5
  SHA512:
6
- metadata.gz: f287bf95ad5156f21473c99ff9f711869628b7115cc5c27e1b3b2da24dc4ea1060a19d104dc83e72fdf6ecc30fa429a2f03920ecb977afecc00a1fcd9d77de08
7
- data.tar.gz: a5ab7543a804567ba9813e24fe73574d4f11eb910d98519fbc847431a1b28ce37eb58597464714b021d86d98252898c59365d87416b859ed46fc8e59fbd161e9
6
+ metadata.gz: b29fad3dabb2793292a2f101eb2136081051e23bb0b63013bb4d1ee6c7b76e4f8b00e8dc2db05a8ec807fffc8b27dc8034613409872ffe9f5ddfad41fec2c831
7
+ data.tar.gz: 7de72831e5fe9d5075abf1a5dae5219fcf3732a85f75b105a6b173c2ad0b56ae21e750d45ef8f8f691378a1843634496bc12cd947a550a789f0db81133004774
@@ -50,9 +50,6 @@ module Multiwoven
50
50
  DATABRICKS_HEALTH_URL = "https://%<databricks_host>s/api/2.0/serving-endpoints/%<endpoint_name>s"
51
51
  DATABRICKS_SERVING_URL = "https://%<databricks_host>s/serving-endpoints/%<endpoint_name>s/invocations"
52
52
 
53
- AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
54
- AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
55
-
56
53
  # HTTP
57
54
  HTTP_GET = "GET"
58
55
  HTTP_POST = "POST"
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Multiwoven
4
4
  module Integrations
5
- VERSION = "0.9.2"
5
+ VERSION = "0.10.0"
6
6
 
7
7
  ENABLED_SOURCES = %w[
8
8
  Snowflake
@@ -17,6 +17,7 @@ module Multiwoven
17
17
  MariaDB
18
18
  Oracle
19
19
  DatabricksModel
20
+ AwsSagemakerModel
20
21
  ].freeze
21
22
 
22
23
  ENABLED_DESTINATIONS = %w[
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven::Integrations::Source
4
+ module AwsSagemakerModel
5
+ include Multiwoven::Integrations::Core
6
+ class Client < SourceConnector
7
+ def check_connection(connection_config)
8
+ connection_config = connection_config.with_indifferent_access
9
+ create_connection(connection_config)
10
+ response = @client.describe_endpoint(endpoint_name: connection_config[:endpoint_name])
11
+ if response.endpoint_status == "InService"
12
+ success_status
13
+ else
14
+ failure_status
15
+ end
16
+ rescue StandardError => e
17
+ ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
18
+ end
19
+
20
+ def discover(_connection_config)
21
+ catalog_json = read_json(CATALOG_SPEC_PATH)
22
+ catalog = build_catalog(catalog_json)
23
+ catalog.to_multiwoven_message
24
+ rescue StandardError => e
25
+ handle_exception(e, {
26
+ context: "AWS:SAGEMAKER MODEL:DISCOVER:EXCEPTION",
27
+ type: "error"
28
+ })
29
+ end
30
+
31
+ def read(sync_config)
32
+ connection_config = sync_config.source.connection_specification
33
+ connection_config = connection_config.with_indifferent_access
34
+ payload = sync_config.model.query
35
+ create_connection(connection_config)
36
+ run_model(connection_config, payload)
37
+ rescue StandardError => e
38
+ handle_exception(e, {
39
+ context: "AWS:SAGEMAKER MODEL:READ:EXCEPTION",
40
+ type: "error",
41
+ sync_id: sync_config.sync_id,
42
+ sync_run_id: sync_config.sync_run_id
43
+ })
44
+ end
45
+
46
+ private
47
+
48
+ def create_connection(connection_config)
49
+ @client = Aws::SageMaker::Client.new(
50
+ region: connection_config[:region],
51
+ access_key_id: connection_config[:access_key],
52
+ secret_access_key: connection_config[:secret_access_key]
53
+ )
54
+
55
+ @client_runtime = Aws::SageMakerRuntime::Client.new(
56
+ region: connection_config[:region],
57
+ access_key_id: connection_config[:access_key],
58
+ secret_access_key: connection_config[:secret_access_key]
59
+ )
60
+ end
61
+
62
+ def run_model(connection_config, payload)
63
+ response = @client_runtime.invoke_endpoint(
64
+ endpoint_name: connection_config[:endpoint_name],
65
+ content_type: "application/json",
66
+ body: payload
67
+ )
68
+ process_response(response)
69
+ rescue StandardError => e
70
+ handle_exception(e, context: "AWS:SAGEMAKER MODEL:RUN_MODEL:EXCEPTION", type: "error")
71
+ end
72
+
73
+ def process_response(response)
74
+ data = JSON.parse(response.body.read)
75
+ [RecordMessage.new(data: { response: data }, emitted_at: Time.now.to_i).to_multiwoven_message]
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,6 @@
1
+ {
2
+ "request_rate_limit": 600,
3
+ "request_rate_limit_unit": "minute",
4
+ "request_rate_concurrency": 10,
5
+ "streams": []
6
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "AwsSagemakerModel",
4
+ "title": "AWS Sagemaker Model",
5
+ "connector_type": "source",
6
+ "category": "AI Model",
7
+ "documentation_url": "https://docs.mutliwoven.com",
8
+ "github_issue_label": "source-aws-sagemaker-model",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,50 @@
1
+ {
2
+ "documentation_url": "https://docs.multiwoven.com/integrations/sources/aws_sagemaker-model",
3
+ "stream_type": "user_defined",
4
+ "connector_query_type": "ai_ml",
5
+ "connection_specification": {
6
+ "$schema": "http://json-schema.org/draft-07/schema#",
7
+ "title": "AWS Sagemaker Model",
8
+ "type": "object",
9
+ "required": ["access_key", "secret_access_key", "region", "endpoint_name", "request_format", "response_format"],
10
+ "properties": {
11
+ "access_key": {
12
+ "description": "The AWS Access Key ID to use for authentication",
13
+ "type": "string",
14
+ "title": "Personal Access Key",
15
+ "order": 0
16
+ },
17
+ "secret_access_key": {
18
+ "description": "The AWS Secret Access Key to use for authentication",
19
+ "type": "string",
20
+ "multiwoven_secret": true,
21
+ "title": "Secret Access Key",
22
+ "order": 1
23
+ },
24
+ "region": {
25
+ "description": "AWS region",
26
+ "type": "string",
27
+ "title": "Region",
28
+ "order": 2
29
+ },
30
+ "endpoint_name": {
31
+ "description": "Endpoint name for AWS Sagemaker",
32
+ "type": "string",
33
+ "title": "Endpoint name",
34
+ "order": 3
35
+ },
36
+ "request_format": {
37
+ "description": "Sample Request Format",
38
+ "type": "string",
39
+ "title": "Request Format",
40
+ "order": 4
41
+ },
42
+ "response_format": {
43
+ "description": "Sample Response Format",
44
+ "type": "string",
45
+ "title": "Response Format",
46
+ "order": 5
47
+ }
48
+ }
49
+ }
50
+ }
@@ -0,0 +1,18 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <svg width="80px" height="80px" viewBox="0 0 80 80" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
3
+ <!-- Generator: Sketch 64 (93537) - https://sketch.com -->
4
+ <title>Icon-Architecture/64/Arch_AWS-SageMaker_64</title>
5
+ <desc>Created with Sketch.</desc>
6
+ <defs>
7
+ <linearGradient x1="0%" y1="100%" x2="100%" y2="0%" id="linearGradient-1">
8
+ <stop stop-color="#055F4E" offset="0%"></stop>
9
+ <stop stop-color="#56C0A7" offset="100%"></stop>
10
+ </linearGradient>
11
+ </defs>
12
+ <g id="Icon-Architecture/64/Arch_AWS-SageMaker_64" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
13
+ <g id="Icon-Architecture-BG/64/Machine-Learning" fill="url(#linearGradient-1)">
14
+ <rect id="Rectangle" x="0" y="0" width="80" height="80"></rect>
15
+ </g>
16
+ <path d="M54.034,26.0335731 C54.034,26.5935654 53.578,27.0495592 53.017,27.0495592 C52.458,27.0495592 52.002,26.5935654 52.002,26.0335731 C52.002,25.4735807 52.458,25.0175869 53.017,25.0175869 C53.578,25.0175869 54.034,25.4735807 54.034,26.0335731 L54.034,26.0335731 Z M48.002,36.000437 C48.002,35.4494445 48.45,35.0004506 49.002,35.0004506 C49.554,35.0004506 50.002,35.4494445 50.002,36.000437 C50.002,36.5514294 49.554,37.0004233 49.002,37.0004233 C48.45,37.0004233 48.002,36.5514294 48.002,36.000437 L48.002,36.000437 Z M48.002,55.0001775 C48.002,54.449185 48.45,54.0001912 49.002,54.0001912 C49.554,54.0001912 50.002,54.449185 50.002,55.0001775 C50.002,55.55117 49.554,56.0001639 49.002,56.0001639 C48.45,56.0001639 48.002,55.55117 48.002,55.0001775 L48.002,55.0001775 Z M58.002,42.000355 C58.002,42.5513475 57.554,43.0003414 57.002,43.0003414 C56.45,43.0003414 56.002,42.5513475 56.002,42.000355 C56.002,41.4493626 56.45,41.0003687 57.002,41.0003687 C57.554,41.0003687 58.002,41.4493626 58.002,42.000355 L58.002,42.000355 Z M65,45.2723104 L59.963,42.3823498 C59.979,42.2563515 60.002,42.1313532 60.002,42.000355 C60.002,40.3463776 58.656,39.000396 57.002,39.000396 C55.347,39.000396 54.002,40.3463776 54.002,42.000355 C54.002,43.6543324 55.347,45.0003141 57.002,45.0003141 C57.801,45.0003141 58.523,44.6813184 59.061,44.1713254 L63.886,46.9392876 L59.555,49.105258 C59.216,49.2752557 59.002,49.621251 59.002,50.0002458 L59.002,58.4411305 L46.983,65.8370295 L41.003,62.4200762 L41.003,56.0001639 L46.186,56.0001639 C46.6,57.161148 47.7,58.0001366 49.002,58.0001366 C50.656,58.0001366 52.002,56.6541549 52.002,55.0001775 C52.002,53.3452001 50.656,52.0002185 49.002,52.0002185 C47.7,52.0002185 46.6,52.838207 46.186,54.0001912 L41.003,54.0001912 L41.003,40.0003823 C41.003,39.6493871 40.818,39.3233916 40.517,39.1423941 L35.516,36.142435 L34.487,37.8574116 L39.003,40.5663746 L39.003,43.5073345 L33.002,48.1232714 L33.002,44.0003277 C33.002,43.6963319 32.864,43.4083358 32.627,43.2193384 L28.002,39.5193889 L28.002,34.535457 L33.556,30.8325075 C33.835,30.6465101 34.002,30.3345143 34.002,30.0005189 L34.002,24.0006008 L32.002,24.0006008 L32.002,29.4655262 L27.013,32.7904808 L22.002,29.4635262 L22.002,21.5746339 L27.002,18.6586738 L27.002,27.0005599 L29.002,27.0005599 L29.002,17.4916897 L33.005,15.1567216 L39.001,18.6156744 L39.002,31.0005052 C39.002,31.3595003 39.194,31.6904958 39.506,31.8684934 L46.042,35.6034424 C46.024,35.7344406 46.002,35.8644388 46.002,36.000437 C46.002,37.6544144 47.347,39.000396 49.002,39.000396 C50.656,39.000396 52.002,37.6544144 52.002,36.000437 C52.002,34.3464595 50.656,33.0004779 49.002,33.0004779 C48.208,33.0004779 47.49,33.3154736 46.953,33.8204667 L41.002,30.4195132 L41.001,18.6176743 L46.964,15.1767213 L58.002,22.5356208 L58.002,25.0005872 L55.851,25.0005872 C55.429,23.844603 54.318,23.0176142 53.017,23.0176142 C51.354,23.0176142 50.002,24.3705958 50.002,26.0335731 C50.002,27.6965504 51.354,29.0495319 53.017,29.0495319 C54.343,29.0495319 55.471,28.1905436 55.875,27.0005599 L58.002,27.0005599 L58.002,30.0005189 C58.002,30.359514 58.194,30.6905095 58.506,30.868507 L65,34.5804564 L65,45.2723104 Z M33.02,65.8370295 L29.867,63.897056 L35.583,59.8141118 L34.421,58.186134 L28.018,62.7590716 L21.002,58.4411305 L21.002,50.5662381 L25.516,47.8572751 L24.487,46.1422985 L19.958,48.8602614 L15.002,46.3822952 L15.001,40.6173739 L20.449,37.8944111 L19.555,36.1054355 L15.001,38.3814044 L15.002,34.5804564 L20.963,31.1745029 L26.002,34.5194572 L26.002,39.4803894 L20.449,43.1673391 L21.555,44.8333163 L26.958,41.2453653 L31.002,44.4803212 L31.002,49.6622504 L26.392,53.207202 L27.611,54.7921804 L39.003,46.0303 L39.003,62.4190762 L33.02,65.8370295 Z M66.496,33.1324761 L60.002,29.4205268 L60.002,22.0006281 C60.002,21.6656327 59.835,21.353637 59.556,21.1686395 L47.556,13.1687487 C47.24,12.9587516 46.832,12.9447518 46.502,13.1347492 L40.004,16.884698 L33.502,13.1347492 C33.19,12.9547517 32.807,12.9547517 32.498,13.1367492 L20.498,20.1366536 C20.19,20.3156511 20.002,20.6446466 20.002,21.0006418 L20.002,29.4205268 L13.506,33.1324761 C13.194,33.3094737 13.002,33.6414692 13.002,34.0004643 L13.002,34.4174586 C13.001,34.4384583 13,34.458458 13,34.4794577 L13,45.3633091 C13,45.3833088 13.001,45.4033086 13.002,45.4223083 L13.002,47.0002868 C13.002,47.3792816 13.216,47.7252769 13.555,47.8942745 L19.002,50.6182374 L19.002,59.0001229 C19.002,59.3471182 19.181,59.6691138 19.477,59.8511113 L32.477,67.851002 C32.638,67.9500007 32.82,68 33.002,68 C33.173,68 33.344,67.9560006 33.498,67.8680018 L40.003,64.1520525 L46.506,67.8680018 C46.821,68.0489993 47.213,68.0419994 47.526,67.851002 L60.526,59.8511113 C60.822,59.6691138 61.002,59.3471182 61.002,59.0001229 L61.002,50.6182374 L66.447,47.8942745 C66.786,47.7252769 67,47.3792816 67,47.0002868 L67,34.0004643 C67,33.6414692 66.807,33.3104737 66.496,33.1324761 L66.496,33.1324761 Z" id="AWS-SageMaker_Icon_64_Squid" fill="#FFFFFF"></path>
17
+ </g>
18
+ </svg>
@@ -32,6 +32,8 @@ require "duckdb"
32
32
  require "iterable-api-client"
33
33
  require "aws-sdk-sts"
34
34
  require "ruby-oci8"
35
+ require "aws-sdk-sagemaker"
36
+ require "aws-sdk-sagemakerruntime"
35
37
 
36
38
  # Service
37
39
  require_relative "integrations/config"
@@ -63,6 +65,7 @@ require_relative "integrations/source/amazon_s3/client"
63
65
  require_relative "integrations/source/maria_db/client"
64
66
  require_relative "integrations/source/oracle_db/client"
65
67
  require_relative "integrations/source/databrics_model/client"
68
+ require_relative "integrations/source/aws_sagemaker_model/client"
66
69
 
67
70
  # Destination
68
71
  require_relative "integrations/destination/klaviyo/client"
@@ -36,8 +36,10 @@ Gem::Specification.new do |spec|
36
36
  spec.add_runtime_dependency "activesupport"
37
37
  spec.add_runtime_dependency "async-websocket"
38
38
  spec.add_runtime_dependency "aws-sdk-athena"
39
+ spec.add_runtime_dependency "aws-sdk-cloudwatchlogs"
39
40
  spec.add_runtime_dependency "aws-sdk-s3"
40
41
  spec.add_runtime_dependency "aws-sdk-sts"
42
+ spec.add_runtime_dependency "aws-sigv4"
41
43
  spec.add_runtime_dependency "csv"
42
44
  spec.add_runtime_dependency "dry-schema"
43
45
  spec.add_runtime_dependency "dry-struct"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiwoven-integrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.2
4
+ version: 0.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Subin T P
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-08-27 00:00:00.000000000 Z
11
+ date: 2024-08-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: aws-sdk-cloudwatchlogs
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: aws-sdk-s3
57
71
  requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
80
94
  - - ">="
81
95
  - !ruby/object:Gem::Version
82
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: aws-sigv4
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
83
111
  - !ruby/object:Gem::Dependency
84
112
  name: csv
85
113
  requirement: !ruby/object:Gem::Requirement
@@ -600,6 +628,11 @@ files:
600
628
  - lib/multiwoven/integrations/source/aws_athena/config/meta.json
601
629
  - lib/multiwoven/integrations/source/aws_athena/config/spec.json
602
630
  - lib/multiwoven/integrations/source/aws_athena/icon.svg
631
+ - lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb
632
+ - lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json
633
+ - lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json
634
+ - lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json
635
+ - lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg
603
636
  - lib/multiwoven/integrations/source/bigquery/client.rb
604
637
  - lib/multiwoven/integrations/source/bigquery/config/meta.json
605
638
  - lib/multiwoven/integrations/source/bigquery/config/spec.json