multiwoven-integrations 0.1.66 → 0.1.67

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d41f10c87a8d9a6b6d5b3f04952afcc4c906266d79a2c0935b4468d6ccb4592e
4
- data.tar.gz: 5a09773b98f94d2ecf401ee8c46affe0684704b65a5d0dd73838418dbcf2d78d
3
+ metadata.gz: da70c8d91dea6167ebb4103cfe0e1999d663ee2ac420d9ae272a5dbd70924380
4
+ data.tar.gz: bc6fd6fd71c8c02dcdcf4e1f3eaf69dee37f3df6ba5f6fb1d4af49aa64220119
5
5
  SHA512:
6
- metadata.gz: 3cff2743efecd6b06a1d7470d11ef3410adea1bd81ee5f028468d964a6630f8f3bafb4f5b7729aaa514afbb4608cacd0ee44723cb27863facc88dd0c5143c773
7
- data.tar.gz: bbfc548d9a8e0df2efcf466a728fd3d13e6eb3a8fe49ed7844d7c55249764b515e57f5c1a915601ec3e570195835b8a3162d5f3b9bd08b092c652c0990e8e015
6
+ metadata.gz: 2040a6232add940bd41cffa131c5738d64a4187df2615f3625910aa6129ae089ffd89027ff78df15d753a27f5eb10cec0357f4b977f25d86c9663480dce88c71
7
+ data.tar.gz: cfe7e24682df19fd0aa10284891d073324933838ceb10bfab3a8dc7259705af422b86d4b5fc85665af580a2097625fb0feafdd05152ea9d2f5bf6d0d0e51fff4
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Multiwoven
4
4
  module Integrations
5
- VERSION = "0.1.66"
5
+ VERSION = "0.1.67"
6
6
 
7
7
  ENABLED_SOURCES = %w[
8
8
  Snowflake
@@ -11,6 +11,7 @@ module Multiwoven
11
11
  Postgresql
12
12
  Databricks
13
13
  SalesforceConsumerGoodsCloud
14
+ AwsAthena
14
15
  ].freeze
15
16
 
16
17
  ENABLED_DESTINATIONS = %w[
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven::Integrations::Source
4
+ module AwsAthena
5
+ include Multiwoven::Integrations::Core
6
+ class Client < SourceConnector
7
+ def check_connection(connection_config)
8
+ connection_config = connection_config.with_indifferent_access
9
+ athena_client = create_connection(connection_config)
10
+ athena_client.list_work_groups
11
+ ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
12
+ rescue StandardError => e
13
+ ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
14
+ end
15
+
16
+ def discover(connection_config)
17
+ connection_config = connection_config.with_indifferent_access
18
+ query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:schema]}' ORDER BY table_name, ordinal_position;"
19
+ db = create_connection(connection_config)
20
+ results = query_execution(db, query)
21
+ catalog = Catalog.new(streams: create_streams(results))
22
+ catalog.to_multiwoven_message
23
+ rescue StandardError => e
24
+ handle_exception(
25
+ "AWS:ATHENA:DISCOVER:EXCEPTION",
26
+ "error",
27
+ e
28
+ )
29
+ end
30
+
31
+ def read(sync_config)
32
+ connection_config = sync_config.source.connection_specification
33
+ connection_config = connection_config.with_indifferent_access
34
+ query = sync_config.model.query
35
+ query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
36
+ db = create_connection(connection_config)
37
+ query(db, query)
38
+ rescue StandardError => e
39
+ handle_exception(
40
+ "AWS:ATHENA:READ:EXCEPTION",
41
+ "error",
42
+ e
43
+ )
44
+ end
45
+
46
+ private
47
+
48
+ def create_connection(connection_config)
49
+ Aws.config.update({ credentials: Aws::Credentials.new(connection_config[:access_key], connection_config[:secret_access_key]), region: connection_config[:region] })
50
+ @database = connection_config[:schema]
51
+ @output_location = connection_config[:output_location]
52
+ Aws::Athena::Client.new
53
+ end
54
+
55
+ def query_execution(db, query)
56
+ response = db.start_query_execution(
57
+ query_string: query,
58
+ query_execution_context: { database: @database },
59
+ result_configuration: { output_location: @output_location }
60
+ )
61
+ query_execution_id = response[:query_execution_id]
62
+ loop do
63
+ response = db.get_query_execution(query_execution_id: query_execution_id)
64
+ status = response.query_execution.status.state
65
+ break if %w[SUCCEEDED FAILED CANCELLED].include?(status)
66
+ end
67
+ transform_results(db.get_query_results(query_execution_id: query_execution_id))
68
+ end
69
+
70
+ def create_streams(records)
71
+ group_by_table(records).map do |_, r|
72
+ Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
73
+ end
74
+ end
75
+
76
+ def transform_results(results)
77
+ columns = results.result_set.result_set_metadata.column_info.map(&:name)
78
+ rows = results.result_set.rows.map do |row|
79
+ row.data.map(&:var_char_value)
80
+ end
81
+ rows.map { |row| columns.zip(row).to_h }
82
+ end
83
+
84
+ def query(db, query)
85
+ records = []
86
+ query_execution(db, query).map do |row|
87
+ records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
88
+ end
89
+ records
90
+ end
91
+
92
+ def group_by_table(records)
93
+ result = {}
94
+ records.each_with_index do |entry, index|
95
+ table_name = entry["table_name"]
96
+ column_data = {
97
+ column_name: entry["column_name"],
98
+ data_type: entry["data_type"],
99
+ is_nullable: entry["is_nullable"] == "YES"
100
+ }
101
+ result[index] ||= {}
102
+ result[index][:tablename] = table_name
103
+ result[index][:columns] = [column_data]
104
+ end
105
+ result
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "AwsAthena",
4
+ "title": "AWS Athena",
5
+ "connector_type": "source",
6
+ "category": "Data Warehouse",
7
+ "documentation_url": "https://docs.mutliwoven.com",
8
+ "github_issue_label": "source-aws-athena",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,63 @@
1
+ {
2
+ "documentation_url": "https://docs.multiwoven.com/integrations/sources/athena",
3
+ "stream_type": "dynamic",
4
+ "connector_query_type": "raw_sql",
5
+ "connection_specification": {
6
+ "$schema": "http://json-schema.org/draft-07/schema#",
7
+ "title": "AWS Athena",
8
+ "type": "object",
9
+ "required": ["access_key", "secret_access_key", "region", "workgroup", "catalog", "output_location"],
10
+ "properties": {
11
+ "access_key": {
12
+ "description": "The AWS Access Key ID to use for authentication.",
13
+ "examples": ["AWSATHENAACCESS"],
14
+ "type": "string",
15
+ "title": "Personal Access Key",
16
+ "order": 0
17
+ },
18
+ "secret_access_key": {
19
+ "description": "The AWS Secret Access Key to use for authentication.",
20
+ "examples": ["AWSATHENASECRET"],
21
+ "type": "string",
22
+ "multiwoven_secret": true,
23
+ "title": "Secret Access Key",
24
+ "order": 1
25
+ },
26
+ "region": {
27
+ "description": "AWS region where Athena is located.",
28
+ "examples": ["ATHENA_REGION"],
29
+ "type": "string",
30
+ "title": "Secret Access Key",
31
+ "order": 2
32
+ },
33
+ "workgroup": {
34
+ "description": "The Athena workgroup you previously set up in AWS.",
35
+ "examples": ["ATHENA_WORKGROUP"],
36
+ "type": "string",
37
+ "title": "Workgroup",
38
+ "order": 3
39
+ },
40
+ "catalog": {
41
+ "description": "The Data catalog name within Athena.",
42
+ "examples": ["ATHENA_CATALOG"],
43
+ "type": "string",
44
+ "title": "Catalog",
45
+ "order": 4
46
+ },
47
+ "schema": {
48
+ "description": "The specific Athena database/schema to connect to.",
49
+ "examples": ["ATHENA_DB"],
50
+ "type": "string",
51
+ "title": "Database",
52
+ "order": 5
53
+ },
54
+ "output_location": {
55
+ "description": "S3 path for query output.",
56
+ "examples": ["s3://example-bucket-name/query-results/"],
57
+ "type": "string",
58
+ "title": "Query",
59
+ "order": 6
60
+ }
61
+ }
62
+ }
63
+ }
@@ -0,0 +1,22 @@
1
+ <svg height="100" viewBox="0 0 100 100" width="100" xmlns="http://www.w3.org/2000/svg">
2
+ <path d="m84 64.68-34-4.19-34 4.19 34 26.62z" fill="#fcbf92"/>
3
+ <path d="m16 64.68 34 10.17v16.45l-34-17.05z" fill="#9d5025"/>
4
+ <path d="m84 64.68-34 10.17v16.45l34-17.05z" fill="#f58534"/>
5
+ <path d="m20.39 50.22h-4.39v9.67l4.39.8 4.88-5.18z" fill="#9d5025"/>
6
+ <path d="m25.27 60.29-4.88.4v-10.47h4.88z" fill="#f58534"/>
7
+ <path d="m28.56 38.16-5.78 1.09v21.94l5.78 1.2 5.29-12.17z" fill="#9d5025"/>
8
+ <path d="m33.85 61.79-5.29.6v-24.23l5.29.6z" fill="#f58534"/>
9
+ <path d="m39.83 43.04-7.98.8v19.24l7.98 1.6 8.47-10.87z" fill="#9d5025"/>
10
+ <path d="m45.21 63.78-5.38.9v-21.64l5.38.4z" fill="#f58534"/>
11
+ <g fill="#9d5025">
12
+ <path d="m50 25.59-5.38 1.7v38.29l5.38 1.09 5.38-20.54z"/>
13
+ <path d="m54.98 63.78 5.19.9 7.98-14.46-7.98-14.36-5.19.8z"/>
14
+ <path d="m68.15 50.22-7.98-14.36-5.19.8"/>
15
+ <path d="m66.05 61.79 5.39.6 5.78-18.25-5.78-18.15-5.39 1.3z"/>
16
+ <path d="m74.73 60.29 4.88.4 4.39-18.05-4.39-18.64-4.88 1.2z"/>
17
+ </g>
18
+ <path d="m50 25.59 5.38 1.7v38.29l-5.38 1.09z" fill="#f58534"/>
19
+ <path d="m68.15 37.46-7.98-1.6v28.82l7.98-1.6z" fill="#f58534"/>
20
+ <path d="m77.22 28.29-5.78-2.3v36.4l5.78-1.2z" fill="#f58534"/>
21
+ <path d="m84 26.19-4.39-2.19v36.69l4.39-.8z" fill="#f58534"/>
22
+ </svg>
@@ -22,6 +22,7 @@ require "stripe"
22
22
  require "net/sftp"
23
23
  require "csv"
24
24
  require "securerandom"
25
+ require "aws-sdk-athena"
25
26
  require "zip"
26
27
  require "zendesk_api"
27
28
 
@@ -49,6 +50,7 @@ require_relative "integrations/source/bigquery/client"
49
50
  require_relative "integrations/source/postgresql/client"
50
51
  require_relative "integrations/source/databricks/client"
51
52
  require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
53
+ require_relative "integrations/source/aws_athena/client"
52
54
 
53
55
  # Destination
54
56
  require_relative "integrations/destination/klaviyo/client"
@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
35
35
 
36
36
  spec.add_runtime_dependency "activesupport"
37
37
  spec.add_runtime_dependency "async-websocket"
38
+ spec.add_runtime_dependency "aws-sdk-athena"
38
39
  spec.add_runtime_dependency "csv"
39
40
  spec.add_runtime_dependency "dry-schema"
40
41
  spec.add_runtime_dependency "dry-struct"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiwoven-integrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.66
4
+ version: 0.1.67
5
5
  platform: ruby
6
6
  authors:
7
7
  - Subin T P
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-05-14 00:00:00.000000000 Z
11
+ date: 2024-05-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: aws-sdk-athena
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: csv
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -486,6 +500,10 @@ files:
486
500
  - lib/multiwoven/integrations/protocol/protocol.rb
487
501
  - lib/multiwoven/integrations/rollout.rb
488
502
  - lib/multiwoven/integrations/service.rb
503
+ - lib/multiwoven/integrations/source/aws_athena/client.rb
504
+ - lib/multiwoven/integrations/source/aws_athena/config/meta.json
505
+ - lib/multiwoven/integrations/source/aws_athena/config/spec.json
506
+ - lib/multiwoven/integrations/source/aws_athena/icon.svg
489
507
  - lib/multiwoven/integrations/source/bigquery/client.rb
490
508
  - lib/multiwoven/integrations/source/bigquery/config/meta.json
491
509
  - lib/multiwoven/integrations/source/bigquery/config/spec.json