multiwoven-integrations 0.1.65 → 0.1.67
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/constants.rb +2 -0
- data/lib/multiwoven/integrations/destination/zendesk/client.rb +118 -0
- data/lib/multiwoven/integrations/destination/zendesk/config/catalog.json +110 -0
- data/lib/multiwoven/integrations/destination/zendesk/config/meta.json +18 -0
- data/lib/multiwoven/integrations/destination/zendesk/config/spec.json +32 -0
- data/lib/multiwoven/integrations/destination/zendesk/icon.svg +63 -0
- data/lib/multiwoven/integrations/rollout.rb +3 -1
- data/lib/multiwoven/integrations/source/aws_athena/client.rb +109 -0
- data/lib/multiwoven/integrations/source/aws_athena/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/aws_athena/config/spec.json +63 -0
- data/lib/multiwoven/integrations/source/aws_athena/icon.svg +22 -0
- data/lib/multiwoven/integrations.rb +4 -0
- data/multiwoven-integrations.gemspec +2 -0
- metadata +39 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: da70c8d91dea6167ebb4103cfe0e1999d663ee2ac420d9ae272a5dbd70924380
|
|
4
|
+
data.tar.gz: bc6fd6fd71c8c02dcdcf4e1f3eaf69dee37f3df6ba5f6fb1d4af49aa64220119
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2040a6232add940bd41cffa131c5738d64a4187df2615f3625910aa6129ae089ffd89027ff78df15d753a27f5eb10cec0357f4b977f25d86c9663480dce88c71
|
|
7
|
+
data.tar.gz: cfe7e24682df19fd0aa10284891d073324933838ceb10bfab3a8dc7259705af422b86d4b5fc85665af580a2097625fb0feafdd05152ea9d2f5bf6d0d0e51fff4
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations
|
|
5
|
+
module Destination
|
|
6
|
+
module Zendesk
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < DestinationConnector
|
|
9
|
+
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
10
|
+
def check_connection(connection_config)
|
|
11
|
+
connection_config = connection_config.with_indifferent_access
|
|
12
|
+
initialize_client(connection_config)
|
|
13
|
+
authenticate_client
|
|
14
|
+
success_status
|
|
15
|
+
rescue StandardError => e
|
|
16
|
+
handle_exception("ZENDESK:CHECK_CONNECTION:EXCEPTION", "error", e)
|
|
17
|
+
failure_status(e)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def discover(_connection_config = nil)
|
|
21
|
+
catalog = build_catalog(load_catalog)
|
|
22
|
+
catalog.to_multiwoven_message
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
handle_exception("ZENDESK:DISCOVER:EXCEPTION", "error", e)
|
|
25
|
+
failure_status(e)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def write(sync_config, records, action = "create")
|
|
29
|
+
@action = sync_config.stream.action || action
|
|
30
|
+
initialize_client(sync_config.destination.connection_specification)
|
|
31
|
+
process_records(records, sync_config.stream)
|
|
32
|
+
rescue StandardError => e
|
|
33
|
+
handle_exception("ZENDESK:WRITE:EXCEPTION", "error", e)
|
|
34
|
+
failure_status(e)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def initialize_client(connection_config)
|
|
40
|
+
connection_config = connection_config.with_indifferent_access
|
|
41
|
+
@client = ZendeskAPI::Client.new do |config|
|
|
42
|
+
config.url = "#{connection_config[:subdomain]}.#{ZENDESK_URL_SUFFIX}"
|
|
43
|
+
config.username = connection_config[:username]
|
|
44
|
+
config.password = connection_config[:password]
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def authenticate_client
|
|
49
|
+
@client.tickets.page(1).per_page(1).fetch
|
|
50
|
+
rescue ZendeskAPI::Error => e
|
|
51
|
+
raise StandardError, "Authentication failed: #{e.message}"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def process_records(records, stream)
|
|
55
|
+
write_success = 0
|
|
56
|
+
write_failure = 0
|
|
57
|
+
|
|
58
|
+
records.each do |record|
|
|
59
|
+
zendesk_data = prepare_record_data(record, stream.name)
|
|
60
|
+
plural_stream_name = pluralize_stream_name(stream.name.downcase)
|
|
61
|
+
|
|
62
|
+
if @action == "create"
|
|
63
|
+
@client.send(plural_stream_name).create!(zendesk_data)
|
|
64
|
+
else
|
|
65
|
+
existing_record = @client.send(plural_stream_name).find(id: record[:id])
|
|
66
|
+
existing_record.update!(zendesk_data)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
write_success += 1
|
|
70
|
+
rescue StandardError => e
|
|
71
|
+
handle_exception("ZENDESK:WRITE_RECORD:EXCEPTION", "error", e)
|
|
72
|
+
write_failure += 1
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
tracking_message(write_success, write_failure)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def pluralize_stream_name(name)
|
|
79
|
+
{ "ticket" => "tickets", "user" => "users" }.fetch(name, name)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def prepare_record_data(record, type)
|
|
83
|
+
case type
|
|
84
|
+
when "Tickets"
|
|
85
|
+
{
|
|
86
|
+
subject: record[:subject],
|
|
87
|
+
comment: { body: record[:description] },
|
|
88
|
+
priority: record[:priority],
|
|
89
|
+
status: record[:status],
|
|
90
|
+
requester_id: record[:requester_id],
|
|
91
|
+
assignee_id: record[:assignee_id],
|
|
92
|
+
tags: record[:tags]
|
|
93
|
+
}
|
|
94
|
+
when "Users"
|
|
95
|
+
{
|
|
96
|
+
name: record[:name],
|
|
97
|
+
email: record[:email],
|
|
98
|
+
role: record[:role]
|
|
99
|
+
}
|
|
100
|
+
else
|
|
101
|
+
raise StandardError, "Unsupported record type: #{type}"
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def load_catalog
|
|
106
|
+
read_json(CATALOG_SPEC_PATH)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def tracking_message(success, failure)
|
|
110
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
111
|
+
success: success, failed: failure
|
|
112
|
+
).to_multiwoven_message
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
{
|
|
2
|
+
"request_rate_limit": 700,
|
|
3
|
+
"request_rate_limit_unit": "minute",
|
|
4
|
+
"request_rate_concurrency": 10,
|
|
5
|
+
"streams": [
|
|
6
|
+
{
|
|
7
|
+
"name": "tickets",
|
|
8
|
+
"action": "create",
|
|
9
|
+
"json_schema": {
|
|
10
|
+
"type": "object",
|
|
11
|
+
"additionalProperties": true,
|
|
12
|
+
"properties": {
|
|
13
|
+
"id": {
|
|
14
|
+
"type": "integer"
|
|
15
|
+
},
|
|
16
|
+
"subject": {
|
|
17
|
+
"type": "string"
|
|
18
|
+
},
|
|
19
|
+
"description": {
|
|
20
|
+
"type": "string"
|
|
21
|
+
},
|
|
22
|
+
"status": {
|
|
23
|
+
"type": "string"
|
|
24
|
+
},
|
|
25
|
+
"priority": {
|
|
26
|
+
"type": "string"
|
|
27
|
+
},
|
|
28
|
+
"requester_id": {
|
|
29
|
+
"type": "integer"
|
|
30
|
+
},
|
|
31
|
+
"assignee_id": {
|
|
32
|
+
"type": "integer"
|
|
33
|
+
},
|
|
34
|
+
"tags": {
|
|
35
|
+
"type": "array",
|
|
36
|
+
"items": {
|
|
37
|
+
"type": "string"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"created_at": {
|
|
41
|
+
"type": "string",
|
|
42
|
+
"format": "date-time"
|
|
43
|
+
},
|
|
44
|
+
"updated_at": {
|
|
45
|
+
"type": "string",
|
|
46
|
+
"format": "date-time"
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
"supported_sync_modes": [
|
|
51
|
+
"incremental"
|
|
52
|
+
],
|
|
53
|
+
"source_defined_cursor": true,
|
|
54
|
+
"default_cursor_field": [
|
|
55
|
+
"updated_at"
|
|
56
|
+
],
|
|
57
|
+
"source_defined_primary_key": [
|
|
58
|
+
[
|
|
59
|
+
"id"
|
|
60
|
+
]
|
|
61
|
+
]
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"name": "users",
|
|
65
|
+
"action": "create",
|
|
66
|
+
"json_schema": {
|
|
67
|
+
"type": "object",
|
|
68
|
+
"additionalProperties": true,
|
|
69
|
+
"properties": {
|
|
70
|
+
"id": {
|
|
71
|
+
"type": "integer"
|
|
72
|
+
},
|
|
73
|
+
"name": {
|
|
74
|
+
"type": "string"
|
|
75
|
+
},
|
|
76
|
+
"email": {
|
|
77
|
+
"type": "string"
|
|
78
|
+
},
|
|
79
|
+
"role": {
|
|
80
|
+
"type": "string"
|
|
81
|
+
},
|
|
82
|
+
"last_login_at": {
|
|
83
|
+
"type": "string",
|
|
84
|
+
"format": "date-time"
|
|
85
|
+
},
|
|
86
|
+
"created_at": {
|
|
87
|
+
"type": "string",
|
|
88
|
+
"format": "date-time"
|
|
89
|
+
},
|
|
90
|
+
"updated_at": {
|
|
91
|
+
"type": "string",
|
|
92
|
+
"format": "date-time"
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
"supported_sync_modes": [
|
|
97
|
+
"incremental"
|
|
98
|
+
],
|
|
99
|
+
"source_defined_cursor": true,
|
|
100
|
+
"default_cursor_field": [
|
|
101
|
+
"updated_at"
|
|
102
|
+
],
|
|
103
|
+
"source_defined_primary_key": [
|
|
104
|
+
[
|
|
105
|
+
"id"
|
|
106
|
+
]
|
|
107
|
+
]
|
|
108
|
+
}
|
|
109
|
+
]
|
|
110
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Zendesk",
|
|
4
|
+
"title": "Zendesk",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Customer Support",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-zendesk",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": [
|
|
14
|
+
"language:ruby",
|
|
15
|
+
"multiwoven"
|
|
16
|
+
]
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/destinations/customer-support/zendesk",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Zendesk",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": [
|
|
9
|
+
"username",
|
|
10
|
+
"password",
|
|
11
|
+
"subdomain"
|
|
12
|
+
],
|
|
13
|
+
"properties": {
|
|
14
|
+
"username": {
|
|
15
|
+
"type": "string",
|
|
16
|
+
"title": "Username",
|
|
17
|
+
"order": 0
|
|
18
|
+
},
|
|
19
|
+
"password": {
|
|
20
|
+
"type": "string",
|
|
21
|
+
"title": "Password",
|
|
22
|
+
"order": 1,
|
|
23
|
+
"multiwoven_secret": true
|
|
24
|
+
},
|
|
25
|
+
"subdomain": {
|
|
26
|
+
"type": "string",
|
|
27
|
+
"title": "Zendesk Subdomain (include https://)",
|
|
28
|
+
"order": 2
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
<?xml version="1.0" standalone="no"?>
|
|
2
|
+
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
|
|
3
|
+
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
|
|
4
|
+
<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
|
|
5
|
+
width="1280.000000pt" height="913.000000pt" viewBox="0 0 1280.000000 913.000000"
|
|
6
|
+
preserveAspectRatio="xMidYMid meet">
|
|
7
|
+
|
|
8
|
+
<g transform="translate(0.000000,913.000000) scale(0.100000,-0.100000)"
|
|
9
|
+
fill="#000000" stroke="none">
|
|
10
|
+
<path d="M2836 9058 c30 -442 184 -789 483 -1089 777 -776 2073 -593 2604 368
|
|
11
|
+
118 214 182 435 203 706 l7 87 -1651 0 -1651 0 5 -72z"/>
|
|
12
|
+
<path d="M6670 7143 l0 -1988 24 25 c13 14 416 500 897 1080 480 580 1210
|
|
13
|
+
1461 1621 1957 411 497 748 905 748 908 0 3 -740 5 -1645 5 l-1645 0 0 -1987z"/>
|
|
14
|
+
<path d="M5913 7440 c-116 -140 -855 -1031 -1641 -1980 l-1429 -1725 1640 -3
|
|
15
|
+
c902 -1 1642 -1 1644 1 2 2 2 894 1 1983 l-3 1979 -212 -255z"/>
|
|
16
|
+
<path d="M8137 5359 c-804 -93 -1430 -760 -1459 -1554 l-3 -80 1643 -3 c1417
|
|
17
|
+
-2 1643 0 1648 12 8 22 -14 228 -36 334 -114 549 -530 1024 -1055 1205 -245
|
|
18
|
+
84 -500 114 -738 86z"/>
|
|
19
|
+
<path d="M6927 2433 c-4 -3 -7 -222 -7 -485 l0 -478 -72 61 c-82 68 -201 131
|
|
20
|
+
-298 156 -86 22 -270 22 -365 0 -312 -75 -535 -294 -621 -612 -25 -93 -30
|
|
21
|
+
-292 -10 -399 60 -318 275 -561 580 -655 57 -18 93 -21 228 -21 145 0 167 2
|
|
22
|
+
236 26 99 33 181 83 260 158 l62 59 0 -111 0 -112 175 0 175 0 0 1210 0 1210
|
|
23
|
+
-168 0 c-93 0 -172 -3 -175 -7z m-383 -1088 c94 -25 166 -66 237 -135 210
|
|
24
|
+
-206 212 -534 4 -746 -201 -205 -534 -205 -740 1 -287 287 -155 774 238 880
|
|
25
|
+
69 18 189 19 261 0z"/>
|
|
26
|
+
<path d="M11260 1230 l0 -1210 175 0 175 0 0 215 0 215 135 147 c75 82 139
|
|
27
|
+
149 143 151 4 1 122 -162 262 -362 l255 -365 199 0 198 -1 -18 28 c-9 15 -150
|
|
28
|
+
216 -312 447 -162 231 -303 432 -313 446 l-19 27 296 323 c163 178 308 336
|
|
29
|
+
322 352 l26 27 -215 0 -214 0 -370 -407 -370 -406 -3 791 -2 792 -175 0 -175
|
|
30
|
+
0 0 -1210z"/>
|
|
31
|
+
<path d="M2388 1695 c-432 -82 -721 -456 -695 -899 22 -371 231 -644 582 -763
|
|
32
|
+
75 -25 93 -27 303 -31 216 -4 225 -3 314 23 50 14 126 44 168 66 82 42 210
|
|
33
|
+
141 210 162 -1 6 -50 63 -109 126 l-108 114 -54 -40 c-194 -146 -470 -176
|
|
34
|
+
-677 -75 -106 51 -211 188 -229 297 l-6 35 647 0 646 0 0 93 c0 299 -120 583
|
|
35
|
+
-308 732 -76 61 -198 120 -297 145 -107 27 -286 34 -387 15z m262 -325 c123
|
|
36
|
+
-23 223 -95 278 -197 26 -51 62 -151 62 -175 0 -4 -205 -8 -454 -8 l-455 0 16
|
|
37
|
+
53 c73 245 294 376 553 327z"/>
|
|
38
|
+
<path d="M4375 1704 c-78 -12 -181 -38 -236 -60 -157 -64 -289 -184 -358 -326
|
|
39
|
+
-77 -156 -75 -138 -79 -750 l-3 -548 180 0 181 0 0 489 c0 286 4 510 10 541
|
|
40
|
+
15 80 55 159 107 212 141 143 433 153 584 20 44 -39 93 -130 108 -200 7 -35
|
|
41
|
+
11 -227 11 -557 l0 -505 181 0 180 0 -4 553 c-3 493 -5 558 -21 612 -76 255
|
|
42
|
+
-274 436 -546 500 -67 15 -242 27 -295 19z"/>
|
|
43
|
+
<path d="M8370 1704 c-14 -2 -52 -9 -85 -15 -128 -23 -272 -94 -388 -193 -109
|
|
44
|
+
-91 -206 -244 -253 -401 -25 -82 -28 -104 -27 -245 0 -184 17 -264 87 -405
|
|
45
|
+
103 -208 278 -349 521 -420 81 -23 102 -25 295 -25 192 0 214 2 294 25 131 38
|
|
46
|
+
247 101 338 183 21 19 38 39 38 46 -1 6 -49 62 -109 125 l-108 113 -48 -36
|
|
47
|
+
c-238 -182 -604 -183 -790 -1 -55 53 -111 155 -122 218 l-6 37 647 0 646 0 0
|
|
48
|
+
93 c0 448 -230 782 -604 877 -77 19 -272 34 -326 24z m200 -334 c123 -23 223
|
|
49
|
+
-95 278 -197 26 -51 62 -151 62 -175 0 -4 -205 -8 -454 -8 l-455 0 16 53 c46
|
|
50
|
+
157 147 266 288 310 89 28 177 34 265 17z"/>
|
|
51
|
+
<path d="M10070 1695 c-365 -74 -562 -393 -420 -681 73 -149 225 -229 572
|
|
52
|
+
-303 228 -49 314 -84 343 -141 47 -91 -25 -216 -145 -254 -172 -55 -370 -6
|
|
53
|
+
-491 121 -31 32 -60 67 -67 78 -10 19 -18 16 -166 -60 -86 -44 -156 -84 -156
|
|
54
|
+
-90 0 -22 108 -152 165 -198 75 -61 196 -121 299 -147 104 -26 394 -29 490 -4
|
|
55
|
+
213 54 387 222 427 411 29 144 -12 296 -106 387 -93 91 -203 137 -477 202
|
|
56
|
+
-232 54 -292 77 -340 124 -32 32 -38 44 -38 79 0 91 67 161 177 187 165 39
|
|
57
|
+
323 -6 433 -122 l50 -53 145 80 c167 93 163 82 65 180 -145 146 -348 220 -594
|
|
58
|
+
218 -55 0 -130 -7 -166 -14z"/>
|
|
59
|
+
<path d="M22 1513 l3 -158 483 -5 483 -5 -496 -504 -495 -505 0 -158 0 -158
|
|
60
|
+
735 0 735 0 0 155 0 155 -500 0 c-275 0 -500 3 -500 6 0 3 223 232 495 507
|
|
61
|
+
l495 502 0 163 0 162 -720 0 -721 0 3 -157z"/>
|
|
62
|
+
</g>
|
|
63
|
+
</svg>
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.1.
|
|
5
|
+
VERSION = "0.1.67"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -11,6 +11,7 @@ module Multiwoven
|
|
|
11
11
|
Postgresql
|
|
12
12
|
Databricks
|
|
13
13
|
SalesforceConsumerGoodsCloud
|
|
14
|
+
AwsAthena
|
|
14
15
|
].freeze
|
|
15
16
|
|
|
16
17
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -25,6 +26,7 @@ module Multiwoven
|
|
|
25
26
|
SalesforceConsumerGoodsCloud
|
|
26
27
|
Sftp
|
|
27
28
|
Postgresql
|
|
29
|
+
Zendesk
|
|
28
30
|
Http
|
|
29
31
|
].freeze
|
|
30
32
|
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module AwsAthena
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
athena_client = create_connection(connection_config)
|
|
10
|
+
athena_client.list_work_groups
|
|
11
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
12
|
+
rescue StandardError => e
|
|
13
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def discover(connection_config)
|
|
17
|
+
connection_config = connection_config.with_indifferent_access
|
|
18
|
+
query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:schema]}' ORDER BY table_name, ordinal_position;"
|
|
19
|
+
db = create_connection(connection_config)
|
|
20
|
+
results = query_execution(db, query)
|
|
21
|
+
catalog = Catalog.new(streams: create_streams(results))
|
|
22
|
+
catalog.to_multiwoven_message
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
handle_exception(
|
|
25
|
+
"AWS:ATHENA:DISCOVER:EXCEPTION",
|
|
26
|
+
"error",
|
|
27
|
+
e
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def read(sync_config)
|
|
32
|
+
connection_config = sync_config.source.connection_specification
|
|
33
|
+
connection_config = connection_config.with_indifferent_access
|
|
34
|
+
query = sync_config.model.query
|
|
35
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
36
|
+
db = create_connection(connection_config)
|
|
37
|
+
query(db, query)
|
|
38
|
+
rescue StandardError => e
|
|
39
|
+
handle_exception(
|
|
40
|
+
"AWS:ATHENA:READ:EXCEPTION",
|
|
41
|
+
"error",
|
|
42
|
+
e
|
|
43
|
+
)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def create_connection(connection_config)
|
|
49
|
+
Aws.config.update({ credentials: Aws::Credentials.new(connection_config[:access_key], connection_config[:secret_access_key]), region: connection_config[:region] })
|
|
50
|
+
@database = connection_config[:schema]
|
|
51
|
+
@output_location = connection_config[:output_location]
|
|
52
|
+
Aws::Athena::Client.new
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def query_execution(db, query)
|
|
56
|
+
response = db.start_query_execution(
|
|
57
|
+
query_string: query,
|
|
58
|
+
query_execution_context: { database: @database },
|
|
59
|
+
result_configuration: { output_location: @output_location }
|
|
60
|
+
)
|
|
61
|
+
query_execution_id = response[:query_execution_id]
|
|
62
|
+
loop do
|
|
63
|
+
response = db.get_query_execution(query_execution_id: query_execution_id)
|
|
64
|
+
status = response.query_execution.status.state
|
|
65
|
+
break if %w[SUCCEEDED FAILED CANCELLED].include?(status)
|
|
66
|
+
end
|
|
67
|
+
transform_results(db.get_query_results(query_execution_id: query_execution_id))
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def create_streams(records)
|
|
71
|
+
group_by_table(records).map do |_, r|
|
|
72
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def transform_results(results)
|
|
77
|
+
columns = results.result_set.result_set_metadata.column_info.map(&:name)
|
|
78
|
+
rows = results.result_set.rows.map do |row|
|
|
79
|
+
row.data.map(&:var_char_value)
|
|
80
|
+
end
|
|
81
|
+
rows.map { |row| columns.zip(row).to_h }
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def query(db, query)
|
|
85
|
+
records = []
|
|
86
|
+
query_execution(db, query).map do |row|
|
|
87
|
+
records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
88
|
+
end
|
|
89
|
+
records
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def group_by_table(records)
|
|
93
|
+
result = {}
|
|
94
|
+
records.each_with_index do |entry, index|
|
|
95
|
+
table_name = entry["table_name"]
|
|
96
|
+
column_data = {
|
|
97
|
+
column_name: entry["column_name"],
|
|
98
|
+
data_type: entry["data_type"],
|
|
99
|
+
is_nullable: entry["is_nullable"] == "YES"
|
|
100
|
+
}
|
|
101
|
+
result[index] ||= {}
|
|
102
|
+
result[index][:tablename] = table_name
|
|
103
|
+
result[index][:columns] = [column_data]
|
|
104
|
+
end
|
|
105
|
+
result
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "AwsAthena",
|
|
4
|
+
"title": "AWS Athena",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "source-aws-athena",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/athena",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "AWS Athena",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["access_key", "secret_access_key", "region", "workgroup", "catalog", "output_location"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"access_key": {
|
|
12
|
+
"description": "The AWS Access Key ID to use for authentication.",
|
|
13
|
+
"examples": ["AWSATHENAACCESS"],
|
|
14
|
+
"type": "string",
|
|
15
|
+
"title": "Personal Access Key",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"secret_access_key": {
|
|
19
|
+
"description": "The AWS Secret Access Key to use for authentication.",
|
|
20
|
+
"examples": ["AWSATHENASECRET"],
|
|
21
|
+
"type": "string",
|
|
22
|
+
"multiwoven_secret": true,
|
|
23
|
+
"title": "Secret Access Key",
|
|
24
|
+
"order": 1
|
|
25
|
+
},
|
|
26
|
+
"region": {
|
|
27
|
+
"description": "AWS region where Athena is located.",
|
|
28
|
+
"examples": ["ATHENA_REGION"],
|
|
29
|
+
"type": "string",
|
|
30
|
+
"title": "Secret Access Key",
|
|
31
|
+
"order": 2
|
|
32
|
+
},
|
|
33
|
+
"workgroup": {
|
|
34
|
+
"description": "The Athena workgroup you previously set up in AWS.",
|
|
35
|
+
"examples": ["ATHENA_WORKGROUP"],
|
|
36
|
+
"type": "string",
|
|
37
|
+
"title": "Workgroup",
|
|
38
|
+
"order": 3
|
|
39
|
+
},
|
|
40
|
+
"catalog": {
|
|
41
|
+
"description": "The Data catalog name within Athena.",
|
|
42
|
+
"examples": ["ATHENA_CATALOG"],
|
|
43
|
+
"type": "string",
|
|
44
|
+
"title": "Catalog",
|
|
45
|
+
"order": 4
|
|
46
|
+
},
|
|
47
|
+
"schema": {
|
|
48
|
+
"description": "The specific Athena database/schema to connect to.",
|
|
49
|
+
"examples": ["ATHENA_DB"],
|
|
50
|
+
"type": "string",
|
|
51
|
+
"title": "Database",
|
|
52
|
+
"order": 5
|
|
53
|
+
},
|
|
54
|
+
"output_location": {
|
|
55
|
+
"description": "S3 path for query output.",
|
|
56
|
+
"examples": ["s3://example-bucket-name/query-results/"],
|
|
57
|
+
"type": "string",
|
|
58
|
+
"title": "Query",
|
|
59
|
+
"order": 6
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
<svg height="100" viewBox="0 0 100 100" width="100" xmlns="http://www.w3.org/2000/svg">
|
|
2
|
+
<path d="m84 64.68-34-4.19-34 4.19 34 26.62z" fill="#fcbf92"/>
|
|
3
|
+
<path d="m16 64.68 34 10.17v16.45l-34-17.05z" fill="#9d5025"/>
|
|
4
|
+
<path d="m84 64.68-34 10.17v16.45l34-17.05z" fill="#f58534"/>
|
|
5
|
+
<path d="m20.39 50.22h-4.39v9.67l4.39.8 4.88-5.18z" fill="#9d5025"/>
|
|
6
|
+
<path d="m25.27 60.29-4.88.4v-10.47h4.88z" fill="#f58534"/>
|
|
7
|
+
<path d="m28.56 38.16-5.78 1.09v21.94l5.78 1.2 5.29-12.17z" fill="#9d5025"/>
|
|
8
|
+
<path d="m33.85 61.79-5.29.6v-24.23l5.29.6z" fill="#f58534"/>
|
|
9
|
+
<path d="m39.83 43.04-7.98.8v19.24l7.98 1.6 8.47-10.87z" fill="#9d5025"/>
|
|
10
|
+
<path d="m45.21 63.78-5.38.9v-21.64l5.38.4z" fill="#f58534"/>
|
|
11
|
+
<g fill="#9d5025">
|
|
12
|
+
<path d="m50 25.59-5.38 1.7v38.29l5.38 1.09 5.38-20.54z"/>
|
|
13
|
+
<path d="m54.98 63.78 5.19.9 7.98-14.46-7.98-14.36-5.19.8z"/>
|
|
14
|
+
<path d="m68.15 50.22-7.98-14.36-5.19.8"/>
|
|
15
|
+
<path d="m66.05 61.79 5.39.6 5.78-18.25-5.78-18.15-5.39 1.3z"/>
|
|
16
|
+
<path d="m74.73 60.29 4.88.4 4.39-18.05-4.39-18.64-4.88 1.2z"/>
|
|
17
|
+
</g>
|
|
18
|
+
<path d="m50 25.59 5.38 1.7v38.29l-5.38 1.09z" fill="#f58534"/>
|
|
19
|
+
<path d="m68.15 37.46-7.98-1.6v28.82l7.98-1.6z" fill="#f58534"/>
|
|
20
|
+
<path d="m77.22 28.29-5.78-2.3v36.4l5.78-1.2z" fill="#f58534"/>
|
|
21
|
+
<path d="m84 26.19-4.39-2.19v36.69l4.39-.8z" fill="#f58534"/>
|
|
22
|
+
</svg>
|
|
@@ -22,7 +22,9 @@ require "stripe"
|
|
|
22
22
|
require "net/sftp"
|
|
23
23
|
require "csv"
|
|
24
24
|
require "securerandom"
|
|
25
|
+
require "aws-sdk-athena"
|
|
25
26
|
require "zip"
|
|
27
|
+
require "zendesk_api"
|
|
26
28
|
|
|
27
29
|
# Service
|
|
28
30
|
require_relative "integrations/config"
|
|
@@ -48,6 +50,7 @@ require_relative "integrations/source/bigquery/client"
|
|
|
48
50
|
require_relative "integrations/source/postgresql/client"
|
|
49
51
|
require_relative "integrations/source/databricks/client"
|
|
50
52
|
require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
|
|
53
|
+
require_relative "integrations/source/aws_athena/client"
|
|
51
54
|
|
|
52
55
|
# Destination
|
|
53
56
|
require_relative "integrations/destination/klaviyo/client"
|
|
@@ -61,6 +64,7 @@ require_relative "integrations/destination/stripe/client"
|
|
|
61
64
|
require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
|
|
62
65
|
require_relative "integrations/destination/sftp/client"
|
|
63
66
|
require_relative "integrations/destination/postgresql/client"
|
|
67
|
+
require_relative "integrations/destination/zendesk/client"
|
|
64
68
|
require_relative "integrations/destination/http/client"
|
|
65
69
|
|
|
66
70
|
module Multiwoven
|
|
@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
|
|
|
35
35
|
|
|
36
36
|
spec.add_runtime_dependency "activesupport"
|
|
37
37
|
spec.add_runtime_dependency "async-websocket"
|
|
38
|
+
spec.add_runtime_dependency "aws-sdk-athena"
|
|
38
39
|
spec.add_runtime_dependency "csv"
|
|
39
40
|
spec.add_runtime_dependency "dry-schema"
|
|
40
41
|
spec.add_runtime_dependency "dry-struct"
|
|
@@ -53,6 +54,7 @@ Gem::Specification.new do |spec|
|
|
|
53
54
|
spec.add_runtime_dependency "sequel"
|
|
54
55
|
spec.add_runtime_dependency "slack-ruby-client"
|
|
55
56
|
spec.add_runtime_dependency "stripe"
|
|
57
|
+
spec.add_runtime_dependency "zendesk_api"
|
|
56
58
|
|
|
57
59
|
spec.add_development_dependency "byebug"
|
|
58
60
|
spec.add_development_dependency "rspec"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.67
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-05-
|
|
11
|
+
date: 2024-05-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -38,6 +38,20 @@ dependencies:
|
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: aws-sdk-athena
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
41
55
|
- !ruby/object:Gem::Dependency
|
|
42
56
|
name: csv
|
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -290,6 +304,20 @@ dependencies:
|
|
|
290
304
|
- - ">="
|
|
291
305
|
- !ruby/object:Gem::Version
|
|
292
306
|
version: '0'
|
|
307
|
+
- !ruby/object:Gem::Dependency
|
|
308
|
+
name: zendesk_api
|
|
309
|
+
requirement: !ruby/object:Gem::Requirement
|
|
310
|
+
requirements:
|
|
311
|
+
- - ">="
|
|
312
|
+
- !ruby/object:Gem::Version
|
|
313
|
+
version: '0'
|
|
314
|
+
type: :runtime
|
|
315
|
+
prerelease: false
|
|
316
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
317
|
+
requirements:
|
|
318
|
+
- - ">="
|
|
319
|
+
- !ruby/object:Gem::Version
|
|
320
|
+
version: '0'
|
|
293
321
|
- !ruby/object:Gem::Dependency
|
|
294
322
|
name: byebug
|
|
295
323
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -463,10 +491,19 @@ files:
|
|
|
463
491
|
- lib/multiwoven/integrations/destination/stripe/config/meta.json
|
|
464
492
|
- lib/multiwoven/integrations/destination/stripe/config/spec.json
|
|
465
493
|
- lib/multiwoven/integrations/destination/stripe/icon.svg
|
|
494
|
+
- lib/multiwoven/integrations/destination/zendesk/client.rb
|
|
495
|
+
- lib/multiwoven/integrations/destination/zendesk/config/catalog.json
|
|
496
|
+
- lib/multiwoven/integrations/destination/zendesk/config/meta.json
|
|
497
|
+
- lib/multiwoven/integrations/destination/zendesk/config/spec.json
|
|
498
|
+
- lib/multiwoven/integrations/destination/zendesk/icon.svg
|
|
466
499
|
- lib/multiwoven/integrations/protocol/protocol.json
|
|
467
500
|
- lib/multiwoven/integrations/protocol/protocol.rb
|
|
468
501
|
- lib/multiwoven/integrations/rollout.rb
|
|
469
502
|
- lib/multiwoven/integrations/service.rb
|
|
503
|
+
- lib/multiwoven/integrations/source/aws_athena/client.rb
|
|
504
|
+
- lib/multiwoven/integrations/source/aws_athena/config/meta.json
|
|
505
|
+
- lib/multiwoven/integrations/source/aws_athena/config/spec.json
|
|
506
|
+
- lib/multiwoven/integrations/source/aws_athena/icon.svg
|
|
470
507
|
- lib/multiwoven/integrations/source/bigquery/client.rb
|
|
471
508
|
- lib/multiwoven/integrations/source/bigquery/config/meta.json
|
|
472
509
|
- lib/multiwoven/integrations/source/bigquery/config/spec.json
|