multiwoven-integrations 0.1.65 → 0.1.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/constants.rb +2 -0
- data/lib/multiwoven/integrations/destination/zendesk/client.rb +118 -0
- data/lib/multiwoven/integrations/destination/zendesk/config/catalog.json +110 -0
- data/lib/multiwoven/integrations/destination/zendesk/config/meta.json +18 -0
- data/lib/multiwoven/integrations/destination/zendesk/config/spec.json +32 -0
- data/lib/multiwoven/integrations/destination/zendesk/icon.svg +63 -0
- data/lib/multiwoven/integrations/rollout.rb +3 -1
- data/lib/multiwoven/integrations/source/aws_athena/client.rb +109 -0
- data/lib/multiwoven/integrations/source/aws_athena/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/aws_athena/config/spec.json +63 -0
- data/lib/multiwoven/integrations/source/aws_athena/icon.svg +22 -0
- data/lib/multiwoven/integrations.rb +4 -0
- data/multiwoven-integrations.gemspec +2 -0
- metadata +39 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: da70c8d91dea6167ebb4103cfe0e1999d663ee2ac420d9ae272a5dbd70924380
|
|
4
|
+
data.tar.gz: bc6fd6fd71c8c02dcdcf4e1f3eaf69dee37f3df6ba5f6fb1d4af49aa64220119
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2040a6232add940bd41cffa131c5738d64a4187df2615f3625910aa6129ae089ffd89027ff78df15d753a27f5eb10cec0357f4b977f25d86c9663480dce88c71
|
|
7
|
+
data.tar.gz: cfe7e24682df19fd0aa10284891d073324933838ceb10bfab3a8dc7259705af422b86d4b5fc85665af580a2097625fb0feafdd05152ea9d2f5bf6d0d0e51fff4
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations
|
|
5
|
+
module Destination
|
|
6
|
+
module Zendesk
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < DestinationConnector
|
|
9
|
+
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
10
|
+
def check_connection(connection_config)
|
|
11
|
+
connection_config = connection_config.with_indifferent_access
|
|
12
|
+
initialize_client(connection_config)
|
|
13
|
+
authenticate_client
|
|
14
|
+
success_status
|
|
15
|
+
rescue StandardError => e
|
|
16
|
+
handle_exception("ZENDESK:CHECK_CONNECTION:EXCEPTION", "error", e)
|
|
17
|
+
failure_status(e)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def discover(_connection_config = nil)
|
|
21
|
+
catalog = build_catalog(load_catalog)
|
|
22
|
+
catalog.to_multiwoven_message
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
handle_exception("ZENDESK:DISCOVER:EXCEPTION", "error", e)
|
|
25
|
+
failure_status(e)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def write(sync_config, records, action = "create")
|
|
29
|
+
@action = sync_config.stream.action || action
|
|
30
|
+
initialize_client(sync_config.destination.connection_specification)
|
|
31
|
+
process_records(records, sync_config.stream)
|
|
32
|
+
rescue StandardError => e
|
|
33
|
+
handle_exception("ZENDESK:WRITE:EXCEPTION", "error", e)
|
|
34
|
+
failure_status(e)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def initialize_client(connection_config)
|
|
40
|
+
connection_config = connection_config.with_indifferent_access
|
|
41
|
+
@client = ZendeskAPI::Client.new do |config|
|
|
42
|
+
config.url = "#{connection_config[:subdomain]}.#{ZENDESK_URL_SUFFIX}"
|
|
43
|
+
config.username = connection_config[:username]
|
|
44
|
+
config.password = connection_config[:password]
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def authenticate_client
|
|
49
|
+
@client.tickets.page(1).per_page(1).fetch
|
|
50
|
+
rescue ZendeskAPI::Error => e
|
|
51
|
+
raise StandardError, "Authentication failed: #{e.message}"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def process_records(records, stream)
|
|
55
|
+
write_success = 0
|
|
56
|
+
write_failure = 0
|
|
57
|
+
|
|
58
|
+
records.each do |record|
|
|
59
|
+
zendesk_data = prepare_record_data(record, stream.name)
|
|
60
|
+
plural_stream_name = pluralize_stream_name(stream.name.downcase)
|
|
61
|
+
|
|
62
|
+
if @action == "create"
|
|
63
|
+
@client.send(plural_stream_name).create!(zendesk_data)
|
|
64
|
+
else
|
|
65
|
+
existing_record = @client.send(plural_stream_name).find(id: record[:id])
|
|
66
|
+
existing_record.update!(zendesk_data)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
write_success += 1
|
|
70
|
+
rescue StandardError => e
|
|
71
|
+
handle_exception("ZENDESK:WRITE_RECORD:EXCEPTION", "error", e)
|
|
72
|
+
write_failure += 1
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
tracking_message(write_success, write_failure)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def pluralize_stream_name(name)
|
|
79
|
+
{ "ticket" => "tickets", "user" => "users" }.fetch(name, name)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def prepare_record_data(record, type)
|
|
83
|
+
case type
|
|
84
|
+
when "Tickets"
|
|
85
|
+
{
|
|
86
|
+
subject: record[:subject],
|
|
87
|
+
comment: { body: record[:description] },
|
|
88
|
+
priority: record[:priority],
|
|
89
|
+
status: record[:status],
|
|
90
|
+
requester_id: record[:requester_id],
|
|
91
|
+
assignee_id: record[:assignee_id],
|
|
92
|
+
tags: record[:tags]
|
|
93
|
+
}
|
|
94
|
+
when "Users"
|
|
95
|
+
{
|
|
96
|
+
name: record[:name],
|
|
97
|
+
email: record[:email],
|
|
98
|
+
role: record[:role]
|
|
99
|
+
}
|
|
100
|
+
else
|
|
101
|
+
raise StandardError, "Unsupported record type: #{type}"
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def load_catalog
|
|
106
|
+
read_json(CATALOG_SPEC_PATH)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def tracking_message(success, failure)
|
|
110
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
111
|
+
success: success, failed: failure
|
|
112
|
+
).to_multiwoven_message
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
{
|
|
2
|
+
"request_rate_limit": 700,
|
|
3
|
+
"request_rate_limit_unit": "minute",
|
|
4
|
+
"request_rate_concurrency": 10,
|
|
5
|
+
"streams": [
|
|
6
|
+
{
|
|
7
|
+
"name": "tickets",
|
|
8
|
+
"action": "create",
|
|
9
|
+
"json_schema": {
|
|
10
|
+
"type": "object",
|
|
11
|
+
"additionalProperties": true,
|
|
12
|
+
"properties": {
|
|
13
|
+
"id": {
|
|
14
|
+
"type": "integer"
|
|
15
|
+
},
|
|
16
|
+
"subject": {
|
|
17
|
+
"type": "string"
|
|
18
|
+
},
|
|
19
|
+
"description": {
|
|
20
|
+
"type": "string"
|
|
21
|
+
},
|
|
22
|
+
"status": {
|
|
23
|
+
"type": "string"
|
|
24
|
+
},
|
|
25
|
+
"priority": {
|
|
26
|
+
"type": "string"
|
|
27
|
+
},
|
|
28
|
+
"requester_id": {
|
|
29
|
+
"type": "integer"
|
|
30
|
+
},
|
|
31
|
+
"assignee_id": {
|
|
32
|
+
"type": "integer"
|
|
33
|
+
},
|
|
34
|
+
"tags": {
|
|
35
|
+
"type": "array",
|
|
36
|
+
"items": {
|
|
37
|
+
"type": "string"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"created_at": {
|
|
41
|
+
"type": "string",
|
|
42
|
+
"format": "date-time"
|
|
43
|
+
},
|
|
44
|
+
"updated_at": {
|
|
45
|
+
"type": "string",
|
|
46
|
+
"format": "date-time"
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
"supported_sync_modes": [
|
|
51
|
+
"incremental"
|
|
52
|
+
],
|
|
53
|
+
"source_defined_cursor": true,
|
|
54
|
+
"default_cursor_field": [
|
|
55
|
+
"updated_at"
|
|
56
|
+
],
|
|
57
|
+
"source_defined_primary_key": [
|
|
58
|
+
[
|
|
59
|
+
"id"
|
|
60
|
+
]
|
|
61
|
+
]
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"name": "users",
|
|
65
|
+
"action": "create",
|
|
66
|
+
"json_schema": {
|
|
67
|
+
"type": "object",
|
|
68
|
+
"additionalProperties": true,
|
|
69
|
+
"properties": {
|
|
70
|
+
"id": {
|
|
71
|
+
"type": "integer"
|
|
72
|
+
},
|
|
73
|
+
"name": {
|
|
74
|
+
"type": "string"
|
|
75
|
+
},
|
|
76
|
+
"email": {
|
|
77
|
+
"type": "string"
|
|
78
|
+
},
|
|
79
|
+
"role": {
|
|
80
|
+
"type": "string"
|
|
81
|
+
},
|
|
82
|
+
"last_login_at": {
|
|
83
|
+
"type": "string",
|
|
84
|
+
"format": "date-time"
|
|
85
|
+
},
|
|
86
|
+
"created_at": {
|
|
87
|
+
"type": "string",
|
|
88
|
+
"format": "date-time"
|
|
89
|
+
},
|
|
90
|
+
"updated_at": {
|
|
91
|
+
"type": "string",
|
|
92
|
+
"format": "date-time"
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
"supported_sync_modes": [
|
|
97
|
+
"incremental"
|
|
98
|
+
],
|
|
99
|
+
"source_defined_cursor": true,
|
|
100
|
+
"default_cursor_field": [
|
|
101
|
+
"updated_at"
|
|
102
|
+
],
|
|
103
|
+
"source_defined_primary_key": [
|
|
104
|
+
[
|
|
105
|
+
"id"
|
|
106
|
+
]
|
|
107
|
+
]
|
|
108
|
+
}
|
|
109
|
+
]
|
|
110
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Zendesk",
|
|
4
|
+
"title": "Zendesk",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Customer Support",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-zendesk",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": [
|
|
14
|
+
"language:ruby",
|
|
15
|
+
"multiwoven"
|
|
16
|
+
]
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/destinations/customer-support/zendesk",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Zendesk",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": [
|
|
9
|
+
"username",
|
|
10
|
+
"password",
|
|
11
|
+
"subdomain"
|
|
12
|
+
],
|
|
13
|
+
"properties": {
|
|
14
|
+
"username": {
|
|
15
|
+
"type": "string",
|
|
16
|
+
"title": "Username",
|
|
17
|
+
"order": 0
|
|
18
|
+
},
|
|
19
|
+
"password": {
|
|
20
|
+
"type": "string",
|
|
21
|
+
"title": "Password",
|
|
22
|
+
"order": 1,
|
|
23
|
+
"multiwoven_secret": true
|
|
24
|
+
},
|
|
25
|
+
"subdomain": {
|
|
26
|
+
"type": "string",
|
|
27
|
+
"title": "Zendesk Subdomain (include https://)",
|
|
28
|
+
"order": 2
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
<?xml version="1.0" standalone="no"?>
|
|
2
|
+
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
|
|
3
|
+
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
|
|
4
|
+
<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
|
|
5
|
+
width="1280.000000pt" height="913.000000pt" viewBox="0 0 1280.000000 913.000000"
|
|
6
|
+
preserveAspectRatio="xMidYMid meet">
|
|
7
|
+
|
|
8
|
+
<g transform="translate(0.000000,913.000000) scale(0.100000,-0.100000)"
|
|
9
|
+
fill="#000000" stroke="none">
|
|
10
|
+
<path d="M2836 9058 c30 -442 184 -789 483 -1089 777 -776 2073 -593 2604 368
|
|
11
|
+
118 214 182 435 203 706 l7 87 -1651 0 -1651 0 5 -72z"/>
|
|
12
|
+
<path d="M6670 7143 l0 -1988 24 25 c13 14 416 500 897 1080 480 580 1210
|
|
13
|
+
1461 1621 1957 411 497 748 905 748 908 0 3 -740 5 -1645 5 l-1645 0 0 -1987z"/>
|
|
14
|
+
<path d="M5913 7440 c-116 -140 -855 -1031 -1641 -1980 l-1429 -1725 1640 -3
|
|
15
|
+
c902 -1 1642 -1 1644 1 2 2 2 894 1 1983 l-3 1979 -212 -255z"/>
|
|
16
|
+
<path d="M8137 5359 c-804 -93 -1430 -760 -1459 -1554 l-3 -80 1643 -3 c1417
|
|
17
|
+
-2 1643 0 1648 12 8 22 -14 228 -36 334 -114 549 -530 1024 -1055 1205 -245
|
|
18
|
+
84 -500 114 -738 86z"/>
|
|
19
|
+
<path d="M6927 2433 c-4 -3 -7 -222 -7 -485 l0 -478 -72 61 c-82 68 -201 131
|
|
20
|
+
-298 156 -86 22 -270 22 -365 0 -312 -75 -535 -294 -621 -612 -25 -93 -30
|
|
21
|
+
-292 -10 -399 60 -318 275 -561 580 -655 57 -18 93 -21 228 -21 145 0 167 2
|
|
22
|
+
236 26 99 33 181 83 260 158 l62 59 0 -111 0 -112 175 0 175 0 0 1210 0 1210
|
|
23
|
+
-168 0 c-93 0 -172 -3 -175 -7z m-383 -1088 c94 -25 166 -66 237 -135 210
|
|
24
|
+
-206 212 -534 4 -746 -201 -205 -534 -205 -740 1 -287 287 -155 774 238 880
|
|
25
|
+
69 18 189 19 261 0z"/>
|
|
26
|
+
<path d="M11260 1230 l0 -1210 175 0 175 0 0 215 0 215 135 147 c75 82 139
|
|
27
|
+
149 143 151 4 1 122 -162 262 -362 l255 -365 199 0 198 -1 -18 28 c-9 15 -150
|
|
28
|
+
216 -312 447 -162 231 -303 432 -313 446 l-19 27 296 323 c163 178 308 336
|
|
29
|
+
322 352 l26 27 -215 0 -214 0 -370 -407 -370 -406 -3 791 -2 792 -175 0 -175
|
|
30
|
+
0 0 -1210z"/>
|
|
31
|
+
<path d="M2388 1695 c-432 -82 -721 -456 -695 -899 22 -371 231 -644 582 -763
|
|
32
|
+
75 -25 93 -27 303 -31 216 -4 225 -3 314 23 50 14 126 44 168 66 82 42 210
|
|
33
|
+
141 210 162 -1 6 -50 63 -109 126 l-108 114 -54 -40 c-194 -146 -470 -176
|
|
34
|
+
-677 -75 -106 51 -211 188 -229 297 l-6 35 647 0 646 0 0 93 c0 299 -120 583
|
|
35
|
+
-308 732 -76 61 -198 120 -297 145 -107 27 -286 34 -387 15z m262 -325 c123
|
|
36
|
+
-23 223 -95 278 -197 26 -51 62 -151 62 -175 0 -4 -205 -8 -454 -8 l-455 0 16
|
|
37
|
+
53 c73 245 294 376 553 327z"/>
|
|
38
|
+
<path d="M4375 1704 c-78 -12 -181 -38 -236 -60 -157 -64 -289 -184 -358 -326
|
|
39
|
+
-77 -156 -75 -138 -79 -750 l-3 -548 180 0 181 0 0 489 c0 286 4 510 10 541
|
|
40
|
+
15 80 55 159 107 212 141 143 433 153 584 20 44 -39 93 -130 108 -200 7 -35
|
|
41
|
+
11 -227 11 -557 l0 -505 181 0 180 0 -4 553 c-3 493 -5 558 -21 612 -76 255
|
|
42
|
+
-274 436 -546 500 -67 15 -242 27 -295 19z"/>
|
|
43
|
+
<path d="M8370 1704 c-14 -2 -52 -9 -85 -15 -128 -23 -272 -94 -388 -193 -109
|
|
44
|
+
-91 -206 -244 -253 -401 -25 -82 -28 -104 -27 -245 0 -184 17 -264 87 -405
|
|
45
|
+
103 -208 278 -349 521 -420 81 -23 102 -25 295 -25 192 0 214 2 294 25 131 38
|
|
46
|
+
247 101 338 183 21 19 38 39 38 46 -1 6 -49 62 -109 125 l-108 113 -48 -36
|
|
47
|
+
c-238 -182 -604 -183 -790 -1 -55 53 -111 155 -122 218 l-6 37 647 0 646 0 0
|
|
48
|
+
93 c0 448 -230 782 -604 877 -77 19 -272 34 -326 24z m200 -334 c123 -23 223
|
|
49
|
+
-95 278 -197 26 -51 62 -151 62 -175 0 -4 -205 -8 -454 -8 l-455 0 16 53 c46
|
|
50
|
+
157 147 266 288 310 89 28 177 34 265 17z"/>
|
|
51
|
+
<path d="M10070 1695 c-365 -74 -562 -393 -420 -681 73 -149 225 -229 572
|
|
52
|
+
-303 228 -49 314 -84 343 -141 47 -91 -25 -216 -145 -254 -172 -55 -370 -6
|
|
53
|
+
-491 121 -31 32 -60 67 -67 78 -10 19 -18 16 -166 -60 -86 -44 -156 -84 -156
|
|
54
|
+
-90 0 -22 108 -152 165 -198 75 -61 196 -121 299 -147 104 -26 394 -29 490 -4
|
|
55
|
+
213 54 387 222 427 411 29 144 -12 296 -106 387 -93 91 -203 137 -477 202
|
|
56
|
+
-232 54 -292 77 -340 124 -32 32 -38 44 -38 79 0 91 67 161 177 187 165 39
|
|
57
|
+
323 -6 433 -122 l50 -53 145 80 c167 93 163 82 65 180 -145 146 -348 220 -594
|
|
58
|
+
218 -55 0 -130 -7 -166 -14z"/>
|
|
59
|
+
<path d="M22 1513 l3 -158 483 -5 483 -5 -496 -504 -495 -505 0 -158 0 -158
|
|
60
|
+
735 0 735 0 0 155 0 155 -500 0 c-275 0 -500 3 -500 6 0 3 223 232 495 507
|
|
61
|
+
l495 502 0 163 0 162 -720 0 -721 0 3 -157z"/>
|
|
62
|
+
</g>
|
|
63
|
+
</svg>
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.1.
|
|
5
|
+
VERSION = "0.1.67"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -11,6 +11,7 @@ module Multiwoven
|
|
|
11
11
|
Postgresql
|
|
12
12
|
Databricks
|
|
13
13
|
SalesforceConsumerGoodsCloud
|
|
14
|
+
AwsAthena
|
|
14
15
|
].freeze
|
|
15
16
|
|
|
16
17
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -25,6 +26,7 @@ module Multiwoven
|
|
|
25
26
|
SalesforceConsumerGoodsCloud
|
|
26
27
|
Sftp
|
|
27
28
|
Postgresql
|
|
29
|
+
Zendesk
|
|
28
30
|
Http
|
|
29
31
|
].freeze
|
|
30
32
|
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module AwsAthena
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
athena_client = create_connection(connection_config)
|
|
10
|
+
athena_client.list_work_groups
|
|
11
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
12
|
+
rescue StandardError => e
|
|
13
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def discover(connection_config)
|
|
17
|
+
connection_config = connection_config.with_indifferent_access
|
|
18
|
+
query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:schema]}' ORDER BY table_name, ordinal_position;"
|
|
19
|
+
db = create_connection(connection_config)
|
|
20
|
+
results = query_execution(db, query)
|
|
21
|
+
catalog = Catalog.new(streams: create_streams(results))
|
|
22
|
+
catalog.to_multiwoven_message
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
handle_exception(
|
|
25
|
+
"AWS:ATHENA:DISCOVER:EXCEPTION",
|
|
26
|
+
"error",
|
|
27
|
+
e
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def read(sync_config)
|
|
32
|
+
connection_config = sync_config.source.connection_specification
|
|
33
|
+
connection_config = connection_config.with_indifferent_access
|
|
34
|
+
query = sync_config.model.query
|
|
35
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
36
|
+
db = create_connection(connection_config)
|
|
37
|
+
query(db, query)
|
|
38
|
+
rescue StandardError => e
|
|
39
|
+
handle_exception(
|
|
40
|
+
"AWS:ATHENA:READ:EXCEPTION",
|
|
41
|
+
"error",
|
|
42
|
+
e
|
|
43
|
+
)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def create_connection(connection_config)
|
|
49
|
+
Aws.config.update({ credentials: Aws::Credentials.new(connection_config[:access_key], connection_config[:secret_access_key]), region: connection_config[:region] })
|
|
50
|
+
@database = connection_config[:schema]
|
|
51
|
+
@output_location = connection_config[:output_location]
|
|
52
|
+
Aws::Athena::Client.new
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def query_execution(db, query)
|
|
56
|
+
response = db.start_query_execution(
|
|
57
|
+
query_string: query,
|
|
58
|
+
query_execution_context: { database: @database },
|
|
59
|
+
result_configuration: { output_location: @output_location }
|
|
60
|
+
)
|
|
61
|
+
query_execution_id = response[:query_execution_id]
|
|
62
|
+
loop do
|
|
63
|
+
response = db.get_query_execution(query_execution_id: query_execution_id)
|
|
64
|
+
status = response.query_execution.status.state
|
|
65
|
+
break if %w[SUCCEEDED FAILED CANCELLED].include?(status)
|
|
66
|
+
end
|
|
67
|
+
transform_results(db.get_query_results(query_execution_id: query_execution_id))
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def create_streams(records)
|
|
71
|
+
group_by_table(records).map do |_, r|
|
|
72
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def transform_results(results)
|
|
77
|
+
columns = results.result_set.result_set_metadata.column_info.map(&:name)
|
|
78
|
+
rows = results.result_set.rows.map do |row|
|
|
79
|
+
row.data.map(&:var_char_value)
|
|
80
|
+
end
|
|
81
|
+
rows.map { |row| columns.zip(row).to_h }
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def query(db, query)
|
|
85
|
+
records = []
|
|
86
|
+
query_execution(db, query).map do |row|
|
|
87
|
+
records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
88
|
+
end
|
|
89
|
+
records
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def group_by_table(records)
|
|
93
|
+
result = {}
|
|
94
|
+
records.each_with_index do |entry, index|
|
|
95
|
+
table_name = entry["table_name"]
|
|
96
|
+
column_data = {
|
|
97
|
+
column_name: entry["column_name"],
|
|
98
|
+
data_type: entry["data_type"],
|
|
99
|
+
is_nullable: entry["is_nullable"] == "YES"
|
|
100
|
+
}
|
|
101
|
+
result[index] ||= {}
|
|
102
|
+
result[index][:tablename] = table_name
|
|
103
|
+
result[index][:columns] = [column_data]
|
|
104
|
+
end
|
|
105
|
+
result
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "AwsAthena",
|
|
4
|
+
"title": "AWS Athena",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "source-aws-athena",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/athena",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "AWS Athena",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["access_key", "secret_access_key", "region", "workgroup", "catalog", "output_location"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"access_key": {
|
|
12
|
+
"description": "The AWS Access Key ID to use for authentication.",
|
|
13
|
+
"examples": ["AWSATHENAACCESS"],
|
|
14
|
+
"type": "string",
|
|
15
|
+
"title": "Personal Access Key",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"secret_access_key": {
|
|
19
|
+
"description": "The AWS Secret Access Key to use for authentication.",
|
|
20
|
+
"examples": ["AWSATHENASECRET"],
|
|
21
|
+
"type": "string",
|
|
22
|
+
"multiwoven_secret": true,
|
|
23
|
+
"title": "Secret Access Key",
|
|
24
|
+
"order": 1
|
|
25
|
+
},
|
|
26
|
+
"region": {
|
|
27
|
+
"description": "AWS region where Athena is located.",
|
|
28
|
+
"examples": ["ATHENA_REGION"],
|
|
29
|
+
"type": "string",
|
|
30
|
+
"title": "Secret Access Key",
|
|
31
|
+
"order": 2
|
|
32
|
+
},
|
|
33
|
+
"workgroup": {
|
|
34
|
+
"description": "The Athena workgroup you previously set up in AWS.",
|
|
35
|
+
"examples": ["ATHENA_WORKGROUP"],
|
|
36
|
+
"type": "string",
|
|
37
|
+
"title": "Workgroup",
|
|
38
|
+
"order": 3
|
|
39
|
+
},
|
|
40
|
+
"catalog": {
|
|
41
|
+
"description": "The Data catalog name within Athena.",
|
|
42
|
+
"examples": ["ATHENA_CATALOG"],
|
|
43
|
+
"type": "string",
|
|
44
|
+
"title": "Catalog",
|
|
45
|
+
"order": 4
|
|
46
|
+
},
|
|
47
|
+
"schema": {
|
|
48
|
+
"description": "The specific Athena database/schema to connect to.",
|
|
49
|
+
"examples": ["ATHENA_DB"],
|
|
50
|
+
"type": "string",
|
|
51
|
+
"title": "Database",
|
|
52
|
+
"order": 5
|
|
53
|
+
},
|
|
54
|
+
"output_location": {
|
|
55
|
+
"description": "S3 path for query output.",
|
|
56
|
+
"examples": ["s3://example-bucket-name/query-results/"],
|
|
57
|
+
"type": "string",
|
|
58
|
+
"title": "Query",
|
|
59
|
+
"order": 6
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
<svg height="100" viewBox="0 0 100 100" width="100" xmlns="http://www.w3.org/2000/svg">
|
|
2
|
+
<path d="m84 64.68-34-4.19-34 4.19 34 26.62z" fill="#fcbf92"/>
|
|
3
|
+
<path d="m16 64.68 34 10.17v16.45l-34-17.05z" fill="#9d5025"/>
|
|
4
|
+
<path d="m84 64.68-34 10.17v16.45l34-17.05z" fill="#f58534"/>
|
|
5
|
+
<path d="m20.39 50.22h-4.39v9.67l4.39.8 4.88-5.18z" fill="#9d5025"/>
|
|
6
|
+
<path d="m25.27 60.29-4.88.4v-10.47h4.88z" fill="#f58534"/>
|
|
7
|
+
<path d="m28.56 38.16-5.78 1.09v21.94l5.78 1.2 5.29-12.17z" fill="#9d5025"/>
|
|
8
|
+
<path d="m33.85 61.79-5.29.6v-24.23l5.29.6z" fill="#f58534"/>
|
|
9
|
+
<path d="m39.83 43.04-7.98.8v19.24l7.98 1.6 8.47-10.87z" fill="#9d5025"/>
|
|
10
|
+
<path d="m45.21 63.78-5.38.9v-21.64l5.38.4z" fill="#f58534"/>
|
|
11
|
+
<g fill="#9d5025">
|
|
12
|
+
<path d="m50 25.59-5.38 1.7v38.29l5.38 1.09 5.38-20.54z"/>
|
|
13
|
+
<path d="m54.98 63.78 5.19.9 7.98-14.46-7.98-14.36-5.19.8z"/>
|
|
14
|
+
<path d="m68.15 50.22-7.98-14.36-5.19.8"/>
|
|
15
|
+
<path d="m66.05 61.79 5.39.6 5.78-18.25-5.78-18.15-5.39 1.3z"/>
|
|
16
|
+
<path d="m74.73 60.29 4.88.4 4.39-18.05-4.39-18.64-4.88 1.2z"/>
|
|
17
|
+
</g>
|
|
18
|
+
<path d="m50 25.59 5.38 1.7v38.29l-5.38 1.09z" fill="#f58534"/>
|
|
19
|
+
<path d="m68.15 37.46-7.98-1.6v28.82l7.98-1.6z" fill="#f58534"/>
|
|
20
|
+
<path d="m77.22 28.29-5.78-2.3v36.4l5.78-1.2z" fill="#f58534"/>
|
|
21
|
+
<path d="m84 26.19-4.39-2.19v36.69l4.39-.8z" fill="#f58534"/>
|
|
22
|
+
</svg>
|
|
@@ -22,7 +22,9 @@ require "stripe"
|
|
|
22
22
|
require "net/sftp"
|
|
23
23
|
require "csv"
|
|
24
24
|
require "securerandom"
|
|
25
|
+
require "aws-sdk-athena"
|
|
25
26
|
require "zip"
|
|
27
|
+
require "zendesk_api"
|
|
26
28
|
|
|
27
29
|
# Service
|
|
28
30
|
require_relative "integrations/config"
|
|
@@ -48,6 +50,7 @@ require_relative "integrations/source/bigquery/client"
|
|
|
48
50
|
require_relative "integrations/source/postgresql/client"
|
|
49
51
|
require_relative "integrations/source/databricks/client"
|
|
50
52
|
require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
|
|
53
|
+
require_relative "integrations/source/aws_athena/client"
|
|
51
54
|
|
|
52
55
|
# Destination
|
|
53
56
|
require_relative "integrations/destination/klaviyo/client"
|
|
@@ -61,6 +64,7 @@ require_relative "integrations/destination/stripe/client"
|
|
|
61
64
|
require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
|
|
62
65
|
require_relative "integrations/destination/sftp/client"
|
|
63
66
|
require_relative "integrations/destination/postgresql/client"
|
|
67
|
+
require_relative "integrations/destination/zendesk/client"
|
|
64
68
|
require_relative "integrations/destination/http/client"
|
|
65
69
|
|
|
66
70
|
module Multiwoven
|
|
@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
|
|
|
35
35
|
|
|
36
36
|
spec.add_runtime_dependency "activesupport"
|
|
37
37
|
spec.add_runtime_dependency "async-websocket"
|
|
38
|
+
spec.add_runtime_dependency "aws-sdk-athena"
|
|
38
39
|
spec.add_runtime_dependency "csv"
|
|
39
40
|
spec.add_runtime_dependency "dry-schema"
|
|
40
41
|
spec.add_runtime_dependency "dry-struct"
|
|
@@ -53,6 +54,7 @@ Gem::Specification.new do |spec|
|
|
|
53
54
|
spec.add_runtime_dependency "sequel"
|
|
54
55
|
spec.add_runtime_dependency "slack-ruby-client"
|
|
55
56
|
spec.add_runtime_dependency "stripe"
|
|
57
|
+
spec.add_runtime_dependency "zendesk_api"
|
|
56
58
|
|
|
57
59
|
spec.add_development_dependency "byebug"
|
|
58
60
|
spec.add_development_dependency "rspec"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.67
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-05-
|
|
11
|
+
date: 2024-05-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -38,6 +38,20 @@ dependencies:
|
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: aws-sdk-athena
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
41
55
|
- !ruby/object:Gem::Dependency
|
|
42
56
|
name: csv
|
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -290,6 +304,20 @@ dependencies:
|
|
|
290
304
|
- - ">="
|
|
291
305
|
- !ruby/object:Gem::Version
|
|
292
306
|
version: '0'
|
|
307
|
+
- !ruby/object:Gem::Dependency
|
|
308
|
+
name: zendesk_api
|
|
309
|
+
requirement: !ruby/object:Gem::Requirement
|
|
310
|
+
requirements:
|
|
311
|
+
- - ">="
|
|
312
|
+
- !ruby/object:Gem::Version
|
|
313
|
+
version: '0'
|
|
314
|
+
type: :runtime
|
|
315
|
+
prerelease: false
|
|
316
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
317
|
+
requirements:
|
|
318
|
+
- - ">="
|
|
319
|
+
- !ruby/object:Gem::Version
|
|
320
|
+
version: '0'
|
|
293
321
|
- !ruby/object:Gem::Dependency
|
|
294
322
|
name: byebug
|
|
295
323
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -463,10 +491,19 @@ files:
|
|
|
463
491
|
- lib/multiwoven/integrations/destination/stripe/config/meta.json
|
|
464
492
|
- lib/multiwoven/integrations/destination/stripe/config/spec.json
|
|
465
493
|
- lib/multiwoven/integrations/destination/stripe/icon.svg
|
|
494
|
+
- lib/multiwoven/integrations/destination/zendesk/client.rb
|
|
495
|
+
- lib/multiwoven/integrations/destination/zendesk/config/catalog.json
|
|
496
|
+
- lib/multiwoven/integrations/destination/zendesk/config/meta.json
|
|
497
|
+
- lib/multiwoven/integrations/destination/zendesk/config/spec.json
|
|
498
|
+
- lib/multiwoven/integrations/destination/zendesk/icon.svg
|
|
466
499
|
- lib/multiwoven/integrations/protocol/protocol.json
|
|
467
500
|
- lib/multiwoven/integrations/protocol/protocol.rb
|
|
468
501
|
- lib/multiwoven/integrations/rollout.rb
|
|
469
502
|
- lib/multiwoven/integrations/service.rb
|
|
503
|
+
- lib/multiwoven/integrations/source/aws_athena/client.rb
|
|
504
|
+
- lib/multiwoven/integrations/source/aws_athena/config/meta.json
|
|
505
|
+
- lib/multiwoven/integrations/source/aws_athena/config/spec.json
|
|
506
|
+
- lib/multiwoven/integrations/source/aws_athena/icon.svg
|
|
470
507
|
- lib/multiwoven/integrations/source/bigquery/client.rb
|
|
471
508
|
- lib/multiwoven/integrations/source/bigquery/config/meta.json
|
|
472
509
|
- lib/multiwoven/integrations/source/bigquery/config/spec.json
|