multiwoven-integrations 0.1.33 → 0.1.34
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/base_connector.rb +8 -0
- data/lib/multiwoven/integrations/core/constants.rb +6 -0
- data/lib/multiwoven/integrations/destination/google_sheets/client.rb +183 -0
- data/lib/multiwoven/integrations/destination/google_sheets/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/destination/google_sheets/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json +74 -0
- data/lib/multiwoven/integrations/destination/google_sheets/icon.svg +1 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations.rb +3 -0
- metadata +7 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 32abb3bef8987a8a76828570507e207089efb5c7c59dd21093774beeef21a265
|
|
4
|
+
data.tar.gz: 9920eb296b386ef3add818a1e7b59ef26e7e8c32aee8a21e8295b82deebc4d44
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 33846b75d3a91aedf93fa367d0cd2385f7dd06066015a782cd149dd1cedefd055f29c9c007062988b13a18b9eb4efac1afd250a4b47531c1fcfd5d0b0f5619c9
|
|
7
|
+
data.tar.gz: 6faa53d0929c0d5ae12da7467e56573e86e1ff3b1898f96762150440a19fff351796198e2fe03f72a12a7008762391ac4714a3f5d5a9703602e6acaf328d27ba
|
|
@@ -57,6 +57,14 @@ module Multiwoven
|
|
|
57
57
|
file_contents = File.read(file_path)
|
|
58
58
|
JSON.parse(file_contents)
|
|
59
59
|
end
|
|
60
|
+
|
|
61
|
+
def success_status
|
|
62
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def failure_status(error)
|
|
66
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
|
|
67
|
+
end
|
|
60
68
|
end
|
|
61
69
|
end
|
|
62
70
|
end
|
|
@@ -36,6 +36,12 @@ module Multiwoven
|
|
|
36
36
|
HTTP_POST = "POST"
|
|
37
37
|
HTTP_PUT = "PUT"
|
|
38
38
|
HTTP_DELETE = "DELETE"
|
|
39
|
+
|
|
40
|
+
JSON_SCHEMA_URL = "http://json-schema.org/draft-07/schema#"
|
|
41
|
+
|
|
42
|
+
# google sheets
|
|
43
|
+
GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive"
|
|
44
|
+
GOOGLE_SPREADSHEET_ID_REGEX = %r{/d/([-\w]{20,})/}.freeze
|
|
39
45
|
end
|
|
40
46
|
end
|
|
41
47
|
end
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations
|
|
5
|
+
module Destination
|
|
6
|
+
module GoogleSheets
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
|
|
9
|
+
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
|
10
|
+
MAX_CHUNK_SIZE = 10_000
|
|
11
|
+
|
|
12
|
+
def check_connection(connection_config)
|
|
13
|
+
authorize_client(connection_config)
|
|
14
|
+
fetch_google_spread_sheets(connection_config)
|
|
15
|
+
success_status
|
|
16
|
+
rescue StandardError => e
|
|
17
|
+
handle_exception("GOOGLE_SHEETS:CRM:DISCOVER:EXCEPTION", "error", e)
|
|
18
|
+
failure_status(e)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def discover(connection_config)
|
|
22
|
+
authorize_client(connection_config)
|
|
23
|
+
spreadsheets = fetch_google_spread_sheets(connection_config)
|
|
24
|
+
catalog = build_catalog_from_spreadsheets(spreadsheets, connection_config)
|
|
25
|
+
catalog.to_multiwoven_message
|
|
26
|
+
rescue StandardError => e
|
|
27
|
+
handle_exception("GOOGLE_SHEETS:CRM:DISCOVER:EXCEPTION", "error", e)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def write(sync_config, records, action = "create")
|
|
31
|
+
setup_write_environment(sync_config, action)
|
|
32
|
+
process_record_chunks(records, sync_config)
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
handle_exception("GOOGLE_SHEETS:CRM:WRITE:EXCEPTION", "error", e)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# To define the level of access granted to your app, you need to identify and declare authorization scopes which is provided by google scopse https://developers.google.com/sheets/api/scopes
|
|
40
|
+
def authorize_client(config)
|
|
41
|
+
credentials = config[:credentials_json]
|
|
42
|
+
@client = Google::Apis::SheetsV4::SheetsService.new
|
|
43
|
+
@client.authorization = Google::Auth::ServiceAccountCredentials.make_creds(
|
|
44
|
+
json_key_io: StringIO.new(credentials.to_json),
|
|
45
|
+
scope: GOOGLE_SHEETS_SCOPE
|
|
46
|
+
)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Extract spreadsheet id from the spreadsheet link and return the metadata for all the sheets
|
|
50
|
+
def fetch_google_spread_sheets(connection_config)
|
|
51
|
+
spreadsheet_id = extract_spreadsheet_id(connection_config[:spreadsheet_link])
|
|
52
|
+
@client.get_spreadsheet(spreadsheet_id)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# dynamically builds catalog based on spreadsheet metadata
|
|
56
|
+
def build_catalog_from_spreadsheets(spreadsheet, connection_config)
|
|
57
|
+
catalog = build_catalog(load_catalog)
|
|
58
|
+
@spreadsheet_id = extract_spreadsheet_id(connection_config[:spreadsheet_link])
|
|
59
|
+
|
|
60
|
+
spreadsheet.sheets.each do |sheet|
|
|
61
|
+
process_sheet_for_catalog(sheet, catalog)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
catalog
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Builds catalog for the single spreadsheet based on column name
|
|
68
|
+
def process_sheet_for_catalog(sheet, catalog)
|
|
69
|
+
sheet_name, last_column_index = extract_sheet_properties(sheet)
|
|
70
|
+
column_names = fetch_column_names(sheet_name, last_column_index)
|
|
71
|
+
catalog.streams << generate_json_schema(column_names, sheet_name) if column_names
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def extract_sheet_properties(sheet)
|
|
75
|
+
[sheet.properties.title, sheet.properties.grid_properties.column_count]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def fetch_column_names(sheet_name, last_column_index)
|
|
79
|
+
header_range = generate_header_range(sheet_name, last_column_index)
|
|
80
|
+
spread_sheet_value(header_range)&.flatten
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def spread_sheet_value(header_range)
|
|
84
|
+
@spread_sheet_value ||= @client.get_spreadsheet_values(@spreadsheet_id, header_range).values
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def generate_header_range(sheet_name, last_column_index)
|
|
88
|
+
"#{sheet_name}!A1:#{column_index_to_letter(last_column_index)}1"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def column_index_to_letter(index)
|
|
92
|
+
("A".."ZZZ").to_a[index - 1]
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def generate_json_schema(column_names, sheet_name)
|
|
96
|
+
{
|
|
97
|
+
name: sheet_name,
|
|
98
|
+
action: "create",
|
|
99
|
+
batch_support: true,
|
|
100
|
+
batch_size: 10_000,
|
|
101
|
+
json_schema: generate_properties_schema(column_names),
|
|
102
|
+
supported_sync_modes: %w[incremental]
|
|
103
|
+
}.with_indifferent_access
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def generate_properties_schema(column_names)
|
|
107
|
+
properties = column_names.each_with_object({}) do |field, props|
|
|
108
|
+
props[field] = { "type" => "string" }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
{ "$schema" => JSON_SCHEMA_URL, "type" => "object", "properties" => properties }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def setup_write_environment(sync_config, action)
|
|
115
|
+
@action = sync_config.stream.action || action
|
|
116
|
+
@spreadsheet_id = extract_spreadsheet_id(sync_config.destination.connection_specification[:spreadsheet_link])
|
|
117
|
+
authorize_client(sync_config.destination.connection_specification)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def extract_spreadsheet_id(link)
|
|
121
|
+
link[GOOGLE_SPREADSHEET_ID_REGEX, 1] || link
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Batch has a limit of sending 2MB data. So creating a chunk of records to meet that limit
|
|
125
|
+
def process_record_chunks(records, sync_config)
|
|
126
|
+
write_success = 0
|
|
127
|
+
write_failure = 0
|
|
128
|
+
|
|
129
|
+
records.each_slice(MAX_CHUNK_SIZE) do |chunk|
|
|
130
|
+
values = prepare_chunk_values(chunk, sync_config.stream)
|
|
131
|
+
update_sheet_values(values, sync_config.stream.name)
|
|
132
|
+
write_success += values.size
|
|
133
|
+
rescue StandardError => e
|
|
134
|
+
handle_exception("GOOGLE_SHEETS:RECORD:WRITE:EXCEPTION", "error", e)
|
|
135
|
+
write_failure += chunk.size
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
tracking_message(write_success, write_failure)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# We need to format the data to adhere to google sheets API format. This converts the sync mapped data to 2D array format expected by google sheets API
|
|
142
|
+
def prepare_chunk_values(chunk, stream)
|
|
143
|
+
last_column_index = spread_sheet_value(stream.name).count
|
|
144
|
+
fields = fetch_column_names(stream.name, last_column_index)
|
|
145
|
+
|
|
146
|
+
chunk.map do |row|
|
|
147
|
+
row_values = Array.new(fields.size, nil)
|
|
148
|
+
row.each do |key, value|
|
|
149
|
+
index = fields.index(key.to_s)
|
|
150
|
+
row_values[index] = value if index
|
|
151
|
+
end
|
|
152
|
+
row_values
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def update_sheet_values(values, stream_name)
|
|
157
|
+
row_count = spread_sheet_value(stream_name).count
|
|
158
|
+
range = "#{stream_name}!A#{row_count + 1}"
|
|
159
|
+
value_range = Google::Apis::SheetsV4::ValueRange.new(range: range, values: values)
|
|
160
|
+
|
|
161
|
+
batch_update_request = Google::Apis::SheetsV4::BatchUpdateValuesRequest.new(
|
|
162
|
+
value_input_option: "RAW",
|
|
163
|
+
data: [value_range]
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
# TODO: Remove & this is added for the test to pass we need
|
|
167
|
+
@client&.batch_update_values(@spreadsheet_id, batch_update_request)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def load_catalog
|
|
171
|
+
read_json(CATALOG_SPEC_PATH)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def tracking_message(success, failure)
|
|
175
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
176
|
+
success: success, failed: failure
|
|
177
|
+
).to_multiwoven_message
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "GoogleSheets",
|
|
4
|
+
"title": "Google Sheets",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Productivity Tools",
|
|
7
|
+
"documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/google-sheets-service-account",
|
|
8
|
+
"github_issue_label": "destination-google-sheets",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/google-sheets-service-account",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Google Sheets",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["credentials_json"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"spreadsheet_link": {
|
|
11
|
+
"type": "string"
|
|
12
|
+
},
|
|
13
|
+
"credentials_json": {
|
|
14
|
+
"type": "object",
|
|
15
|
+
"description": "You can get the keys from the Google Cloud web console. First, go to the IAM page and select Service Accounts from the left menu. Next, locate your service account in the list, click on its Keys tab, and then click Add Key. Lastly, click Create new key and select JSON.",
|
|
16
|
+
"title": "",
|
|
17
|
+
"properties": {
|
|
18
|
+
"type": {
|
|
19
|
+
"type": "string",
|
|
20
|
+
"enum": ["service_account"]
|
|
21
|
+
},
|
|
22
|
+
"project_id": {
|
|
23
|
+
"type": "string"
|
|
24
|
+
},
|
|
25
|
+
"private_key_id": {
|
|
26
|
+
"type": "string"
|
|
27
|
+
},
|
|
28
|
+
"private_key": {
|
|
29
|
+
"type": "string"
|
|
30
|
+
},
|
|
31
|
+
"client_email": {
|
|
32
|
+
"type": "string",
|
|
33
|
+
"format": "email"
|
|
34
|
+
},
|
|
35
|
+
"client_id": {
|
|
36
|
+
"type": "string"
|
|
37
|
+
},
|
|
38
|
+
"auth_uri": {
|
|
39
|
+
"type": "string",
|
|
40
|
+
"format": "uri"
|
|
41
|
+
},
|
|
42
|
+
"token_uri": {
|
|
43
|
+
"type": "string",
|
|
44
|
+
"format": "uri"
|
|
45
|
+
},
|
|
46
|
+
"auth_provider_x509_cert_url": {
|
|
47
|
+
"type": "string",
|
|
48
|
+
"format": "uri"
|
|
49
|
+
},
|
|
50
|
+
"client_x509_cert_url": {
|
|
51
|
+
"type": "string",
|
|
52
|
+
"format": "uri"
|
|
53
|
+
},
|
|
54
|
+
"universe_domain": {
|
|
55
|
+
"type": "string"
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
"required": [
|
|
59
|
+
"type",
|
|
60
|
+
"project_id",
|
|
61
|
+
"private_key_id",
|
|
62
|
+
"private_key",
|
|
63
|
+
"client_email",
|
|
64
|
+
"client_id",
|
|
65
|
+
"auth_uri",
|
|
66
|
+
"token_uri",
|
|
67
|
+
"auth_provider_x509_cert_url",
|
|
68
|
+
"client_x509_cert_url",
|
|
69
|
+
"universe_domain"
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 242423 333333" shape-rendering="geometricPrecision" text-rendering="geometricPrecision" image-rendering="optimizeQuality" fill-rule="evenodd" clip-rule="evenodd"><defs><mask id="c"><linearGradient id="a" gradientUnits="userSpaceOnUse" x1="200294" y1="91174.8" x2="200294" y2="176113"><stop offset="0" stop-opacity=".02" stop-color="#fff"/><stop offset="1" stop-opacity=".2" stop-color="#fff"/></linearGradient><path fill="url(#a)" d="M158015 84111h84558v99065h-84558z"/></mask><mask id="e"><radialGradient id="b" gradientUnits="userSpaceOnUse" cx="0" cy="0" r="0" fx="0" fy="0"><stop offset="0" stop-opacity="0" stop-color="#fff"/><stop offset="1" stop-opacity=".098" stop-color="#fff"/></radialGradient><path fill="url(#b)" d="M-150-150h242723v333633H-150z"/></mask><radialGradient id="f" gradientUnits="userSpaceOnUse" cx="9696.85" cy="10000.4" r="166667" fx="9696.85" fy="10000.4"><stop offset="0" stop-color="#fff"/><stop offset="1" stop-color="#fff"/></radialGradient><linearGradient id="d" gradientUnits="userSpaceOnUse" x1="200294" y1="95125.2" x2="200294" y2="172162"><stop offset="0" stop-color="#263138"/><stop offset="1" stop-color="#263138"/></linearGradient></defs><g fill-rule="nonzero"><path d="M151513 0H22729C10227 0 1 10227 1 22728v287877c0 12505 10227 22728 22728 22728h196966c12505 0 22728-10224 22728-22728V90911l-53028-37880L151513 0z" fill="#0f9c57"/><path d="M60606 162880v109853h121216V162880H60606zm53032 94698H75757v-18938h37881v18938zm0-30301H75757v-18946h37881v18946zm0-30310H75757v-18936h37881v18936zm53030 60611h-37884v-18938h37884v18938zm0-30301h-37884v-18946h37884v18946zm0-30310h-37884v-18936h37884v18936z" fill="#f0f0f0"/><path mask="url(#c)" fill="url(#d)" d="M158165 84261l84258 84245V90911z"/><path d="M151513 0v68184c0 12557 10173 22727 22727 22727h68183L151513 0z" fill="#87cdac"/><path d="M22728 0C10226 0 0 10227 0 22729v1893C0 12123 10227 1894 22728 1894h128784V1H22728z" fill="#fff" fill-opacity=".2"/><path d="M219694 331443H22728C10226 331443 0 321213 0 308715v1890c0 12505 10227 22728 22728 22728h196966c12505 0 22728-10224 22728-22728v-1890c0 12499-10224 22728-22728 22728z" fill="#263138" fill-opacity=".2"/><path d="M174239 90911c-12554 0-22727-10170-22727-22727v1893c0 12557 10173 22727 22727 22727h68183v-1893h-68183z" fill="#263138" fill-opacity=".102"/><path d="M151513 0H22729C10227 0 1 10227 1 22729v287876c0 12505 10227 22728 22728 22728h196966c12505 0 22728-10224 22728-22728V90911L151513 0z" mask="url(#e)" fill="url(#f)"/></g></svg>
|
|
@@ -16,6 +16,8 @@ require "slack-ruby-client"
|
|
|
16
16
|
require "git"
|
|
17
17
|
require "ruby-limiter"
|
|
18
18
|
require "hubspot-api-client"
|
|
19
|
+
require "google/apis/sheets_v4"
|
|
20
|
+
require "stringio"
|
|
19
21
|
|
|
20
22
|
# Service
|
|
21
23
|
require_relative "integrations/config"
|
|
@@ -45,6 +47,7 @@ require_relative "integrations/destination/salesforce_crm/client"
|
|
|
45
47
|
require_relative "integrations/destination/facebook_custom_audience/client"
|
|
46
48
|
require_relative "integrations/destination/slack/client"
|
|
47
49
|
require_relative "integrations/destination/hubspot/client"
|
|
50
|
+
require_relative "integrations/destination/google_sheets/client"
|
|
48
51
|
|
|
49
52
|
module Multiwoven
|
|
50
53
|
module Integrations
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.34
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-03-
|
|
11
|
+
date: 2024-03-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -335,6 +335,11 @@ files:
|
|
|
335
335
|
- lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json
|
|
336
336
|
- lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json
|
|
337
337
|
- lib/multiwoven/integrations/destination/facebook_custom_audience/icon.svg
|
|
338
|
+
- lib/multiwoven/integrations/destination/google_sheets/client.rb
|
|
339
|
+
- lib/multiwoven/integrations/destination/google_sheets/config/catalog.json
|
|
340
|
+
- lib/multiwoven/integrations/destination/google_sheets/config/meta.json
|
|
341
|
+
- lib/multiwoven/integrations/destination/google_sheets/config/spec.json
|
|
342
|
+
- lib/multiwoven/integrations/destination/google_sheets/icon.svg
|
|
338
343
|
- lib/multiwoven/integrations/destination/hubspot/client.rb
|
|
339
344
|
- lib/multiwoven/integrations/destination/hubspot/config/catalog.json
|
|
340
345
|
- lib/multiwoven/integrations/destination/hubspot/config/meta.json
|