etl-integrations 0.1.81

data/lib/multiwoven/integrations/destination/facebook_custom_audience/icon.svg

@@ -0,0 +1,23 @@
+<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1365.3333 1365.3333" height="1365.3333" width="1365.3333" xml:space="preserve" id="svg2" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <metadata id="metadata8">
+  <rdf:RDF>
+   <cc:Work rdf:about="">
+    <dc:format>
+     image/svg+xml
+    </dc:format>
+    <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage">
+    </dc:type>
+   </cc:Work>
+  </rdf:RDF>
+ </metadata>
+ <defs id="defs6">
+ </defs>
+ <g transform="matrix(1.3333333,0,0,-1.3333333,0,1365.3333)" id="g10">
+  <g transform="scale(0.1)" id="g12">
+   <path id="path14" style="fill:#1877f2;fill-opacity:1;fill-rule:nonzero;stroke:none" d="m 10240,5120 c 0,2827.7 -2292.3,5120 -5120,5120 C 2292.3,10240 0,7947.7 0,5120 0,2564.46 1872.31,446.301 4320,62.1992 V 3640 H 3020 v 1480 h 1300 v 1128 c 0,1283.2 764.38,1992 1933.9,1992 560.17,0 1146.1,-100 1146.1,-100 V 6880 H 6754.38 C 6118.35,6880 5920,6485.33 5920,6080.43 V 5120 H 7340 L 7113,3640 H 5920 V 62.1992 C 8367.69,446.301 10240,2564.46 10240,5120">
+   </path>
+   <path id="path16" style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none" d="m 7113,3640 227,1480 H 5920 v 960.43 c 0,404.9 198.35,799.57 834.38,799.57 H 7400 v 1260 c 0,0 -585.93,100 -1146.1,100 C 5084.38,8240 4320,7531.2 4320,6248 V 5120 H 3020 V 3640 H 4320 V 62.1992 C 4580.67,21.3008 4847.84,0 5120,0 c 272.16,0 539.33,21.3008 800,62.1992 V 3640 h 1193">
+   </path>
+  </g>
+ </g>
+</svg>

data/lib/multiwoven/integrations/destination/google_sheets/client.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module GoogleSheets
+        include Multiwoven::Integrations::Core
+
+        class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+          prepend Multiwoven::Integrations::Core::Fullrefresher
+          prepend Multiwoven::Integrations::Core::RateLimiter
+          MAX_CHUNK_SIZE = 10_000
+
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            authorize_client(connection_config)
+            fetch_google_spread_sheets(connection_config)
+            success_status
+          rescue StandardError => e
+            handle_exception("GOOGLE_SHEETS:CRM:DISCOVER:EXCEPTION", "error", e)
+            failure_status(e)
+          end
+
+          def discover(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            authorize_client(connection_config)
+            spreadsheets = fetch_google_spread_sheets(connection_config)
+            catalog = build_catalog_from_spreadsheets(spreadsheets, connection_config)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception("GOOGLE_SHEETS:CRM:DISCOVER:EXCEPTION", "error", e)
+          end
+
+          def write(sync_config, records, action = "create")
+            setup_write_environment(sync_config, action)
+            process_record_chunks(records, sync_config)
+          rescue StandardError => e
+            handle_exception("GOOGLE_SHEETS:CRM:WRITE:EXCEPTION", "error", e)
+          end
+
+          def clear_all_records(sync_config)
+            setup_write_environment(sync_config, "clear")
+            connection_specification = sync_config.destination.connection_specification.with_indifferent_access
+            spreadsheet = fetch_google_spread_sheets(connection_specification)
+            sheet_ids = spreadsheet.sheets.map(&:properties).map(&:sheet_id)
+
+            delete_extra_sheets(sheet_ids)
+
+            unless sheet_ids.empty?
+              clear_response = clear_sheet_data(spreadsheet.sheets.first.properties.title)
+              return control_message("Successfully cleared data.", "succeeded") if clear_response&.cleared_range
+            end
+
+            control_message("Failed to clear data.", "failed")
+          rescue StandardError => e
+            control_message(e.message, "failed")
+          end
+
+          private
+
+          # To define the level of access granted to your app, you need to identify and declare authorization scopes which is provided by google scopse https://developers.google.com/sheets/api/scopes
+          def authorize_client(config)
+            credentials = config[:credentials_json]
+            @client = Google::Apis::SheetsV4::SheetsService.new
+            @client.authorization = Google::Auth::ServiceAccountCredentials.make_creds(
+              json_key_io: StringIO.new(credentials.to_json),
+              scope: GOOGLE_SHEETS_SCOPE
+            )
+          end
+
+          # Extract spreadsheet id from the spreadsheet link and return the metadata for all the sheets
+          def fetch_google_spread_sheets(connection_config)
+            spreadsheet_id = extract_spreadsheet_id(connection_config[:spreadsheet_link])
+            @client.get_spreadsheet(spreadsheet_id)
+          end
+
+          # dynamically builds catalog based on spreadsheet metadata
+          def build_catalog_from_spreadsheets(spreadsheet, connection_config)
+            catalog = build_catalog(load_catalog)
+            @spreadsheet_id = extract_spreadsheet_id(connection_config[:spreadsheet_link])
+
+            spreadsheet.sheets.each do |sheet|
+              process_sheet_for_catalog(sheet, catalog)
+            end
+
+            catalog
+          end
+
+          # Builds catalog for the single spreadsheet based on column name
+          def process_sheet_for_catalog(sheet, catalog)
+            sheet_name, last_column_index = extract_sheet_properties(sheet)
+            column_names = fetch_column_names(sheet_name, last_column_index)
+            catalog.streams << generate_json_schema(column_names, sheet_name) if column_names
+          end
+
+          def extract_sheet_properties(sheet)
+            [sheet.properties.title, sheet.properties.grid_properties.column_count]
+          end
+
+          def fetch_column_names(sheet_name, last_column_index)
+            header_range = generate_header_range(sheet_name, last_column_index)
+            spread_sheet_value(header_range)&.flatten
+          end
+
+          def spread_sheet_value(header_range)
+            @spread_sheet_value ||= @client.get_spreadsheet_values(@spreadsheet_id, header_range).values
+          end
+
+          def generate_header_range(sheet_name, last_column_index)
+            "#{sheet_name}!A1:#{column_index_to_letter(last_column_index)}1"
+          end
+
+          def column_index_to_letter(index)
+            ("A".."ZZZ").to_a[index - 1]
+          end
+
+          def generate_json_schema(column_names, sheet_name)
+            {
+              name: sheet_name,
+              action: "create",
+              batch_support: true,
+              batch_size: 10_000,
+              json_schema: generate_properties_schema(column_names),
+              supported_sync_modes: %w[incremental full_refresh]
+            }.with_indifferent_access
+          end
+
+          def generate_properties_schema(column_names)
+            properties = column_names.each_with_object({}) do |field, props|
+              props[field] = { "type" => "string" }
+            end
+
+            { "$schema" => JSON_SCHEMA_URL, "type" => "object", "properties" => properties }
+          end
+
+          def setup_write_environment(sync_config, action)
+            @action = sync_config.stream.action || action
+            connection_specification = sync_config.destination.connection_specification.with_indifferent_access
+            @spreadsheet_id = extract_spreadsheet_id(connection_specification[:spreadsheet_link])
+            authorize_client(connection_specification)
+          end
+
+          def extract_spreadsheet_id(link)
+            link[GOOGLE_SPREADSHEET_ID_REGEX, 1] || link
+          end
+
+          # Batch has a limit of sending 2MB data. So creating a chunk of records to meet that limit
+          def process_record_chunks(records, sync_config)
+            write_success = 0
+            write_failure = 0
+
+            records.each_slice(MAX_CHUNK_SIZE) do |chunk|
+              values = prepare_chunk_values(chunk, sync_config.stream)
+              update_sheet_values(values, sync_config.stream.name)
+              write_success += values.size
+            rescue StandardError => e
+              handle_exception("GOOGLE_SHEETS:RECORD:WRITE:EXCEPTION", "error", e)
+              write_failure += chunk.size
+            end
+
+            tracking_message(write_success, write_failure)
+          end
+
+          # We need to format the data to adhere to google sheets API format. This converts the sync mapped data to 2D array format expected by google sheets API
+          def prepare_chunk_values(chunk, stream)
+            last_column_index = spread_sheet_value(stream.name).count
+            fields = fetch_column_names(stream.name, last_column_index)
+
+            chunk.map do |row|
+              row_values = Array.new(fields.size, nil)
+              row.each do |key, value|
+                index = fields.index(key.to_s)
+                row_values[index] = value if index
+              end
+              row_values
+            end
+          end
+
+          def update_sheet_values(values, stream_name)
+            row_count = spread_sheet_value(stream_name).count
+            range = "#{stream_name}!A#{row_count + 1}"
+            value_range = Google::Apis::SheetsV4::ValueRange.new(range: range, values: values)
+
+            batch_update_request = Google::Apis::SheetsV4::BatchUpdateValuesRequest.new(
+              value_input_option: "RAW",
+              data: [value_range]
+            )
+
+            # TODO: Remove & this is added for the test to pass we need
+            @client&.batch_update_values(@spreadsheet_id, batch_update_request)
+          end
+
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
+          end
+
+          def tracking_message(success, failure)
+            Multiwoven::Integrations::Protocol::TrackingMessage.new(
+              success: success, failed: failure
+            ).to_multiwoven_message
+          end
+
+          def delete_extra_sheets(sheet_ids)
+            # Leave one sheet intact as a spreadsheet must have at least one sheet.
+            # Delete all other sheets.
+            (sheet_ids.length - 1).times do |i|
+              request = Google::Apis::SheetsV4::BatchUpdateSpreadsheetRequest.new(
+                requests: [{ delete_sheet: { sheet_id: sheet_ids[i + 1] } }]
+              )
+              @client.batch_update_spreadsheet(@spreadsheet_id, request)
+            end
+          end
+
+          def clear_sheet_data(sheet_title)
+            clear_request = Google::Apis::SheetsV4::ClearValuesRequest.new
+            @client&.clear_values(@spreadsheet_id, "#{sheet_title}!A2:Z", clear_request)
+          end
+
+          def control_message(message, status)
+            ControlMessage.new(
+              type: "full_refresh",
+              emitted_at: Time.now.to_i,
+              status: ConnectionStatusType[status],
+              meta: { detail: message }
+            ).to_multiwoven_message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/google_sheets/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 300,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/multiwoven/integrations/destination/google_sheets/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "GoogleSheets",
+    "title": "Google Sheets",
+    "connector_type": "destination",
+    "category": "Productivity Tools",
+    "documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/google-sheets-service-account",
+    "github_issue_label": "destination-google-sheets",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json

@@ -0,0 +1,74 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/google-sheets-service-account",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Google Sheets",
+    "type": "object",
+    "required": ["credentials_json"],
+    "properties": {
+      "spreadsheet_link": {
+        "type": "string"
+      },
+      "credentials_json": {
+        "type": "object",
+        "description": "You can get the keys from the Google Cloud web console. First, go to the IAM page and select Service Accounts from the left menu. Next, locate your service account in the list, click on its Keys tab, and then click Add Key. Lastly, click Create new key and select JSON.",
+        "title": "",
+        "properties": {
+          "type": {
+            "type": "string",
+            "enum": ["service_account"]
+          },
+          "project_id": {
+            "type": "string"
+          },
+          "private_key_id": {
+            "type": "string"
+          },
+          "private_key": {
+            "type": "string"
+          },
+          "client_email": {
+            "type": "string",
+            "format": "email"
+          },
+          "client_id": {
+            "type": "string"
+          },
+          "auth_uri": {
+            "type": "string",
+            "format": "uri"
+          },
+          "token_uri": {
+            "type": "string",
+            "format": "uri"
+          },
+          "auth_provider_x509_cert_url": {
+            "type": "string",
+            "format": "uri"
+          },
+          "client_x509_cert_url": {
+            "type": "string",
+            "format": "uri"
+          },
+          "universe_domain": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "type",
+          "project_id",
+          "private_key_id",
+          "private_key",
+          "client_email",
+          "client_id",
+          "auth_uri",
+          "token_uri",
+          "auth_provider_x509_cert_url",
+          "client_x509_cert_url",
+          "universe_domain"
+        ]
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/destination/google_sheets/icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 242423 333333" shape-rendering="geometricPrecision" text-rendering="geometricPrecision" image-rendering="optimizeQuality" fill-rule="evenodd" clip-rule="evenodd"><defs><mask id="c"><linearGradient id="a" gradientUnits="userSpaceOnUse" x1="200294" y1="91174.8" x2="200294" y2="176113"><stop offset="0" stop-opacity=".02" stop-color="#fff"/><stop offset="1" stop-opacity=".2" stop-color="#fff"/></linearGradient><path fill="url(#a)" d="M158015 84111h84558v99065h-84558z"/></mask><mask id="e"><radialGradient id="b" gradientUnits="userSpaceOnUse" cx="0" cy="0" r="0" fx="0" fy="0"><stop offset="0" stop-opacity="0" stop-color="#fff"/><stop offset="1" stop-opacity=".098" stop-color="#fff"/></radialGradient><path fill="url(#b)" d="M-150-150h242723v333633H-150z"/></mask><radialGradient id="f" gradientUnits="userSpaceOnUse" cx="9696.85" cy="10000.4" r="166667" fx="9696.85" fy="10000.4"><stop offset="0" stop-color="#fff"/><stop offset="1" stop-color="#fff"/></radialGradient><linearGradient id="d" gradientUnits="userSpaceOnUse" x1="200294" y1="95125.2" x2="200294" y2="172162"><stop offset="0" stop-color="#263138"/><stop offset="1" stop-color="#263138"/></linearGradient></defs><g fill-rule="nonzero"><path d="M151513 0H22729C10227 0 1 10227 1 22728v287877c0 12505 10227 22728 22728 22728h196966c12505 0 22728-10224 22728-22728V90911l-53028-37880L151513 0z" fill="#0f9c57"/><path d="M60606 162880v109853h121216V162880H60606zm53032 94698H75757v-18938h37881v18938zm0-30301H75757v-18946h37881v18946zm0-30310H75757v-18936h37881v18936zm53030 60611h-37884v-18938h37884v18938zm0-30301h-37884v-18946h37884v18946zm0-30310h-37884v-18936h37884v18936z" fill="#f0f0f0"/><path mask="url(#c)" fill="url(#d)" d="M158165 84261l84258 84245V90911z"/><path d="M151513 0v68184c0 12557 10173 22727 22727 22727h68183L151513 0z" fill="#87cdac"/><path d="M22728 0C10226 0 0 10227 0 22729v1893C0 12123 10227 1894 22728 1894h128784V1H22728z" fill="#fff" fill-opacity=".2"/><path d="M219694 331443H22728C10226 331443 0 321213 0 308715v1890c0 12505 10227 22728 22728 22728h196966c12505 0 22728-10224 22728-22728v-1890c0 12499-10224 22728-22728 22728z" fill="#263138" fill-opacity=".2"/><path d="M174239 90911c-12554 0-22727-10170-22727-22727v1893c0 12557 10173 22727 22727 22727h68183v-1893h-68183z" fill="#263138" fill-opacity=".102"/><path d="M151513 0H22729C10227 0 1 10227 1 22729v287876c0 12505 10227 22728 22728 22728h196966c12505 0 22728-10224 22728-22728V90911L151513 0z" mask="url(#e)" fill="url(#f)"/></g></svg>

data/lib/multiwoven/integrations/destination/hubspot/client.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module Hubspot
+        include Multiwoven::Integrations::Core
+
+        class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            initialize_client(connection_config)
+            authenticate_client
+            success_status
+          rescue StandardError => e
+            handle_exception("HUBSPOT:CRM:DISCOVER:EXCEPTION", "error", e)
+            failure_status(e)
+          end
+
+          def discover(_connection_config = nil)
+            catalog = build_catalog(load_catalog)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception("HUBSPOT:CRM:DISCOVER:EXCEPTION", "error", e)
+          end
+
+          def write(sync_config, records, action = "create")
+            @action = sync_config.stream.action || action
+            initialize_client(sync_config.destination.connection_specification)
+            process_records(records, sync_config.stream)
+          rescue StandardError => e
+            handle_exception("HUBSPOT:CRM:WRITE:EXCEPTION", "error", e)
+          end
+
+          private
+
+          def initialize_client(config)
+            config = config.with_indifferent_access
+            @client = ::Hubspot::Client.new(access_token: config[:access_token])
+          end
+
+          def process_records(records, stream)
+            write_success = 0
+            write_failure = 0
+            properties = stream.json_schema.with_indifferent_access[:properties]
+            records.each do |record_object|
+              record = extract_data(record_object, properties)
+              send_data_to_hubspot(stream.name, record)
+              write_success += 1
+            rescue StandardError => e
+              handle_exception("HUBSPOT:CRM:WRITE:EXCEPTION", "error", e)
+              write_failure += 1
+            end
+            tracking_message(write_success, write_failure)
+          end
+
+          def send_data_to_hubspot(stream_name, record = {})
+            args = build_args(@action, stream_name, record)
+            hubspot_stream = @client.crm.send(stream_name)
+            hubspot_data = { simple_public_object_input_for_create: args }
+            hubspot_stream.basic_api.send(@action, hubspot_data)
+          end
+
+          def build_args(action, stream_name, record)
+            case action
+            when :upsert
+              [stream_name, record[:external_key], record]
+            when :destroy
+              [stream_name, record[:id]]
+            else
+              record
+            end
+          end
+
+          def authenticate_client
+            @client.crm.contacts.basic_api.get_page
+          end
+
+          def success_status
+            ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+          end
+
+          def failure_status(error)
+            ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
+          end
+
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
+          end
+
+          def tracking_message(success, failure)
+            Multiwoven::Integrations::Protocol::TrackingMessage.new(
+              success: success, failed: failure
+            ).to_multiwoven_message
+          end
+
+          def log_debug(message)
+            Multiwoven::Integrations::Service.logger.debug(message)
+          end
+        end
+      end
+    end
+  end
+end