outhad-integrations 0.32.0

data/lib/outhad/integrations/destination/microsoft_dynamics/client.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Destination
+  module MicrosoftDynamics
+    include Outhad::Integrations::Core
+    API_VERSION = "9.2"
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        initialize_client(connection_config)
+        token_response = create_access_token
+        uri = URI.parse(format(MS_DYNAMICS_WHOAMI_API, instance_url: @instance_url, api_version: API_VERSION))
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+        request = Net::HTTP::Get.new(uri)
+        auth_headers(token_response["access_token"]).each { |key, value| request[key] = value }
+        response = http.request(request)
+        response_body = JSON.parse(response.body)
+
+        if success?(response) && response_body.key?("UserId")
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def write(sync_config, records, _action = "upsert")
+        @sync_config = sync_config
+        stream = @sync_config.stream
+        connection_config = @sync_config.destination.connection_specification.with_indifferent_access
+        create_connection(connection_config)
+        build_url(stream)
+        process_records(records, stream)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: @sync_config.sync_id,
+                           sync_run_id: @sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_access_token
+        uri = URI.parse("https://login.microsoftonline.com/#{@tenant_id}/oauth2/v2.0/token")
+
+        payload = {
+          client_id: @client_id,
+          client_secret: @client_secret,
+          scope: "https://#{@instance_url}.crm.dynamics.com/.default",
+          grant_type: "client_credentials"
+        }
+
+        response = Net::HTTP.post_form(uri, payload)
+        JSON.parse(response.body)
+      end
+
+      def get_access_token(cache)
+        cache_key = "dynamics_#{@instance_url}_#{@tenant_id}_#{@client_id}"
+        cached_token = cache.read(cache_key)
+        if cached_token
+          @access_token = cached_token
+        else
+          new_token = create_access_token
+          # max expiration is 3 minutes. No way to make it higher
+          cache.write(cache_key, new_token["access_token"], expires_in: 180)
+          @access_token = new_token["access_token"]
+        end
+      end
+
+      def create_connection(connection_config)
+        cache = defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : ActiveSupport::Cache::MemoryStore.new
+        initialize_client(connection_config)
+        get_access_token(cache)
+      end
+
+      def initialize_client(connection_config)
+        @tenant_id = connection_config[:tenant_id]
+        @client_id = connection_config[:application_id]
+        @instance_url = connection_config[:instance_url]
+        @client_secret = connection_config[:client_secret]
+      end
+
+      def process_records(records, stream)
+        write_success = 0
+        write_failure = 0
+        properties = stream.json_schema[:properties]
+        log_message_array = []
+
+        records.each do |record_object|
+          record = extract_data(record_object, properties)
+          response = send_data_to_dynamics(record)
+          response_code = response.code.to_i
+          if response_code >= 200 && response_code < 300
+            write_success += 1
+            log_message_array << log_request_response("info", record, response["location"])
+          else
+            write_failure += 1
+            log_message_array << log_request_response("error", record, response.body)
+          end
+        rescue StandardError => e
+          # TODO: add sync_id and sync run id to the logs
+          handle_exception(e, {
+                             context: "MICROSOFT:DYNAMICS:WRITE:EXCEPTION",
+                             type: "error",
+                             sync_id: @sync_config.sync_id,
+                             sync_run_id: @sync_config.sync_run_id
+                           })
+          write_failure += 1
+          log_message_array << log_request_response("error", record, e.message)
+        end
+        tracking_message(write_success, write_failure, log_message_array)
+      end
+
+      def build_url(stream)
+        @destination_url = format(MS_DYNAMICS_REST_API, instance_url: @instance_url, api_version: API_VERSION, entity: stream.name)
+      end
+
+      def send_data_to_dynamics(payload)
+        uri = URI.parse(@destination_url)
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+
+        request = Net::HTTP::Post.new(uri)
+        auth_headers(@access_token).each { |key, value| request[key] = value }
+        request.body = payload.to_json
+        http.request(request)
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/microsoft_dynamics/config/catalog.json

@@ -0,0 +1,161 @@
+{
+  "request_rate_limit": 40000,
+  "request_rate_limit_unit": "day",
+  "request_rate_concurrency": 10,
+  "streams": [
+    {
+      "name": "accounts",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "required": ["name"],
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "account_number": {
+            "type": "string"
+          },
+          "telephone1": {
+            "type": "string"
+          },
+          "emailaddress1": {
+            "type": "string",
+            "format": "email"
+          },
+          "websiteurl": {
+            "type": "string",
+            "format": "uri"
+          },
+          "address1_line1": {
+            "type": "string"
+          },
+          "address1_city": {
+            "type": "string"
+          },
+          "address1_stateorprovince": {
+            "type": "string"
+          },
+          "address1_postalcode": {
+            "type": "string"
+          },
+          "industrycode": {
+            "type": "integer"
+          },
+          "revenue": {
+            "type": "number",
+            "format": "float"
+          },
+          "numberofemployees": {
+            "type": "integer"
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"],
+      "source_defined_cursor": false,
+      "source_defined_primary_key": ["account_number"]
+    },
+    {
+      "name": "contacts",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "required": ["emailaddress1", "firstname", "lastname"],
+        "properties": {
+          "firstname": {
+            "type": "string"
+          },
+          "lastname": {
+            "type": "string"
+          },
+          "emailaddress1": {
+            "type": "string",
+            "format": "email"
+          },
+          "mobilephone": {
+            "type": "string"
+          },
+          "telephone1": {
+            "type": "string"
+          },
+          "address1_line1": {
+            "type": "string"
+          },
+          "address1_city": {
+            "type": "string"
+          },
+          "address1_stateorprovince": {
+            "type": "string"
+          },
+          "address1_postalcode": {
+            "type": "string"
+          },
+          "jobtitle": {
+            "type": "string"
+          },
+          "parentcustomerid": {
+            "type": "string",
+            "description": "ID of the associated Account"
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"],
+      "source_defined_cursor": false,
+      "source_defined_primary_key": ["emailaddress1"]
+    },
+    {
+      "name": "opportunities",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "required": ["name", "customerid", "estimatedvalue"],
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "customerid": {
+            "type": "string",
+            "description": "ID of the associated Account or Contact"
+          },
+          "estimatedvalue": {
+            "type": "number",
+            "format": "float"
+          },
+          "description": {
+            "type": "string"
+          },
+          "closeprobability": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 100
+          },
+          "estimatedclosedate": {
+            "type": "string",
+            "format": "date"
+          },
+          "actualclosedate": {
+            "type": "string",
+            "format": "date"
+          },
+          "opportunityratingcode": {
+            "type": "string",
+            "enum": ["Hot", "Warm", "Cold"]
+          },
+          "stageid": {
+            "type": "string"
+          },
+          "ownerid": {
+            "type": "string",
+            "description": "ID of the owner (user or team)"
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"],
+      "source_defined_cursor": false,
+      "source_defined_primary_key": ["customerid"]
+    }
+  ]
+}

data/lib/outhad/integrations/destination/microsoft_dynamics/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "MicrosoftDynamics",
+    "title": "Microsoft Dynamics",
+    "connector_type": "destination",
+    "category": "CRM",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/crm/microsoft_dynamics",
+    "github_issue_label": "destination-microsoft-dynamics",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/destination/microsoft_dynamics/config/spec.json

@@ -0,0 +1,35 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/crm/microsoft_dynamics",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Microsoft Dynamics",
+    "type": "object",
+    "required": ["instance_url", "tenant_id", "application_id", "client_secret"],
+    "properties": {
+      "instance_url": {
+        "type": "string",
+        "title": "Organization Name",
+        "order": 0
+      },
+      "tenant_id": {
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Tenant ID",
+        "order": 1
+      },
+      "application_id": {
+        "type": "string",
+        "title": "Application ID",
+        "order": 2
+      },
+      "client_secret": {
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Client Secret",
+        "order": 3
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/microsoft_dynamics/icon.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg fill="#000000" width="800px" height="800px" viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg"><title>Dynamics 365 icon</title><path d="M4.59 7.41l4.94 3.54L4.59 24zm0-7.41v6.36l9.53 5.29 4.59-3.52zm0 24l14.82-8.47v-6.7Z"/></svg>

data/lib/outhad/integrations/destination/microsoft_excel/client.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Destination
+  module MicrosoftExcel
+    include Outhad::Integrations::Core
+    class Client < DestinationConnector
+      prepend Outhad::Integrations::Core::RateLimiter
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        drive_id = create_connection(connection_config)
+        if drive_id
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:EXCEL:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        connection_config = connection_config.with_indifferent_access
+        token = connection_config[:token]
+        drive_id = create_connection(connection_config)
+        records = get_file(token, drive_id)
+        records.each do |record|
+          file_id = record[:id]
+          record[:worksheets] = get_file_data(token, drive_id, file_id)
+        end
+        catalog = Catalog.new(streams: create_streams(records, catalog_json))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:EXCEL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def write(sync_config, records, _action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        token = connection_config[:token]
+        file_name = sync_config.stream.name.split(", ").first
+        sheet_name = sync_config.stream.name.split(", ").last
+        drive_id = create_connection(connection_config)
+        excel_files = get_file(token, drive_id)
+        worksheet = excel_files.find { |file| file[:name] == file_name }
+        item_id = worksheet[:id]
+        table = get_table(token, drive_id, item_id, sheet_name)
+        write_url = format(MS_EXCEL_TABLE_ROW_WRITE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name,
+                                                         table_name: table["name"])
+        payload = { values: records.map(&:values) }
+        process_write_request(write_url, payload, token, sync_config)
+      end
+
+      private
+
+      def create_connection(connection_config)
+        token = connection_config[:token]
+        response = Outhad::Integrations::Core::HttpClient.request(
+          MS_EXCEL_AUTH_ENDPOINT,
+          HTTP_GET,
+          headers: auth_headers(token)
+        )
+        JSON.parse(response.body)["id"]
+      end
+
+      def get_table(token, drive_id, item_id, sheet_name)
+        table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name)
+        response = Outhad::Integrations::Core::HttpClient.request(
+          table_url,
+          HTTP_GET,
+          headers: auth_headers(token)
+        )
+        JSON.parse(response.body)["value"].first
+      end
+
+      def get_file(token, drive_id)
+        url = format(MS_EXCEL_FILES_API, drive_id: drive_id)
+        response = Outhad::Integrations::Core::HttpClient.request(
+          url,
+          HTTP_GET,
+          headers: auth_headers(token)
+        )
+        files = JSON.parse(response.body)["value"]
+        excel_files = files.select { |file| file["name"].match(/\.(xlsx|xls|xlsm)$/) }
+        excel_files.map { |file| { name: file["name"], id: file["id"] } }
+      end
+
+      def get_all_sheets(token, drive_id, item_id)
+        base_url = format(MS_EXCEL_WORKSHEETS_API, drive_id: drive_id, item_id: item_id)
+        worksheet_response = Outhad::Integrations::Core::HttpClient.request(
+          base_url,
+          HTTP_GET,
+          headers: auth_headers(token)
+        )
+        JSON.parse(worksheet_response.body)["value"]
+      end
+
+      def get_file_data(token, drive_id, item_id)
+        result = []
+        worksheets_data = get_all_sheets(token, drive_id, item_id)
+        worksheets_data.each do |sheet|
+          sheet_name = sheet["name"]
+          sheet_url = format(MS_EXCEL_SHEET_RANGE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name)
+
+          sheet_response = Outhad::Integrations::Core::HttpClient.request(
+            sheet_url,
+            HTTP_GET,
+            headers: auth_headers(token)
+          )
+          sheets_data = JSON.parse(sheet_response.body)
+          column_names = if sheets_data.key?("error")
+                           ["Column A"]
+                         else
+                           sheets_data["values"].first
+                         end
+          result << {
+            sheet_name: sheet_name,
+            column_names: column_names
+          }
+        end
+        result
+      end
+
+      def create_streams(records, catalog_json)
+        group_by_table(records).flat_map do |_, record|
+          record.map do |_, r|
+            Outhad::Integrations::Protocol::Stream.new(
+              name: r[:workbook],
+              action: StreamAction["fetch"],
+              json_schema: convert_to_json_schema(r[:columns]),
+              request_rate_limit: catalog_json["request_rate_limit"] || 60,
+              request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
+              request_rate_concurrency: catalog_json["request_rate_concurrency"] || 1
+            )
+          end
+        end
+      end
+
+      def group_by_table(records)
+        result = {}
+
+        records.each_with_index do |entries, entries_index|
+          entries[:worksheets].each_with_index do |sheet, entry_index|
+            workbook_sheet = "#{entries[:name]}, #{sheet[:sheet_name]}"
+            columns = sheet[:column_names].map do |column_name|
+              column_name = "empty column" if column_name.empty?
+              {
+                column_name: column_name,
+                data_type: "String",
+                is_nullable: true
+              }
+            end
+            result[entries_index] ||= {}
+            result[entries_index][entry_index] = { workbook: workbook_sheet, columns: columns }
+          end
+        end
+        result
+      end
+
+      def process_write_request(write_url, payload, token, sync_config)
+        write_success = 0
+        write_failure = 0
+        log_message_array = []
+
+        begin
+          response = Outhad::Integrations::Core::HttpClient.request(
+            write_url,
+            HTTP_POST,
+            payload: payload,
+            headers: auth_headers(token)
+          )
+          if success?(response)
+            write_success += 1
+          else
+            write_failure += 1
+          end
+          log_message_array << log_request_response("info", [HTTP_POST, write_url, payload], response)
+        rescue StandardError => e
+          handle_exception(e, {
+                             context: "MICROSOFT:EXCEL:RECORD:WRITE:EXCEPTION",
+                             type: "error",
+                             sync_id: sync_config.sync_id,
+                             sync_run_id: sync_config.sync_run_id
+                           })
+          write_failure += 1
+          log_message_array << log_request_response("error", [HTTP_POST, write_url, payload], e.message)
+        end
+
+        tracking_message(write_success, write_failure, log_message_array)
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/microsoft_excel/config/catalog.json

@@ -0,0 +1,7 @@
+{
+  "request_rate_limit": 6000,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 1,
+  "streams": []
+}
+

data/lib/outhad/integrations/destination/microsoft_excel/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "MicrosoftExcel",
+    "title": "Microsoft Excel",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/productivity-tools/microsoft-excel",
+    "github_issue_label": "destination-microsoft-excel",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/destination/microsoft_excel/config/spec.json

@@ -0,0 +1,19 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/productivity-tools/microsoft-excel",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Microsoft Excel",
+    "type": "object",
+    "required": ["token"],
+    "properties": {
+      "token": {
+        "description": "Token from Microsoft Graph.",
+        "type": "string",
+        "title": "Token",
+        "order": 0
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/microsoft_excel/icon.svg

@@ -0,0 +1,18 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="800" width="1200" viewBox="-343.4625 -532.5 2976.675 3195">
+<path d="M1437.75 1011.75L532.5 852v1180.393c0 53.907 43.7 97.607 97.607 97.607h1562.036c53.907 0 97.607-43.7 97.607-97.607V1597.5z" fill="#185C37"/>
+<path d="M1437.75 0H630.107C576.2 0 532.5 43.7 532.5 97.607V532.5l905.25 532.5L1917 1224.75 2289.75 1065V532.5z" fill="#21A366"/>
+<path d="M532.5 532.5h905.25V1065H532.5z" fill="#107C41"/>
+<path d="M1180.393 426H532.5v1331.25h647.893c53.834-.175 97.432-43.773 97.607-97.607V523.607c-.175-53.834-43.773-97.432-97.607-97.607z" opacity=".1"/>
+<path d="M1127.143 479.25H532.5V1810.5h594.643c53.834-.175 97.432-43.773 97.607-97.607V576.857c-.175-53.834-43.773-97.432-97.607-97.607z" opacity=".2"/>
+<path d="M1127.143 479.25H532.5V1704h594.643c53.834-.175 97.432-43.773 97.607-97.607V576.857c-.175-53.834-43.773-97.432-97.607-97.607z" opacity=".2"/>
+<path d="M1073.893 479.25H532.5V1704h541.393c53.834-.175 97.432-43.773 97.607-97.607V576.857c-.175-53.834-43.773-97.432-97.607-97.607z" opacity=".2"/>
+<linearGradient gradientTransform="matrix(1 0 0 -1 0 2132)" y2="404.982" x2="967.987" y1="1729.018" x1="203.513" gradientUnits="userSpaceOnUse" id="a">
+<stop offset="0" stop-color="#18884f"/>
+<stop offset=".5" stop-color="#117e43"/>
+<stop offset="1" stop-color="#0b6631"/>
+</linearGradient>
+<path d="M97.607 479.25h976.285c53.907 0 97.607 43.7 97.607 97.607v976.285c0 53.907-43.7 97.607-97.607 97.607H97.607C43.7 1650.75 0 1607.05 0 1553.143V576.857c0-53.907 43.7-97.607 97.607-97.607z" fill="url(#a)"/>
+<path d="M302.3 1382.264l205.332-318.169L319.5 747.683h151.336l102.666 202.35c9.479 19.223 15.975 33.494 19.49 42.919h1.331a798.667 798.667 0 0121.3-44.677L725.371 747.79H864.3l-192.925 314.548L869.2 1382.263H721.378L602.79 1160.158a186.298 186.298 0 01-14.164-29.66h-1.757a140.458 140.458 0 01-13.739 28.755l-122.102 223.011z" fill="#FFF"/>
+<path d="M2192.143 0H1437.75v532.5h852V97.607C2289.75 43.7 2246.05 0 2192.143 0z" fill="#33C481"/>
+<path d="M1437.75 1065h852v532.5h-852z" fill="#107C41"/>
+</svg>