multiwoven-integrations 0.7.8 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5064957ae11ed9958ef813d12fb45ff06bd36f77bcb0b06409664d8564c780cb
-  data.tar.gz: 577dc47d95729aa81e3f98994945fb1715f9da72b4437bdaa523ce8914cde4ac
+  metadata.gz: 0d6c095f758588db92c0232cc771651405956eca4cb6c6ddcd76bf4076347e94
+  data.tar.gz: e42faebd684460ed4aeebdc09520c929a68b1757410598ccc0679e0cb1474849
 SHA512:
-  metadata.gz: 04ae67e7a763b3e951efea585b4ef96b7226219acb0bd0fdfa2afa057838a847d4617463036b866e95e830862eb440bf43e18210bb62dcc5b4340952802b8234
-  data.tar.gz: f36f8cbc8b4b9e52c140b8da44e5ac76c5eed33c621268cd5407d0d7c4869902ae10fe4c0047020d1496d8643bcb95452b9376cbff76f79ab2db0bf6da8a2e9a
+  metadata.gz: 1d29ac63ac873aa8467ffa10e280353dec803ede7cc670d91233bf70e8e2fdd56b55894c1ca8beca6783b130e75cc68721a4a0e41ea4af75a8bdcaa2b008159c
+  data.tar.gz: 9bf9ae850bb8e2b75466e20afb9fa8ca769368af72ec7f6ffe76e8bf7469d744497694ffcf9a2464ae275b2cb1a7fd484d0b08a4017280f03b577a2ec795bb2a

data/lib/multiwoven/integrations/core/base_connector.rb

@@ -63,7 +63,8 @@ module Multiwoven
       end
 
       def failure_status(error)
-        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
+        message = error&.message || "failed"
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: message).to_multiwoven_message
       end
     end
   end

data/lib/multiwoven/integrations/core/constants.rb

@@ -36,6 +36,17 @@ module Multiwoven
       AIRTABLE_BASES_ENDPOINT = "https://api.airtable.com/v0/meta/bases"
       AIRTABLE_GET_BASE_SCHEMA_ENDPOINT = "https://api.airtable.com/v0/meta/bases/{baseId}/tables"
 
+      MS_EXCEL_AUTH_ENDPOINT = "https://graph.microsoft.com/v1.0/me"
+      MS_EXCEL_TABLE_ROW_WRITE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets/%<sheet_name>s/tables/%<table_name>s/rows"
+      MS_EXCEL_TABLE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/workbook/"\
+      "worksheets/sheet/tables?$select=name"
+      MS_EXCEL_FILES_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/root/children"
+      MS_EXCEL_WORKSHEETS_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets"
+      MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets/%<sheet_name>s/range(address='A1:Z1')/usedRange?$select=values"
+
       AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
       AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
 
@@ -44,6 +55,7 @@ module Multiwoven
       HTTP_POST = "POST"
       HTTP_PUT = "PUT"
       HTTP_DELETE = "DELETE"
+      HTTP_PATCH = "PATCH"
 
       # google sheets
       GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive"

data/lib/multiwoven/integrations/core/destination_connector.rb

@@ -15,6 +15,14 @@ module Multiwoven
           success: success, failed: failure, logs: log_message_array
         ).to_multiwoven_message
       end
+
+      def auth_headers(access_token)
+        {
+          "Accept" => "application/json",
+          "Authorization" => "Bearer #{access_token}",
+          "Content-Type" => "application/json"
+        }
+      end
     end
   end
 end

data/lib/multiwoven/integrations/core/http_client.rb

@@ -19,13 +19,14 @@ module Multiwoven
                           when Constants::HTTP_GET then Net::HTTP::Get
                           when Constants::HTTP_POST then Net::HTTP::Post
                           when Constants::HTTP_PUT then Net::HTTP::Put
+                          when Constants::HTTP_PATCH then Net::HTTP::Patch
                           when Constants::HTTP_DELETE then Net::HTTP::Delete
                           else raise ArgumentError, "Unsupported HTTP method: #{method}"
                           end
 
           request = request_class.new(uri)
           headers.each { |key, value| request[key] = value }
-          request.body = payload.to_json if payload && %w[POST PUT].include?(method.upcase)
+          request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase)
           request
         end
       end

data/lib/multiwoven/integrations/destination/airtable/client.rb

@@ -109,14 +109,6 @@ module Multiwoven
             }
           end
 
-          def auth_headers(access_token)
-            {
-              "Accept" => "application/json",
-              "Authorization" => "Bearer #{access_token}",
-              "Content-Type" => "application/json"
-            }
-          end
-
           def base_id_exists?(bases, base_id)
             return if extract_bases(bases).any? { |base| base["id"] == base_id }
 

data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb

@@ -110,14 +110,6 @@ module Multiwoven::Integrations::Destination
         [schema, data]
       end
 
-      def auth_headers(access_token)
-        {
-          "Accept" => "application/json",
-          "Authorization" => "Bearer #{access_token}",
-          "Content-Type" => "application/json"
-        }
-      end
-
       def ad_account_exists?(response, ad_account_id)
         return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" }
 

data/lib/multiwoven/integrations/destination/klaviyo/client.rb

@@ -38,6 +38,7 @@ module Multiwoven::Integrations::Destination
 
         request_method = sync_config.stream.request_method
 
+        log_message_array = []
         write_success = 0
         write_failure = 0
         records.each do |record|
@@ -45,6 +46,7 @@ module Multiwoven::Integrations::Destination
           # Add hardcode values into payload
           record["data"] ||= {}
           record["data"]["type"] = sync_config.stream.name
+          args = [request_method, url, record]
 
           response = Multiwoven::Integrations::Core::HttpClient.request(
             url,
@@ -57,6 +59,7 @@ module Multiwoven::Integrations::Destination
           else
             write_failure += 1
           end
+          log_message_array << log_request_response("info", args, response)
         rescue StandardError => e
           handle_exception(e, {
                              context: "KLAVIYO:RECORD:WRITE:FAILURE",
@@ -65,12 +68,9 @@ module Multiwoven::Integrations::Destination
                              sync_run_id: sync_config.sync_run_id
                            })
           write_failure += 1
+          log_message_array << log_request_response("error", args, e.message)
         end
-        tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
-          success: write_success,
-          failed: write_failure
-        )
-        tracker.to_multiwoven_message
+        tracking_message(write_success, write_failure, log_message_array)
       rescue StandardError => e
         # TODO: Handle rate limiting seperately
         handle_exception(e, {

data/lib/multiwoven/integrations/destination/microsoft_excel/client.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Destination
+  module MicrosoftExcel
+    include Multiwoven::Integrations::Core
+    class Client < DestinationConnector
+      prepend Multiwoven::Integrations::Core::RateLimiter
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        drive_id = create_connection(connection_config)
+        if drive_id
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:EXCEL:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        connection_config = connection_config.with_indifferent_access
+        token = connection_config[:token]
+        drive_id = create_connection(connection_config)
+        records = get_file(token, drive_id)
+        records.each do |record|
+          file_id = record[:id]
+          record[:worksheets] = get_file_data(token, drive_id, file_id)
+        end
+        catalog = Catalog.new(streams: create_streams(records, catalog_json))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:EXCEL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def write(sync_config, records, _action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        token = connection_config[:token]
+        file_name = sync_config.stream.name.split(", ").first
+        sheet_name = sync_config.stream.name.split(", ").last
+        drive_id = create_connection(connection_config)
+        excel_files = get_file(token, drive_id)
+        worksheet = excel_files.find { |file| file[:name] == file_name }
+        item_id = worksheet[:id]
+
+        table = get_table(token, drive_id, item_id)
+        write_url = format(MS_EXCEL_TABLE_ROW_WRITE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name,
+                                                         table_name: table["name"])
+        payload = { values: records.map(&:values) }
+        process_write_request(write_url, payload, token, sync_config)
+      end
+
+      private
+
+      def create_connection(connection_config)
+        token = connection_config[:token]
+        response = Multiwoven::Integrations::Core::HttpClient.request(
+          MS_EXCEL_AUTH_ENDPOINT,
+          HTTP_GET,
+          headers: auth_headers(token)
+        )
+        JSON.parse(response.body)["id"]
+      end
+
+      def get_table(token, drive_id, item_id)
+        table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id)
+        response = Multiwoven::Integrations::Core::HttpClient.request(
+          table_url,
+          HTTP_GET,
+          headers: auth_headers(token)
+        )
+        JSON.parse(response.body)["value"].first
+      end
+
+      def get_file(token, drive_id)
+        url = format(MS_EXCEL_FILES_API, drive_id: drive_id)
+        response = Multiwoven::Integrations::Core::HttpClient.request(
+          url,
+          HTTP_GET,
+          headers: auth_headers(token)
+        )
+        files = JSON.parse(response.body)["value"]
+        excel_files = files.select { |file| file["name"].match(/\.(xlsx|xls|xlsm)$/) }
+        excel_files.map { |file| { name: file["name"], id: file["id"] } }
+      end
+
+      def get_all_sheets(token, drive_id, item_id)
+        base_url = format(MS_EXCEL_WORKSHEETS_API, drive_id: drive_id, item_id: item_id)
+        worksheet_response = Multiwoven::Integrations::Core::HttpClient.request(
+          base_url,
+          HTTP_GET,
+          headers: auth_headers(token)
+        )
+        JSON.parse(worksheet_response.body)["value"]
+      end
+
+      def get_file_data(token, drive_id, item_id)
+        result = []
+        worksheets_data = get_all_sheets(token, drive_id, item_id)
+        worksheets_data.each do |sheet|
+          sheet_name = sheet["name"]
+          sheet_url = format(MS_EXCEL_SHEET_RANGE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name)
+
+          sheet_response = Multiwoven::Integrations::Core::HttpClient.request(
+            sheet_url,
+            HTTP_GET,
+            headers: auth_headers(token)
+          )
+          sheets_data = JSON.parse(sheet_response.body)
+          result << {
+            sheet_name: sheet_name,
+            column_names: sheets_data["values"].first
+          }
+        end
+        result
+      end
+
+      def create_streams(records, catalog_json)
+        group_by_table(records).flat_map do |_, record|
+          record.map do |_, r|
+            Multiwoven::Integrations::Protocol::Stream.new(
+              name: r[:workbook],
+              action: StreamAction["fetch"],
+              json_schema: convert_to_json_schema(r[:columns]),
+              request_rate_limit: catalog_json["request_rate_limit"] || 60,
+              request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
+              request_rate_concurrency: catalog_json["request_rate_concurrency"] || 1
+            )
+          end
+        end
+      end
+
+      def group_by_table(records)
+        result = {}
+
+        records.each_with_index do |entries, entries_index|
+          entries[:worksheets].each_with_index do |sheet, entry_index|
+            workbook_sheet = "#{entries[:name]}, #{sheet[:sheet_name]}"
+            columns = sheet[:column_names].map do |column_name|
+              column_name = "empty column" if column_name.empty?
+              {
+                column_name: column_name,
+                data_type: "String",
+                is_nullable: true
+              }
+            end
+            result[entries_index] ||= {}
+            result[entries_index][entry_index] = { workbook: workbook_sheet, columns: columns }
+          end
+        end
+        result
+      end
+
+      def process_write_request(write_url, payload, token, sync_config)
+        write_success = 0
+        write_failure = 0
+        log_message_array = []
+
+        begin
+          response = Multiwoven::Integrations::Core::HttpClient.request(
+            write_url,
+            HTTP_POST,
+            payload: payload,
+            headers: auth_headers(token)
+          )
+          if success?(response)
+            write_success += 1
+          else
+            write_failure += 1
+          end
+          log_message_array << log_request_response("info", [HTTP_POST, write_url, payload], response)
+        rescue StandardError => e
+          handle_exception(e, {
+                             context: "MICROSOFT:EXCEL:RECORD:WRITE:EXCEPTION",
+                             type: "error",
+                             sync_id: sync_config.sync_id,
+                             sync_run_id: sync_config.sync_run_id
+                           })
+          write_failure += 1
+          log_message_array << log_request_response("error", [HTTP_POST, write_url, payload], e.message)
+        end
+
+        tracking_message(write_success, write_failure, log_message_array)
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json

@@ -0,0 +1,7 @@
+{
+  "request_rate_limit": 6000,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}
+

data/lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "MicrosoftExcel",
+    "title": "Microsoft Excel",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/microsoft_excel",
+    "github_issue_label": "destination-microsoft-excel",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json

@@ -0,0 +1,19 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/microsoft_excel",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Microsoft Excel",
+    "type": "object",
+    "required": ["token"],
+    "properties": {
+      "token": {
+        "description": "Token from Microsoft Graph.",
+        "type": "string",
+        "title": "Token",
+        "order": 0
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/destination/microsoft_excel/icon.svg

@@ -0,0 +1,18 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="800" width="1200" viewBox="-343.4625 -532.5 2976.675 3195">
+<path d="M1437.75 1011.75L532.5 852v1180.393c0 53.907 43.7 97.607 97.607 97.607h1562.036c53.907 0 97.607-43.7 97.607-97.607V1597.5z" fill="#185C37"/>
+<path d="M1437.75 0H630.107C576.2 0 532.5 43.7 532.5 97.607V532.5l905.25 532.5L1917 1224.75 2289.75 1065V532.5z" fill="#21A366"/>
+<path d="M532.5 532.5h905.25V1065H532.5z" fill="#107C41"/>
+<path d="M1180.393 426H532.5v1331.25h647.893c53.834-.175 97.432-43.773 97.607-97.607V523.607c-.175-53.834-43.773-97.432-97.607-97.607z" opacity=".1"/>
+<path d="M1127.143 479.25H532.5V1810.5h594.643c53.834-.175 97.432-43.773 97.607-97.607V576.857c-.175-53.834-43.773-97.432-97.607-97.607z" opacity=".2"/>
+<path d="M1127.143 479.25H532.5V1704h594.643c53.834-.175 97.432-43.773 97.607-97.607V576.857c-.175-53.834-43.773-97.432-97.607-97.607z" opacity=".2"/>
+<path d="M1073.893 479.25H532.5V1704h541.393c53.834-.175 97.432-43.773 97.607-97.607V576.857c-.175-53.834-43.773-97.432-97.607-97.607z" opacity=".2"/>
+<linearGradient gradientTransform="matrix(1 0 0 -1 0 2132)" y2="404.982" x2="967.987" y1="1729.018" x1="203.513" gradientUnits="userSpaceOnUse" id="a">
+<stop offset="0" stop-color="#18884f"/>
+<stop offset=".5" stop-color="#117e43"/>
+<stop offset="1" stop-color="#0b6631"/>
+</linearGradient>
+<path d="M97.607 479.25h976.285c53.907 0 97.607 43.7 97.607 97.607v976.285c0 53.907-43.7 97.607-97.607 97.607H97.607C43.7 1650.75 0 1607.05 0 1553.143V576.857c0-53.907 43.7-97.607 97.607-97.607z" fill="url(#a)"/>
+<path d="M302.3 1382.264l205.332-318.169L319.5 747.683h151.336l102.666 202.35c9.479 19.223 15.975 33.494 19.49 42.919h1.331a798.667 798.667 0 0121.3-44.677L725.371 747.79H864.3l-192.925 314.548L869.2 1382.263H721.378L602.79 1160.158a186.298 186.298 0 01-14.164-29.66h-1.757a140.458 140.458 0 01-13.739 28.755l-122.102 223.011z" fill="#FFF"/>
+<path d="M2192.143 0H1437.75v532.5h852V97.607C2289.75 43.7 2246.05 0 2192.143 0z" fill="#33C481"/>
+<path d="M1437.75 1065h852v532.5h-852z" fill="#107C41"/>
+</svg>

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.7.8"
+    VERSION = "0.8.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -36,6 +36,7 @@ module Multiwoven
       MariaDB
       DatabricksLakehouse
       Oracle
+      MicrosoftExcel
     ].freeze
   end
 end

data/lib/multiwoven/integrations.rb

@@ -81,6 +81,7 @@ require_relative "integrations/destination/iterable/client"
 require_relative "integrations/destination/maria_db/client"
 require_relative "integrations/destination/databricks_lakehouse/client"
 require_relative "integrations/destination/oracle_db/client"
+require_relative "integrations/destination/microsoft_excel/client"
 
 module Multiwoven
   module Integrations

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.8.0
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-09 00:00:00.000000000 Z
+date: 2024-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -544,6 +544,11 @@ files:
 - lib/multiwoven/integrations/destination/maria_db/config/meta.json
 - lib/multiwoven/integrations/destination/maria_db/config/spec.json
 - lib/multiwoven/integrations/destination/maria_db/icon.svg
+- lib/multiwoven/integrations/destination/microsoft_excel/client.rb
+- lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json
+- lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json
+- lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json
+- lib/multiwoven/integrations/destination/microsoft_excel/icon.svg
 - lib/multiwoven/integrations/destination/oracle_db/client.rb
 - lib/multiwoven/integrations/destination/oracle_db/config/meta.json
 - lib/multiwoven/integrations/destination/oracle_db/config/spec.json