multiwoven-integrations 0.38.1 → 0.39.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 828f206c7576ddd3e84774cc75175213d1bef9a78df725564c67ac37d5e474d8
-  data.tar.gz: 98b0930cef5d402c99ba5272a97cb918bb301573de8202cfaafe54d825f817e4
+  metadata.gz: 6ec4e8509d2c6cf3e2e3ab4ecab5198b748956f0d8b8108b6374855b8f866531
+  data.tar.gz: 374946a9e09d51b4bc82a0f72693991472659e11a7651f39b2aa1f678a06adda
 SHA512:
-  metadata.gz: afd08217ce63822cf47858221c70cda09b9755b17ce2e08e6fe7156b49b74a9c6c3525e52c2f07b6946ef27d7eccd0fe113846dd51bfa48467f3c8be1641c341
-  data.tar.gz: f0c1d17204b01e640d49ea0cb8747baead11ecbea7c4dd4e1c7a3aa9a938e8c60ca0cbb48044fa4a3b1aaed6cca51546928be77c206c4a263e965fca77cf74fa
+  metadata.gz: 8e3a05154a4092ea28e4d66584da10e120ecc1043607337453ead51585e7cfad4ff88cf5e1516832c48ad3f12845a996d810bce2088df3a98b7a30b1cb77ea4b
+  data.tar.gz: d6de430fa20d12ecb00d2ae65898f40108ffc5c8b8762874fa2766dc9dad9b687692516651da6e9674f2322497c04c44c350a5fc465b00b19c3b0b3b4b872378

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.38.1"
+    VERSION = "0.39.1"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -36,6 +36,7 @@ module Multiwoven
       Http
       Aisquared
       OneDrive
+      MicrosoftDynamics
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/microsoft_dynamics/client.rb

@@ -0,0 +1,327 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module MicrosoftDynamics
+    include Multiwoven::Integrations::Core
+
+    API_VERSION = "9.2"
+    EXPIRED_ACCESS_TOKEN_ERROR_CODE = "InvalidAuthenticationToken"
+    # Dynamics Web API max page size; $skip is not supported, so paging uses @odata.nextLink.
+    PAGE_SIZE = 5000
+    DYNAMICS_OBJECTS = %w[accounts contacts opportunities leads].freeze
+    ENTITY_ORDER_BY = {
+      "accounts" => "accountid",
+      "contacts" => "contactid",
+      "opportunities" => "opportunityid",
+      "leads" => "leadid"
+    }.freeze
+
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        response = dynamics_request(whoami_url)
+        response_body = JSON.parse(response.body)
+
+        if success?(response) && response_body.key?("UserId")
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+
+        streams = DYNAMICS_OBJECTS.filter_map do |entity|
+          create_stream_for_entity(entity)
+        rescue StandardError => e
+          handle_exception(e, {
+                             context: "MICROSOFT:DYNAMICS:DISCOVER:LOOP_EXCEPTION",
+                             type: "error"
+                           })
+          nil
+        end
+
+        Catalog.new(streams: streams).to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        @connector_instance = sync_config&.source&.connector_instance
+        create_connection(connection_config)
+
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        query(nil, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        load_connection_config(connection_config)
+      end
+
+      def load_connection_config(connection_config)
+        @tenant_id = connection_config[:tenant_id]
+        @client_id = connection_config[:application_id]
+        @instance_url = connection_config[:instance_url]
+        @client_secret = connection_config[:client_secret]
+        stored_token = @connector_instance&.configuration&.dig("access_token")
+        @access_token = stored_token.presence || refresh_access_token
+      end
+
+      def refresh_access_token
+        @access_token = fetch_access_token
+        persist_access_token(@access_token)
+        @access_token
+      end
+
+      def persist_access_token(token)
+        return unless @connector_instance&.configuration
+
+        config = @connector_instance.configuration
+        config = {} unless config.is_a?(Hash)
+        @connector_instance.update!(configuration: config.merge("access_token" => token))
+      end
+
+      def fetch_access_token
+        response = Multiwoven::Integrations::Core::HttpClient.request(
+          format(MICROSOFT_GRAPH_TOKEN_URL, tenant_id: @tenant_id),
+          HTTP_POST,
+          payload: form_urlencoded_payload(
+            client_id: @client_id,
+            client_secret: @client_secret,
+            scope: "https://#{@instance_url}.crm.dynamics.com/.default",
+            grant_type: "client_credentials"
+          ),
+          headers: {
+            "Content-Type" => "application/x-www-form-urlencoded"
+          }
+        )
+        raise dynamics_api_error(response.body) unless success?(response)
+
+        JSON.parse(response.body)["access_token"]
+      end
+
+      def query(_connection, sql_query)
+        entity, select_fields, limit, offset = parse_sql_query(sql_query)
+        records = fetch_entity_records(entity, select_fields: select_fields, limit: limit, offset: offset)
+        records.map do |row|
+          RecordMessage.new(data: sanitize_record(row), emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+
+      def create_stream_for_entity(entity)
+        records = fetch_entity_records(entity, limit: 1)
+        raise StandardError, "No records found for #{entity}" if records.empty?
+
+        columns = records.first.keys.reject { |key| odata_annotation?(key) }.map do |key|
+          { column_name: key, type: "string" }
+        end
+
+        Multiwoven::Integrations::Protocol::Stream.new(
+          name: entity,
+          action: StreamAction["fetch"],
+          json_schema: convert_to_json_schema(columns),
+          supported_sync_modes: %w[incremental]
+        )
+      end
+
+      def fetch_entity_records(entity, select_fields: nil, limit: nil, offset: nil)
+        offset = offset.to_i
+        limit = limit&.to_i
+        records = []
+        skipped = 0
+        url = entity_url(
+          entity,
+          select_fields: select_fields,
+          top: page_size_for(limit: limit, offset: offset)
+        )
+
+        while url.present?
+          page_records, next_url = fetch_entity_page(url)
+          break if page_records.empty?
+
+          skipped = collect_page_records(
+            records,
+            page_records,
+            offset: offset,
+            limit: limit,
+            skipped: skipped
+          )
+          break if limit_reached?(records, limit)
+
+          url = next_url
+        end
+
+        records
+      end
+
+      def fetch_entity_page(url)
+        response = dynamics_request(url)
+        raise dynamics_api_error(response.body) unless success?(response)
+
+        body = JSON.parse(response.body)
+        [body["value"] || [], body["@odata.nextLink"]]
+      end
+
+      def collect_page_records(records, page_records, offset:, limit:, skipped:)
+        page_records.each do |record|
+          if skipped < offset
+            skipped += 1
+            next
+          end
+
+          records << record
+          break if limit_reached?(records, limit)
+        end
+        skipped
+      end
+
+      def limit_reached?(records, limit)
+        limit.present? && limit.positive? && records.size >= limit
+      end
+
+      def parse_sql_query(sql_query)
+        query = sql_query.to_s.strip.chomp(";")
+        entity = query[/FROM\s+([^\s;]+)/i, 1]
+        raise ArgumentError, "Could not extract entity name from query" if entity.blank?
+
+        select_clause = query[/SELECT\s+(.+?)\s+FROM/i, 1]
+        select_fields = if select_clause.nil? || select_clause.strip == "*"
+                          nil
+                        else
+                          select_clause.split(",").map(&:strip)
+                        end
+
+        limit = query[/LIMIT\s+(\d+)/i, 1]&.to_i
+        offset = query[/OFFSET\s+(\d+)/i, 1]&.to_i
+
+        [entity, select_fields, limit, offset]
+      end
+
+      # CRM rejects OData $skip ("Skip Clause is not supported").
+      # Batching uses $top + @odata.nextLink, applying OFFSET in-process.
+      def entity_url(entity, select_fields: nil, top: nil)
+        entity = entity.to_s
+        raise ArgumentError, "Invalid entity name: #{entity}" unless entity.match?(/\A\w+\z/)
+
+        base = format(MS_DYNAMICS_REST_API, instance_url: @instance_url, api_version: API_VERSION, entity: entity)
+        query_parts = []
+        query_parts << "$select=#{URI.encode_www_form_component(select_fields.join(","))}" if select_fields.present?
+        order_by = ENTITY_ORDER_BY[entity]
+        query_parts << "$orderby=#{URI.encode_www_form_component(order_by)}" if order_by.present?
+        query_parts << "$top=#{top.to_i}" if top.present? && top.to_i.positive?
+        query_parts.empty? ? base : "#{base}?#{query_parts.join("&")}"
+      end
+
+      def page_size_for(limit:, offset:)
+        offset = offset.to_i
+        limit = limit&.to_i
+        return [PAGE_SIZE, offset + limit].min if offset.positive? && limit.present? && limit.positive?
+        return limit if limit.present? && limit.positive?
+
+        PAGE_SIZE
+      end
+
+      def whoami_url
+        format(MS_DYNAMICS_WHOAMI_API, instance_url: @instance_url, api_version: API_VERSION)
+      end
+
+      def dynamics_request(url)
+        response = dynamics_http_get(url)
+        return response unless expired_access_token_error?(response)
+
+        refresh_access_token
+        dynamics_http_get(url)
+      end
+
+      def dynamics_http_get(url)
+        Multiwoven::Integrations::Core::HttpClient.request(
+          url,
+          HTTP_GET,
+          headers: {
+            "Accept" => "application/json",
+            "Authorization" => "Bearer #{@access_token}",
+            "Content-Type" => "application/json"
+          }
+        )
+      end
+
+      def sanitize_record(record)
+        record.each_with_object({}) do |(key, value), result|
+          next if odata_annotation?(key)
+          next if value.is_a?(Hash) || value.is_a?(Array)
+
+          result[key] = value
+        end
+      end
+
+      def odata_annotation?(key)
+        key.to_s.start_with?("@") || key.to_s.include?("@odata")
+      end
+
+      def dynamics_api_error(response_body)
+        parsed = JSON.parse(response_body)
+        error = parsed["error"]
+
+        message = if error.is_a?(Hash)
+                    "#{error["code"]}: #{error["message"]}"
+                  elsif error.is_a?(String)
+                    description = parsed["error_description"]
+                    description.present? ? "#{error}: #{description}" : error
+                  else
+                    response_body
+                  end
+
+        StandardError.new(message)
+      rescue JSON::ParserError, TypeError
+        StandardError.new(response_body.to_s)
+      end
+
+      def expired_access_token_error?(response)
+        return true if response.code.to_s == "401"
+
+        error = JSON.parse(response.body)["error"]
+        return false unless error.is_a?(Hash)
+
+        error["code"] == EXPIRED_ACCESS_TOKEN_ERROR_CODE
+      rescue JSON::ParserError, TypeError
+        false
+      end
+
+      # HttpClient.request always calls payload.to_json.
+      # Microsoft OAuth token endpoints require
+      # application/x-www-form-urlencoded bodies instead of JSON.
+      # This wrapper overrides to_json so HttpClient sends a
+      # form-encoded string rather than a JSON document.
+      def form_urlencoded_payload(fields)
+        payload = Object.new
+        payload.define_singleton_method(:to_json) do |*_args|
+          URI.encode_www_form(fields)
+        end
+        payload
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/microsoft_dynamics/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "MicrosoftDynamics",
+    "title": "Microsoft Dynamics",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/microsoft_dynamics",
+    "github_issue_label": "source-microsoft-dynamics",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/microsoft_dynamics/config/spec.json

@@ -0,0 +1,35 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/microsoft_dynamics",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Microsoft Dynamics",
+    "type": "object",
+    "required": ["instance_url", "tenant_id", "application_id", "client_secret"],
+    "properties": {
+      "instance_url": {
+        "type": "string",
+        "title": "Organization Name",
+        "order": 0
+      },
+      "tenant_id": {
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Tenant ID",
+        "order": 1
+      },
+      "application_id": {
+        "type": "string",
+        "title": "Application ID",
+        "order": 2
+      },
+      "client_secret": {
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Client Secret",
+        "order": 3
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/microsoft_dynamics/icon.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg fill="#000000" width="800px" height="800px" viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg"><title>Dynamics 365 icon</title><path d="M4.59 7.41l4.94 3.54L4.59 24zm0-7.41v6.36l9.53 5.29 4.59-3.52zm0 24l14.82-8.47v-6.7Z"/></svg>

data/lib/multiwoven/integrations/source/one_drive/client.rb

@@ -91,6 +91,7 @@ module Multiwoven::Integrations::Source
         raise ArgumentError, "User Name or Share URL is required" unless connection_config[:share_url].present? || connection_config[:user_name].present?
 
         load_connection_config(connection_config)
+        @sync_id ||= "preview"
 
         if @share_url.present?
           @drive_id = shared_folder_reference[:drive_id]
@@ -310,7 +311,7 @@ module Multiwoven::Integrations::Source
         local_file = nil
         file_name = extract_file_name_from_query(query)
         local_basename = File.basename(file_name)
-        local_file = download_file_to_local(file_name, @sync_id)
+        local_file = download_spreadsheet_for_query(file_name)
 
         file = read_local_file(connection, local_basename, local_file)
         query = apply_local_file_to_query(query, file)
@@ -321,6 +322,34 @@ module Multiwoven::Integrations::Source
         cleanup_ephemeral_download(local_file)
       end
 
+      # Prefer Graph item IDs (same as discover) so nested / shared-folder files
+      # resolve during model preview. Resolve id with one path metadata GET —
+      # never a full folder listing / recursive BFS. Fall back to path-based download.
+      def download_spreadsheet_for_query(file_name)
+        file_item = fetch_drive_item_by_path(file_name)
+        if file_item&.[]("id")
+          download_file_to_local(
+            file_name,
+            @sync_id,
+            item_id: file_item["id"],
+            drive_id: file_item.dig("parentReference", "driveId")
+          )
+        else
+          download_file_to_local(file_name, @sync_id)
+        end
+      end
+
+      def fetch_drive_item_by_path(file_name)
+        return fetch_shared_item_metadata if @share_url.present? && shared_folder_reference[:is_file]
+
+        response = microsoft_graph_request(single_file_item_url(file_name))
+        return unless success?(response)
+
+        JSON.parse(response.body)
+      rescue StandardError
+        nil
+      end
+
       def extract_file_name_from_query(sql_query)
         match = sql_query.match(
           /\bFROM\s+(?:[`"]([^`"]+)[`"]|'([^']+)'|([^\s;]+))/i
@@ -379,6 +408,7 @@ module Multiwoven::Integrations::Source
       end
 
       def local_download_path(file_name, sync_id)
+        sync_id = sync_id.presence || "preview"
         download_path = ENV["FILE_DOWNLOAD_PATH"]
         if download_path
           File.join(download_path, "syncs", sync_id, File.basename(file_name))

data/lib/multiwoven/integrations.rb

@@ -102,6 +102,7 @@ require_relative "integrations/source/google_drive/client"
 require_relative "integrations/source/http/client"
 require_relative "integrations/source/aisquared/client"
 require_relative "integrations/source/one_drive/client"
+require_relative "integrations/source/microsoft_dynamics/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.38.1
+  version: 0.39.1
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-30 00:00:00.000000000 Z
+date: 2026-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -802,6 +802,10 @@ files:
 - lib/multiwoven/integrations/source/maria_db/config/meta.json
 - lib/multiwoven/integrations/source/maria_db/config/spec.json
 - lib/multiwoven/integrations/source/maria_db/icon.svg
+- lib/multiwoven/integrations/source/microsoft_dynamics/client.rb
+- lib/multiwoven/integrations/source/microsoft_dynamics/config/meta.json
+- lib/multiwoven/integrations/source/microsoft_dynamics/config/spec.json
+- lib/multiwoven/integrations/source/microsoft_dynamics/icon.svg
 - lib/multiwoven/integrations/source/odoo/client.rb
 - lib/multiwoven/integrations/source/odoo/config/meta.json
 - lib/multiwoven/integrations/source/odoo/config/spec.json