multiwoven-integrations 0.38.0 → 0.39.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6517ac7cba3e8d4ef88e44ea54ba106168a1145ddd028fbb5d12ee15a6d24420
-  data.tar.gz: aeca6c1672dc44259ec5ef8b71e22e1c28c817d8e4ebf37ac5801fbfefc568ad
+  metadata.gz: 8bea6c5689d8543401961ecd7840b04b2e2db88d3401d9d8c59ba09646149da7
+  data.tar.gz: e2a9a283a5718016f63ee7a0de2725d017fd5dc4543cd8e05bf1b07f4877a239
 SHA512:
-  metadata.gz: b4c29de3371f2c52c9377da307c68cb18dbe21006602dcba6a39d172d452956276213f7efc56bf9990edf6cb6b5b81eaf1534a95ec8cb854f2cbdc9e8be6d77d
-  data.tar.gz: 3881622dda89bdcdee30bc421344201e8ea8d1c0bd4164911464beeabc5c522b03155edcc19bea8a5e8444f6ec145ef28f8dcd3c1b2d30a631a55925ac229c6b
+  metadata.gz: b9df4fb01b5a0834fc5e1475d910da9327e842b104dd0acfb84fb6c33ad394ce90ae57d0fe634d29ef4fab3229313347ff681b46157e0a6313eb1d4677cdf669
+  data.tar.gz: 718c8dde52b98bb30bb90d809309edc620a5ff8ff78b7a88f8de7bd219076baf5bf02d97e55b21674602a5349a834243c2796b50dd6b863e5412091981c12380

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.38.0"
+    VERSION = "0.39.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -36,6 +36,7 @@ module Multiwoven
       Http
       Aisquared
       OneDrive
+      MicrosoftDynamics
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/microsoft_dynamics/client.rb

@@ -0,0 +1,327 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module MicrosoftDynamics
+    include Multiwoven::Integrations::Core
+
+    API_VERSION = "9.2"
+    EXPIRED_ACCESS_TOKEN_ERROR_CODE = "InvalidAuthenticationToken"
+    # Dynamics Web API max page size; $skip is not supported, so paging uses @odata.nextLink.
+    PAGE_SIZE = 5000
+    DYNAMICS_OBJECTS = %w[accounts contacts opportunities leads].freeze
+    ENTITY_ORDER_BY = {
+      "accounts" => "accountid",
+      "contacts" => "contactid",
+      "opportunities" => "opportunityid",
+      "leads" => "leadid"
+    }.freeze
+
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        response = dynamics_request(whoami_url)
+        response_body = JSON.parse(response.body)
+
+        if success?(response) && response_body.key?("UserId")
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+
+        streams = DYNAMICS_OBJECTS.filter_map do |entity|
+          create_stream_for_entity(entity)
+        rescue StandardError => e
+          handle_exception(e, {
+                             context: "MICROSOFT:DYNAMICS:DISCOVER:LOOP_EXCEPTION",
+                             type: "error"
+                           })
+          nil
+        end
+
+        Catalog.new(streams: streams).to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        @connector_instance = sync_config&.source&.connector_instance
+        create_connection(connection_config)
+
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        query(nil, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MICROSOFT:DYNAMICS:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        load_connection_config(connection_config)
+      end
+
+      def load_connection_config(connection_config)
+        @tenant_id = connection_config[:tenant_id]
+        @client_id = connection_config[:application_id]
+        @instance_url = connection_config[:instance_url]
+        @client_secret = connection_config[:client_secret]
+        stored_token = @connector_instance&.configuration&.dig("access_token")
+        @access_token = stored_token.presence || refresh_access_token
+      end
+
+      def refresh_access_token
+        @access_token = fetch_access_token
+        persist_access_token(@access_token)
+        @access_token
+      end
+
+      def persist_access_token(token)
+        return unless @connector_instance&.configuration
+
+        config = @connector_instance.configuration
+        config = {} unless config.is_a?(Hash)
+        @connector_instance.update!(configuration: config.merge("access_token" => token))
+      end
+
+      def fetch_access_token
+        response = Multiwoven::Integrations::Core::HttpClient.request(
+          format(MICROSOFT_GRAPH_TOKEN_URL, tenant_id: @tenant_id),
+          HTTP_POST,
+          payload: form_urlencoded_payload(
+            client_id: @client_id,
+            client_secret: @client_secret,
+            scope: "https://#{@instance_url}.crm.dynamics.com/.default",
+            grant_type: "client_credentials"
+          ),
+          headers: {
+            "Content-Type" => "application/x-www-form-urlencoded"
+          }
+        )
+        raise dynamics_api_error(response.body) unless success?(response)
+
+        JSON.parse(response.body)["access_token"]
+      end
+
+      def query(_connection, sql_query)
+        entity, select_fields, limit, offset = parse_sql_query(sql_query)
+        records = fetch_entity_records(entity, select_fields: select_fields, limit: limit, offset: offset)
+        records.map do |row|
+          RecordMessage.new(data: sanitize_record(row), emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+
+      def create_stream_for_entity(entity)
+        records = fetch_entity_records(entity, limit: 1)
+        raise StandardError, "No records found for #{entity}" if records.empty?
+
+        columns = records.first.keys.reject { |key| odata_annotation?(key) }.map do |key|
+          { column_name: key, type: "string" }
+        end
+
+        Multiwoven::Integrations::Protocol::Stream.new(
+          name: entity,
+          action: StreamAction["fetch"],
+          json_schema: convert_to_json_schema(columns),
+          supported_sync_modes: %w[incremental]
+        )
+      end
+
+      def fetch_entity_records(entity, select_fields: nil, limit: nil, offset: nil)
+        offset = offset.to_i
+        limit = limit&.to_i
+        records = []
+        skipped = 0
+        url = entity_url(
+          entity,
+          select_fields: select_fields,
+          top: page_size_for(limit: limit, offset: offset)
+        )
+
+        while url.present?
+          page_records, next_url = fetch_entity_page(url)
+          break if page_records.empty?
+
+          skipped = collect_page_records(
+            records,
+            page_records,
+            offset: offset,
+            limit: limit,
+            skipped: skipped
+          )
+          break if limit_reached?(records, limit)
+
+          url = next_url
+        end
+
+        records
+      end
+
+      def fetch_entity_page(url)
+        response = dynamics_request(url)
+        raise dynamics_api_error(response.body) unless success?(response)
+
+        body = JSON.parse(response.body)
+        [body["value"] || [], body["@odata.nextLink"]]
+      end
+
+      def collect_page_records(records, page_records, offset:, limit:, skipped:)
+        page_records.each do |record|
+          if skipped < offset
+            skipped += 1
+            next
+          end
+
+          records << record
+          break if limit_reached?(records, limit)
+        end
+        skipped
+      end
+
+      def limit_reached?(records, limit)
+        limit.present? && limit.positive? && records.size >= limit
+      end
+
+      def parse_sql_query(sql_query)
+        query = sql_query.to_s.strip.chomp(";")
+        entity = query[/FROM\s+([^\s;]+)/i, 1]
+        raise ArgumentError, "Could not extract entity name from query" if entity.blank?
+
+        select_clause = query[/SELECT\s+(.+?)\s+FROM/i, 1]
+        select_fields = if select_clause.nil? || select_clause.strip == "*"
+                          nil
+                        else
+                          select_clause.split(",").map(&:strip)
+                        end
+
+        limit = query[/LIMIT\s+(\d+)/i, 1]&.to_i
+        offset = query[/OFFSET\s+(\d+)/i, 1]&.to_i
+
+        [entity, select_fields, limit, offset]
+      end
+
+      # CRM rejects OData $skip ("Skip Clause is not supported").
+      # Batching uses $top + @odata.nextLink, applying OFFSET in-process.
+      def entity_url(entity, select_fields: nil, top: nil)
+        entity = entity.to_s
+        raise ArgumentError, "Invalid entity name: #{entity}" unless entity.match?(/\A\w+\z/)
+
+        base = format(MS_DYNAMICS_REST_API, instance_url: @instance_url, api_version: API_VERSION, entity: entity)
+        query_parts = []
+        query_parts << "$select=#{URI.encode_www_form_component(select_fields.join(","))}" if select_fields.present?
+        order_by = ENTITY_ORDER_BY[entity]
+        query_parts << "$orderby=#{URI.encode_www_form_component(order_by)}" if order_by.present?
+        query_parts << "$top=#{top.to_i}" if top.present? && top.to_i.positive?
+        query_parts.empty? ? base : "#{base}?#{query_parts.join("&")}"
+      end
+
+      def page_size_for(limit:, offset:)
+        offset = offset.to_i
+        limit = limit&.to_i
+        return [PAGE_SIZE, offset + limit].min if offset.positive? && limit.present? && limit.positive?
+        return limit if limit.present? && limit.positive?
+
+        PAGE_SIZE
+      end
+
+      def whoami_url
+        format(MS_DYNAMICS_WHOAMI_API, instance_url: @instance_url, api_version: API_VERSION)
+      end
+
+      def dynamics_request(url)
+        response = dynamics_http_get(url)
+        return response unless expired_access_token_error?(response)
+
+        refresh_access_token
+        dynamics_http_get(url)
+      end
+
+      def dynamics_http_get(url)
+        Multiwoven::Integrations::Core::HttpClient.request(
+          url,
+          HTTP_GET,
+          headers: {
+            "Accept" => "application/json",
+            "Authorization" => "Bearer #{@access_token}",
+            "Content-Type" => "application/json"
+          }
+        )
+      end
+
+      def sanitize_record(record)
+        record.each_with_object({}) do |(key, value), result|
+          next if odata_annotation?(key)
+          next if value.is_a?(Hash) || value.is_a?(Array)
+
+          result[key] = value
+        end
+      end
+
+      def odata_annotation?(key)
+        key.to_s.start_with?("@") || key.to_s.include?("@odata")
+      end
+
+      def dynamics_api_error(response_body)
+        parsed = JSON.parse(response_body)
+        error = parsed["error"]
+
+        message = if error.is_a?(Hash)
+                    "#{error["code"]}: #{error["message"]}"
+                  elsif error.is_a?(String)
+                    description = parsed["error_description"]
+                    description.present? ? "#{error}: #{description}" : error
+                  else
+                    response_body
+                  end
+
+        StandardError.new(message)
+      rescue JSON::ParserError, TypeError
+        StandardError.new(response_body.to_s)
+      end
+
+      def expired_access_token_error?(response)
+        return true if response.code.to_s == "401"
+
+        error = JSON.parse(response.body)["error"]
+        return false unless error.is_a?(Hash)
+
+        error["code"] == EXPIRED_ACCESS_TOKEN_ERROR_CODE
+      rescue JSON::ParserError, TypeError
+        false
+      end
+
+      # HttpClient.request always calls payload.to_json.
+      # Microsoft OAuth token endpoints require
+      # application/x-www-form-urlencoded bodies instead of JSON.
+      # This wrapper overrides to_json so HttpClient sends a
+      # form-encoded string rather than a JSON document.
+      def form_urlencoded_payload(fields)
+        payload = Object.new
+        payload.define_singleton_method(:to_json) do |*_args|
+          URI.encode_www_form(fields)
+        end
+        payload
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/microsoft_dynamics/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "MicrosoftDynamics",
+    "title": "Microsoft Dynamics",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/microsoft_dynamics",
+    "github_issue_label": "source-microsoft-dynamics",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/microsoft_dynamics/config/spec.json

@@ -0,0 +1,35 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/microsoft_dynamics",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Microsoft Dynamics",
+    "type": "object",
+    "required": ["instance_url", "tenant_id", "application_id", "client_secret"],
+    "properties": {
+      "instance_url": {
+        "type": "string",
+        "title": "Organization Name",
+        "order": 0
+      },
+      "tenant_id": {
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Tenant ID",
+        "order": 1
+      },
+      "application_id": {
+        "type": "string",
+        "title": "Application ID",
+        "order": 2
+      },
+      "client_secret": {
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Client Secret",
+        "order": 3
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/microsoft_dynamics/icon.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg fill="#000000" width="800px" height="800px" viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg"><title>Dynamics 365 icon</title><path d="M4.59 7.41l4.94 3.54L4.59 24zm0-7.41v6.36l9.53 5.29 4.59-3.52zm0 24l14.82-8.47v-6.7Z"/></svg>

data/lib/multiwoven/integrations/source/one_drive/client.rb

@@ -82,6 +82,7 @@ module Multiwoven::Integrations::Source
         @data_type = connection_config[:data_type]
         @file_name = connection_config[:file_name]
         @share_url = connection_config[:share_url]
+        @is_recursive = [true, "true"].include?(connection_config[:is_recursive])
         stored_token = @connector_instance&.configuration&.dig("access_token")
         @access_token = stored_token.presence || refresh_access_token
       end
@@ -173,11 +174,12 @@ module Multiwoven::Integrations::Source
 
       def list_files_in_folder(_connection_config)
         files_in_folder.map do |file|
+          relative_path = relative_file_path(file)
           RecordMessage.new(
             data: {
               element_id: file["id"],
               file_name: file["name"],
-              file_path: file["name"],
+              file_path: relative_path,
               size: file["size"],
               file_type: File.extname(file["name"]).sub(".", ""),
               created_date: file["createdDateTime"],
@@ -190,10 +192,12 @@ module Multiwoven::Integrations::Source
       end
 
       def download_unstructured_file(_connection_config, file_path, sync_id)
-        file_name = resolve_download_file_name(file_path)
-        file_item = files_in_folder.find { |item| item["name"] == file_name }
+        lookup_path = resolve_download_file_name(file_path)
+        file_item = find_file_item(lookup_path)
         raise StandardError, "File not found." if file_item.nil?
 
+        file_name = file_item["name"]
+        relative_path = relative_file_path(file_item)
         local_path = download_file_to_local(
           file_name,
           sync_id,
@@ -206,7 +210,7 @@ module Multiwoven::Integrations::Source
             element_id: file_item["id"],
             local_path: local_path,
             file_name: file_name,
-            file_path: file_name,
+            file_path: relative_path,
             size: file_item["size"],
             file_type: File.extname(file_name).sub(".", ""),
             created_date: file_item["createdDateTime"],
@@ -220,19 +224,35 @@ module Multiwoven::Integrations::Source
       def files_in_folder
         records = fetch_list_items
         records["value"].select do |item|
-          item["folder"].blank? && matching_file_name?(item["name"])
+          item["folder"].blank? && matching_file_name?(item["name"], relative_file_path(item))
         end
       end
 
+      def find_file_item(lookup_path)
+        files = files_in_folder
+        exact_match = files.find do |item|
+          relative_path = relative_file_path(item)
+          relative_path == lookup_path
+        end
+        return exact_match if exact_match
+        return if lookup_path.include?("/")
+
+        files.find { |item| item["name"] == lookup_path }
+      end
+
+      def relative_file_path(file)
+        file["relative_path"].presence || file["name"]
+      end
+
       def resolve_download_file_name(file_path)
-        return File.basename(file_path) unless file_path.to_s.start_with?("http")
+        return file_path.to_s.strip unless file_path.to_s.start_with?("http")
 
         @file_name.to_s.strip.presence || File.basename(file_path)
       end
 
-      def matching_file_name?(name)
+      def matching_file_name?(name, relative_path = nil)
         configured_name = @file_name.to_s.strip
-        configured_name.blank? || configured_name == name
+        configured_name.blank? || configured_name == name || configured_name == relative_path
       end
 
       def discover_stream_for_file(conn, file)
@@ -240,7 +260,7 @@ module Multiwoven::Integrations::Source
         columns = build_discover_columns(describe_results)
 
         Multiwoven::Integrations::Protocol::Stream.new(
-          name: stream_name_for(file["name"]),
+          name: stream_name_for(file),
           action: StreamAction["fetch"],
           json_schema: convert_to_json_schema(columns)
         )
@@ -289,9 +309,10 @@ module Multiwoven::Integrations::Source
       def query(connection, query)
         local_file = nil
         file_name = extract_file_name_from_query(query)
+        local_basename = File.basename(file_name)
         local_file = download_file_to_local(file_name, @sync_id)
 
-        file = read_local_file(connection, file_name, local_file)
+        file = read_local_file(connection, local_basename, local_file)
         query = apply_local_file_to_query(query, file)
         get_results(connection, query).map do |row|
           RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
@@ -393,7 +414,8 @@ module Multiwoven::Integrations::Source
       end
 
       def single_file_item_url(file_name)
-        encoded_file = URI::DEFAULT_PARSER.escape(file_name)
+        # Preserve path separators for nested files; encode other unsafe chars.
+        encoded_file = file_name.to_s.split("/").map { |segment| URI::DEFAULT_PARSER.escape(segment) }.join("/")
 
         if @share_url.present?
           shared = shared_folder_reference
@@ -440,7 +462,35 @@ module Multiwoven::Integrations::Source
 
         return { "value" => [fetch_shared_item_metadata] } if @share_url.present? && shared_folder_reference[:is_file]
 
-        paginated_graph_collection(list_items_url)
+        collect_files_from_folder(list_items_url, recursive: @is_recursive)
+      end
+
+      # Lists files under the configured folder. When recursive is true, BFS over
+      # /children so nested folder files are included in syncs.
+      def collect_files_from_folder(root_url, recursive: false)
+        files = []
+        queue = [[root_url, ""]]
+
+        until queue.empty?
+          url, prefix = queue.shift
+          page = paginated_graph_collection(url)
+
+          page["value"].each do |item|
+            relative_path = prefix.empty? ? item["name"].to_s : "#{prefix}/#{item["name"]}"
+
+            if item["folder"].present?
+              next unless recursive
+
+              drive_id = item.dig("parentReference", "driveId") || @drive_id
+              queue << ["#{drive_item_url(drive_id, item["id"])}/children", relative_path]
+            else
+              item["relative_path"] = relative_path
+              files << item
+            end
+          end
+        end
+
+        { "value" => files }
       end
 
       def fetch_shared_item_metadata
@@ -460,7 +510,9 @@ module Multiwoven::Integrations::Source
         response = microsoft_graph_request(single_file_item_url(@file_name))
         raise graph_api_error(response.body) unless success?(response)
 
-        JSON.parse(response.body)
+        item = JSON.parse(response.body)
+        item["relative_path"] = @file_name.to_s.strip
+        item
       end
 
       def paginated_graph_collection(url)
@@ -498,14 +550,15 @@ module Multiwoven::Integrations::Source
         records["value"].select do |record|
           record["folder"].blank? &&
             SPREADSHEET_EXTENSIONS.include?(File.extname(record["name"].to_s).downcase) &&
-            matching_file_name?(record["name"])
+            matching_file_name?(record["name"], relative_file_path(record))
         end
       end
 
-      # Keep the file extension in the stream name — TableSelector generates
-      # `SELECT * FROM ${stream.name}`, and read_local_file keys off File.extname.
-      def stream_name_for(file_name)
-        File.basename(file_name)
+      # Keep the relative path (with extension) in the stream name — TableSelector
+      # generates `SELECT * FROM ${stream.name}`, and read_local_file keys off
+      # File.extname. Relative paths also disambiguate nested duplicates.
+      def stream_name_for(file)
+        relative_file_path(file)
       end
 
       def encode_sharing_url(url)

data/lib/multiwoven/integrations/source/one_drive/config/spec.json

@@ -45,6 +45,12 @@
         "title": "Share URL",
         "description": "OneDrive or SharePoint sharing link for the folder to read from, skip if using User Name to access Root Folder."
       },
+      "is_recursive": {
+        "type": "boolean",
+        "title": "Enable recursive",
+        "description": "Enables recursive folder traversal. When true, all files and subfolders are read. Default is false, reading only the specified folder.",
+        "default": false
+      },
       "file_name": {
         "type": "string",
         "title": "File Name",

data/lib/multiwoven/integrations.rb

@@ -102,6 +102,7 @@ require_relative "integrations/source/google_drive/client"
 require_relative "integrations/source/http/client"
 require_relative "integrations/source/aisquared/client"
 require_relative "integrations/source/one_drive/client"
+require_relative "integrations/source/microsoft_dynamics/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.38.0
+  version: 0.39.0
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-17 00:00:00.000000000 Z
+date: 2026-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -802,6 +802,10 @@ files:
 - lib/multiwoven/integrations/source/maria_db/config/meta.json
 - lib/multiwoven/integrations/source/maria_db/config/spec.json
 - lib/multiwoven/integrations/source/maria_db/icon.svg
+- lib/multiwoven/integrations/source/microsoft_dynamics/client.rb
+- lib/multiwoven/integrations/source/microsoft_dynamics/config/meta.json
+- lib/multiwoven/integrations/source/microsoft_dynamics/config/spec.json
+- lib/multiwoven/integrations/source/microsoft_dynamics/icon.svg
 - lib/multiwoven/integrations/source/odoo/client.rb
 - lib/multiwoven/integrations/source/odoo/config/meta.json
 - lib/multiwoven/integrations/source/odoo/config/spec.json