multiwoven-integrations 0.38.0 → 0.38.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6517ac7cba3e8d4ef88e44ea54ba106168a1145ddd028fbb5d12ee15a6d24420
-  data.tar.gz: aeca6c1672dc44259ec5ef8b71e22e1c28c817d8e4ebf37ac5801fbfefc568ad
+  metadata.gz: 828f206c7576ddd3e84774cc75175213d1bef9a78df725564c67ac37d5e474d8
+  data.tar.gz: 98b0930cef5d402c99ba5272a97cb918bb301573de8202cfaafe54d825f817e4
 SHA512:
-  metadata.gz: b4c29de3371f2c52c9377da307c68cb18dbe21006602dcba6a39d172d452956276213f7efc56bf9990edf6cb6b5b81eaf1534a95ec8cb854f2cbdc9e8be6d77d
-  data.tar.gz: 3881622dda89bdcdee30bc421344201e8ea8d1c0bd4164911464beeabc5c522b03155edcc19bea8a5e8444f6ec145ef28f8dcd3c1b2d30a631a55925ac229c6b
+  metadata.gz: afd08217ce63822cf47858221c70cda09b9755b17ce2e08e6fe7156b49b74a9c6c3525e52c2f07b6946ef27d7eccd0fe113846dd51bfa48467f3c8be1641c341
+  data.tar.gz: f0c1d17204b01e640d49ea0cb8747baead11ecbea7c4dd4e1c7a3aa9a938e8c60ca0cbb48044fa4a3b1aaed6cca51546928be77c206c4a263e965fca77cf74fa

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.38.0"
+    VERSION = "0.38.1"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/one_drive/client.rb

@@ -82,6 +82,7 @@ module Multiwoven::Integrations::Source
         @data_type = connection_config[:data_type]
         @file_name = connection_config[:file_name]
         @share_url = connection_config[:share_url]
+        @is_recursive = [true, "true"].include?(connection_config[:is_recursive])
         stored_token = @connector_instance&.configuration&.dig("access_token")
         @access_token = stored_token.presence || refresh_access_token
       end
@@ -173,11 +174,12 @@ module Multiwoven::Integrations::Source
 
       def list_files_in_folder(_connection_config)
         files_in_folder.map do |file|
+          relative_path = relative_file_path(file)
           RecordMessage.new(
             data: {
               element_id: file["id"],
               file_name: file["name"],
-              file_path: file["name"],
+              file_path: relative_path,
               size: file["size"],
               file_type: File.extname(file["name"]).sub(".", ""),
               created_date: file["createdDateTime"],
@@ -190,10 +192,12 @@ module Multiwoven::Integrations::Source
       end
 
       def download_unstructured_file(_connection_config, file_path, sync_id)
-        file_name = resolve_download_file_name(file_path)
-        file_item = files_in_folder.find { |item| item["name"] == file_name }
+        lookup_path = resolve_download_file_name(file_path)
+        file_item = find_file_item(lookup_path)
         raise StandardError, "File not found." if file_item.nil?
 
+        file_name = file_item["name"]
+        relative_path = relative_file_path(file_item)
         local_path = download_file_to_local(
           file_name,
           sync_id,
@@ -206,7 +210,7 @@ module Multiwoven::Integrations::Source
             element_id: file_item["id"],
             local_path: local_path,
             file_name: file_name,
-            file_path: file_name,
+            file_path: relative_path,
             size: file_item["size"],
             file_type: File.extname(file_name).sub(".", ""),
             created_date: file_item["createdDateTime"],
@@ -220,19 +224,35 @@ module Multiwoven::Integrations::Source
       def files_in_folder
         records = fetch_list_items
         records["value"].select do |item|
-          item["folder"].blank? && matching_file_name?(item["name"])
+          item["folder"].blank? && matching_file_name?(item["name"], relative_file_path(item))
         end
       end
 
+      def find_file_item(lookup_path)
+        files = files_in_folder
+        exact_match = files.find do |item|
+          relative_path = relative_file_path(item)
+          relative_path == lookup_path
+        end
+        return exact_match if exact_match
+        return if lookup_path.include?("/")
+
+        files.find { |item| item["name"] == lookup_path }
+      end
+
+      def relative_file_path(file)
+        file["relative_path"].presence || file["name"]
+      end
+
       def resolve_download_file_name(file_path)
-        return File.basename(file_path) unless file_path.to_s.start_with?("http")
+        return file_path.to_s.strip unless file_path.to_s.start_with?("http")
 
         @file_name.to_s.strip.presence || File.basename(file_path)
       end
 
-      def matching_file_name?(name)
+      def matching_file_name?(name, relative_path = nil)
         configured_name = @file_name.to_s.strip
-        configured_name.blank? || configured_name == name
+        configured_name.blank? || configured_name == name || configured_name == relative_path
       end
 
       def discover_stream_for_file(conn, file)
@@ -240,7 +260,7 @@ module Multiwoven::Integrations::Source
         columns = build_discover_columns(describe_results)
 
         Multiwoven::Integrations::Protocol::Stream.new(
-          name: stream_name_for(file["name"]),
+          name: stream_name_for(file),
           action: StreamAction["fetch"],
           json_schema: convert_to_json_schema(columns)
         )
@@ -289,9 +309,10 @@ module Multiwoven::Integrations::Source
       def query(connection, query)
         local_file = nil
         file_name = extract_file_name_from_query(query)
+        local_basename = File.basename(file_name)
         local_file = download_file_to_local(file_name, @sync_id)
 
-        file = read_local_file(connection, file_name, local_file)
+        file = read_local_file(connection, local_basename, local_file)
         query = apply_local_file_to_query(query, file)
         get_results(connection, query).map do |row|
           RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
@@ -393,7 +414,8 @@ module Multiwoven::Integrations::Source
       end
 
       def single_file_item_url(file_name)
-        encoded_file = URI::DEFAULT_PARSER.escape(file_name)
+        # Preserve path separators for nested files; encode other unsafe chars.
+        encoded_file = file_name.to_s.split("/").map { |segment| URI::DEFAULT_PARSER.escape(segment) }.join("/")
 
         if @share_url.present?
           shared = shared_folder_reference
@@ -440,7 +462,35 @@ module Multiwoven::Integrations::Source
 
         return { "value" => [fetch_shared_item_metadata] } if @share_url.present? && shared_folder_reference[:is_file]
 
-        paginated_graph_collection(list_items_url)
+        collect_files_from_folder(list_items_url, recursive: @is_recursive)
+      end
+
+      # Lists files under the configured folder. When recursive is true, BFS over
+      # /children so nested folder files are included in syncs.
+      def collect_files_from_folder(root_url, recursive: false)
+        files = []
+        queue = [[root_url, ""]]
+
+        until queue.empty?
+          url, prefix = queue.shift
+          page = paginated_graph_collection(url)
+
+          page["value"].each do |item|
+            relative_path = prefix.empty? ? item["name"].to_s : "#{prefix}/#{item["name"]}"
+
+            if item["folder"].present?
+              next unless recursive
+
+              drive_id = item.dig("parentReference", "driveId") || @drive_id
+              queue << ["#{drive_item_url(drive_id, item["id"])}/children", relative_path]
+            else
+              item["relative_path"] = relative_path
+              files << item
+            end
+          end
+        end
+
+        { "value" => files }
       end
 
       def fetch_shared_item_metadata
@@ -460,7 +510,9 @@ module Multiwoven::Integrations::Source
         response = microsoft_graph_request(single_file_item_url(@file_name))
         raise graph_api_error(response.body) unless success?(response)
 
-        JSON.parse(response.body)
+        item = JSON.parse(response.body)
+        item["relative_path"] = @file_name.to_s.strip
+        item
       end
 
       def paginated_graph_collection(url)
@@ -498,14 +550,15 @@ module Multiwoven::Integrations::Source
         records["value"].select do |record|
           record["folder"].blank? &&
             SPREADSHEET_EXTENSIONS.include?(File.extname(record["name"].to_s).downcase) &&
-            matching_file_name?(record["name"])
+            matching_file_name?(record["name"], relative_file_path(record))
         end
       end
 
-      # Keep the file extension in the stream name — TableSelector generates
-      # `SELECT * FROM ${stream.name}`, and read_local_file keys off File.extname.
-      def stream_name_for(file_name)
-        File.basename(file_name)
+      # Keep the relative path (with extension) in the stream name — TableSelector
+      # generates `SELECT * FROM ${stream.name}`, and read_local_file keys off
+      # File.extname. Relative paths also disambiguate nested duplicates.
+      def stream_name_for(file)
+        relative_file_path(file)
       end
 
       def encode_sharing_url(url)

data/lib/multiwoven/integrations/source/one_drive/config/spec.json

@@ -45,6 +45,12 @@
         "title": "Share URL",
         "description": "OneDrive or SharePoint sharing link for the folder to read from, skip if using User Name to access Root Folder."
       },
+      "is_recursive": {
+        "type": "boolean",
+        "title": "Enable recursive",
+        "description": "Enables recursive folder traversal. When true, all files and subfolders are read. Default is false, reading only the specified folder.",
+        "default": false
+      },
       "file_name": {
         "type": "string",
         "title": "File Name",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.38.0
+  version: 0.38.1
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-17 00:00:00.000000000 Z
+date: 2026-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport