multiwoven-integrations 0.1.42 → 0.1.43

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eee51b7ba03e877f2765da694f9ca7b4ede8e44e07ca74931d808f79fb828c06
-  data.tar.gz: 21a23a656e09175222172eea24e562c95ccc97e7bebe92443cc60678d68050bf
+  metadata.gz: c2f57a8f398e40f9a12c710487ff9818640abab9e2b445a1ad1a3959350c35c6
+  data.tar.gz: 60837c95c23653870d97ca8daa865987e15d3c677f9111aaca1221d56d9eb323
 SHA512:
-  metadata.gz: 388d743e298e3296fbd9bd12e5d29037d72bf4912f93ae8ae8b9d7f5285dbb2d39f39f1500f6ee3b6214e186564287eec8c3fe14d66f55cc53861585e3929e33
-  data.tar.gz: e527726e1e69c57c07d7d6ab28194f2a84dfbbffb941f61e2b5fca927cecf486edcb7fa2cecbbcc8ccc1713c396e1723161e5ea55f0cc8d310ce2d888f6de845
+  metadata.gz: 474a754e9cbd76963d786e6c059f96d731cc05ef7d26427e6f8606a8383dfa130e2a4905a9ebaed6af65d7495de56cea80c566940ed8e76355a8c6b27b34f783
+  data.tar.gz: e4073c88c0b76e56f46497841abdcc34b02f8b6cfa33b25b418abc4d37759702cc295c8957d2632301ce8cf596526e622e2087b35a484ea187e1bfcc670d457e

data/lib/multiwoven/integrations/core/utils.rb

@@ -79,7 +79,8 @@ module Multiwoven
           streams: streams,
           request_rate_limit: catalog_json["request_rate_limit"] || 60,
           request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
-          request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10
+          request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10,
+          schema_mode: catalog_json["schema_mode"] || ["schema"]
         )
       end
 
@@ -95,8 +96,7 @@ module Multiwoven
           request_rate_limit: stream_json["request_rate_limit"].to_i,
           request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
           request_rate_concurrency: stream_json["request_rate_concurrency"].to_i,
-          supported_sync_modes: stream_json["supported_sync_modes"],
-          schema_mode: stream_json["schema_mode"]
+          supported_sync_modes: stream_json["supported_sync_modes"]
         )
       end
     end

data/lib/multiwoven/integrations/destination/sftp/client.rb

@@ -3,7 +3,7 @@
 module Multiwoven::Integrations::Destination
   module Sftp
     include Multiwoven::Integrations::Core
-    class Client < DestinationConnector
+    class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
       prepend Multiwoven::Integrations::Core::Fullrefresher
       prepend Multiwoven::Integrations::Core::RateLimiter
 
@@ -37,16 +37,21 @@ module Multiwoven::Integrations::Destination
       def write(sync_config, records, _action = "insert")
         connection_config = sync_config.destination.connection_specification.with_indifferent_access
         file_path = generate_file_path(sync_config)
+        local_file_name = generate_local_file_name(sync_config)
         csv_content = generate_csv_content(records)
         write_success = 0
         write_failure = 0
-        # 10000 records in single
-        with_sftp_client(connection_config) do |sftp|
-          sftp.file.open(file_path, "w") { |file| file.puts(csv_content) }
-          write_success += records.size
-        rescue StandardError => e
-          handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
-          write_failure += records.size
+
+        Tempfile.create([local_file_name, ".csv"]) do |tempfile|
+          tempfile.write(csv_content)
+          tempfile.close
+          with_sftp_client(connection_config) do |sftp|
+            sftp.upload!(tempfile.path, file_path)
+            write_success += records.size
+          rescue StandardError => e
+            handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
+            write_failure += records.size
+          end
         end
         tracking_message(write_success, write_failure)
       rescue StandardError => e
@@ -78,9 +83,15 @@ module Multiwoven::Integrations::Destination
       private
 
       def generate_file_path(sync_config)
+        connection_specification = sync_config.destination.connection_specification.with_indifferent_access
         timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
         file_name = "#{sync_config.stream.name}_#{timestamp}.csv"
-        File.join(sync_config.destination.connection_specification[:destination_path], file_name)
+        File.join(connection_specification[:destination_path], file_name)
+      end
+
+      def generate_local_file_name(sync_config)
+        timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
+        "#{sync_config.stream.name}_#{timestamp}"
       end
 
       def generate_csv_content(records)

data/lib/multiwoven/integrations/destination/sftp/config/catalog.json

@@ -2,13 +2,13 @@
   "request_rate_limit": 600,
   "request_rate_limit_unit": "minute",
   "request_rate_concurrency": 10,
+  "schema_mode": ["schemaless"],
   "streams": [
     {
       "name": "sftp",
       "batch_support": true,
-      "batch_size": 10000,
+      "batch_size": 100000,
       "action": "create",
-      "schema_mode": ["schemaless"],
       "json_schema": {},
       "supported_sync_modes": ["full_refresh","incremental"]
     }

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -125,7 +125,6 @@ module Multiwoven
       attribute? :request_rate_limit, Types::Integer
       attribute? :request_rate_limit_unit, RequestRateLimitingUnit
       attribute? :request_rate_concurrency, Types::Integer
-      attribute? :schema_mode, Types::Array.of(SchemaMode).optional.default(["schema"])
 
       def rate_limit_unit_seconds
         case request_rate_limit_unit
@@ -148,6 +147,7 @@ module Multiwoven
       attribute? :request_rate_limit, Types::Integer.default(60)
       attribute? :request_rate_limit_unit, RequestRateLimitingUnit
       attribute? :request_rate_concurrency, Types::Integer.default(10)
+      attribute? :schema_mode, Types::Array.of(SchemaMode).optional.default(["schema"])
 
       def to_multiwoven_message
         MultiwovenMessage.new(

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.42"
+    VERSION = "0.1.43"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb

@@ -39,8 +39,13 @@ module Multiwoven
           def read(sync_config)
             connection_config = sync_config.source.connection_specification.with_indifferent_access
             initialize_client(connection_config)
+            return [] if sync_config.offset&.> 2000
+
+            # TODO: Salesforce imposes a limit on the use of OFFSET in SOQL queries, where you cannot skip(offset) more than 2000 records.
+            # This limitation can hinder the retrieval of large datasets in a single query.
+            # To overcome this, we need a cursor-based pagination strategy instead of relying on OFFSET.
+            # query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
             query = sync_config.model.query
-            query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
             exclude_keys = ["attributes"]
             queried_data = @client.query(query)
             results = queried_data.map do |record|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.42
+  version: 0.1.43
 platform: ruby
 authors:
 - Subin T P