multiwoven-integrations 0.1.48 → 0.1.49

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f070f9727eba1e360878e33850045dfe4b508f351d91cb6fa3e6ae46edda8ad7
-  data.tar.gz: 425485c695c2803835c98549695d343d63db2c5d388328dc983759a50fdd63ee
+  metadata.gz: 96d535997b9e775eb4f677ba0be5722ee3aaa88f3226a6fa3b4aef24fb949ffe
+  data.tar.gz: a1c0be33b6c70efadb6c4bed8016a8b8c65cbb66ef430194d50cd0220b325559
 SHA512:
-  metadata.gz: ae737e08dbae12d4cea2c51082684f5dcfa1d50d75100252bcaaf3171922d185072ad5894a9546f695a22b3b8112b176fb8f6a1d9caa3a8614240243188373a8
-  data.tar.gz: 7d1adab5630abc8fb506391f49476b285f6753f79f8b779bf87db87ffb918c5e794dba28ee419b19629ea6f87ed93040bd81af787e92fc02f2cebef6d058dfd1
+  metadata.gz: 406892b62ca959ec83fd237568a6d5475f51656d1083b54ac87ff1c08a4cbb49ffc5eca436694e30514c6280cbb943ad8b66f894cb549d4241bacb6e10895fd3
+  data.tar.gz: bc3bf83c198472ef8af07f6763a02e31b8abc7742bf3b5f0fff3483094526870f88ad5f88df6291301eec3e207541e62a84d249f6c5df861ffc7debdea35db18

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.48"
+    VERSION = "0.1.49"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb

@@ -47,10 +47,10 @@ module Multiwoven
             # To overcome this, we need a cursor-based pagination strategy instead of relying on OFFSET.
             # query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
             query = sync_config.model.query
-            exclude_keys = ["attributes"]
+            query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
             queried_data = @client.query(query)
             results = queried_data.map do |record|
-              record.reject { |key, _| exclude_keys.include?(key) }
+              flatten_nested_hash(record)
             end
             results.map do |row|
               RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
@@ -62,16 +62,27 @@ module Multiwoven
           private
 
           def query(connection, query)
-            exclude_keys = ["attributes"]
             queried_data = connection.query(query)
+
             results = queried_data.map do |record|
-              record.reject { |key, _| exclude_keys.include?(key) }
+              flatten_nested_hash(record)
             end
             results.map do |row|
               RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
             end
           end
 
+          def flatten_nested_hash(record, prefix = nil)
+            record = record.reject { |key, _| key == "attributes" }
+            record.flat_map do |key, value|
+              if value.is_a?(Hash)
+                flatten_nested_hash(value, prefix ? "#{prefix}_#{key}" : key)
+              else
+                { prefix ? "#{prefix}_#{key}" : key => value }
+              end
+            end.reduce({}, :merge)
+          end
+
           def create_connection(connection_config)
             initialize_client(connection_config)
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.48
+  version: 0.1.49
 platform: ruby
 authors:
 - Subin T P