multiwoven-integrations 0.1.63 → 0.1.64

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d27c9915ed80faa33f88467a0dddfaeda4c5c91560ca7b94ba9970ece2430bb5
-  data.tar.gz: abefc5bb7771864fe8d1e4b6aee4810fa4f36e8db507c634f99d2546cae46df1
+  metadata.gz: a04192999a6c02ae4dd3cf752c22f602d9ed196e3a293f67c53727b89ec0ce3a
+  data.tar.gz: 955cab0486157d96c00ba540de776dd9d32687366ddd260bfa4e9a81c95e017d
 SHA512:
-  metadata.gz: e39874d8fce8e065b862a460b84e7e35ca8cef3c1a897a507c6590feb39d45c4ec5a1b4c4ee206f9b7ce8e5763a5cfe77ea4b60ed4861e9f97b83c75ce40f37d
-  data.tar.gz: 251d70c151bc3bcb39bc5d3e20464dbc351353dd814b370f45631b3f58ae586ba1e21ab277d67b89943ad5905623a1aa0eb64629ebd0049ec6a774374fe55d41
+  metadata.gz: 30a2e1bbd87278e54f40ce5c8d07630d4b4e4b3a1b7f077c5f1bfdf9e00d37dd402bbe3fbcfa477cf7fd331929a9f2e41d8506e7ac9388e3403ae73e2b474be3
+  data.tar.gz: 77840062ae81f7efa968919d7377a0e324c6b7720ffe3f237996a55c850e8f7dc68f2a241086780ea4d22d66a2b3c3a5d57e3045aa6c921a909d6e2a36584acc

data/lib/multiwoven/integrations/destination/sftp/client.rb

@@ -39,27 +39,58 @@ module Multiwoven::Integrations::Destination
         file_path = generate_file_path(sync_config)
         local_file_name = generate_local_file_name(sync_config)
         csv_content = generate_csv_content(records)
+        records_size = records.size
         write_success = 0
-        write_failure = 0
 
+        case connection_config[:format][:compression_type]
+        when CompressionType.enum("zip")
+          write_success = write_compressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
+        when CompressionType.enum("un_compressed")
+          write_success = write_uncompressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
+        else
+          raise ArgumentError, "Unsupported compression type: #{connection_config[:format][:compression_type]}"
+        end
+        write_failure = records.size - write_success
+        tracking_message(write_success, write_failure)
+      rescue StandardError => e
+        handle_exception(
+          "SFTP:WRITE:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def write_compressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
+        write_success = 0
+        Tempfile.create([local_file_name, ".zip"]) do |tempfile|
+          Zip::File.open(tempfile.path, Zip::File::CREATE) do |zipfile|
+            zipfile.get_output_stream("#{local_file_name}.csv") { |f| f.write(csv_content) }
+          end
+          with_sftp_client(connection_config) do |sftp|
+            sftp.upload!(tempfile.path, file_path)
+            write_success = records_size
+          rescue StandardError => e
+            handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
+            write_success = 0
+          end
+        end
+        write_success
+      end
+
+      def write_uncompressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
+        write_success = 0
         Tempfile.create([local_file_name, ".csv"]) do |tempfile|
           tempfile.write(csv_content)
           tempfile.close
           with_sftp_client(connection_config) do |sftp|
             sftp.upload!(tempfile.path, file_path)
-            write_success += records.size
+            write_success = records_size
           rescue StandardError => e
             handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
-            write_failure += records.size
+            write_success = 0
           end
         end
-        tracking_message(write_success, write_failure)
-      rescue StandardError => e
-        handle_exception(
-          "SFTP:WRITE:EXCEPTION",
-          "error",
-          e
-        )
+        write_success
       end
 
       def clear_all_records(sync_config)
@@ -82,7 +113,13 @@ module Multiwoven::Integrations::Destination
       def generate_file_path(sync_config)
         connection_specification = sync_config.destination.connection_specification.with_indifferent_access
         timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
-        file_name = "#{connection_specification[:file_name]}_#{timestamp}.csv"
+        format = connection_specification[:format]
+        extension = if format[:compression_type] == "un_compressed"
+                      format[:format_type]
+                    else
+                      format[:compression_type]
+                    end
+        file_name = "#{connection_specification[:file_name]}_#{timestamp}.#{extension}"
         File.join(connection_specification[:destination_path], file_name)
       end
 

data/lib/multiwoven/integrations/destination/sftp/config/spec.json

@@ -4,7 +4,7 @@
   "connection_specification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "SFTP",
-    "required": ["host", "username", "password", "destination_path"],
+    "required": ["host", "username", "password", "destination_path", "format" ],
     "properties": {
       "host": {
         "title": "Host",
@@ -45,6 +45,28 @@
         "type": "string",
         "description": "Name of the file to be written.",
         "order": 5
+      },
+      "format": {
+        "title": "Output Format",
+        "type": "object",
+        "description": "Format of the data output.",
+        "order": 6,
+        "required": ["format_type"],
+        "properties": {
+          "format_type": {
+            "title": "File Format Type",
+            "type": "string",
+            "enum": ["csv"],
+            "default": "csv"
+          },
+          "compression_type": {
+            "title": "Compression Type",
+            "description": "Whether the output files should be compressed.",
+            "type": "string",
+            "enum": ["un_compressed", "zip"],
+            "default": "un_compressed"
+          }
+        }
       }
     }
   }

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -26,6 +26,8 @@ module Multiwoven
     LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
     RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
     SchemaMode = Types::String.enum("schema", "schemaless")
+    FileFormatType = Types::String.enum("csv")
+    CompressionType = Types::String.enum("un_compressed", "zip")
 
     class ProtocolModel < Dry::Struct
       extend Multiwoven::Integrations::Core::Utils

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.63"
+    VERSION = "0.1.64"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations.rb

@@ -22,6 +22,7 @@ require "stripe"
 require "net/sftp"
 require "csv"
 require "securerandom"
+require "zip"
 
 # Service
 require_relative "integrations/config"

data/multiwoven-integrations.gemspec

@@ -49,6 +49,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "restforce"
   spec.add_runtime_dependency "ruby-limiter"
   spec.add_runtime_dependency "ruby-odbc"
+  spec.add_runtime_dependency "rubyzip"
   spec.add_runtime_dependency "sequel"
   spec.add_runtime_dependency "slack-ruby-client"
   spec.add_runtime_dependency "stripe"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.63
+  version: 0.1.64
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-04-30 00:00:00.000000000 Z
+date: 2024-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -234,6 +234,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement