multiwoven-integrations 0.42.0 → 0.43.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2aafe56dba0839f408ce9b67bd4a5a3e0e7b06081cea437da55299fb83236aa5
-  data.tar.gz: 73de5d0ebc7c416a889f40c6e1131fb3b8d4fb15b8414b0ecfea0d068745dc52
+  metadata.gz: 4fa94a603b7e80f97d72f8d40959925b4f7fa434b1870773011561e2e11300b0
+  data.tar.gz: 71586d86bae1182eaddbeadc983f0745a1b2c32aaa61b09bc2fb0c47a7c10381
 SHA512:
-  metadata.gz: 9fe49be29f5706af6f8cc6df96394082143c419c56d82cde9222969fd1b45b949d00e1023317fa54e8b3171c649967b0ca8aa70aff8515745ef6cd15d0f8e323
-  data.tar.gz: 80a9c368b79c7251c88692c30f17325690ebc6e96384ae4010ad8a8f878a091f523e50fe1d5a4a7ffaf6ca32922d20711609905b53fb70791ebda2fb5e2db2cc
+  metadata.gz: c7797375fb072c56132d1509a3fd6cb8a2432fca25bbb62461d5497bd9bd4bc9927ea7c0b87a362c61919cf1539e6cd9d151e76484a7b690d56ec0d6e774f4c7
+  data.tar.gz: e19a77ece6aa7b4fa10550ed62a1e3dc5414802845f3add6d7588f5b151a886a5cda66534b63f1b9857add61adb8a1214d56c861b224c2ae92a9d87fa3d21bba

data/.ruby-version

@@ -1 +1 @@
-3.2.0
+3.4.11

data/lib/multiwoven/integrations/core/query_builder.rb

@@ -9,7 +9,7 @@ module Multiwoven
         case action.downcase
         when "destination_insert"
           columns = record.keys.join(", ")
-          values = record.values.map { |value| "'#{value}'" }.join(", ")
+          values = record.values.map { |value| "'#{Utils.serialize_value(value)}'" }.join(", ")
           # TODO: support bulk insert
           "INSERT INTO #{table} (#{columns}) VALUES (#{values});"
         when "destination_update"
@@ -21,7 +21,7 @@ module Multiwoven
           end
 
           primary_key_value = record.delete(primary_key) # Remove and return the primary key value
-          set_clause = record.map { |key, value| "#{key} = '#{value}'" }.join(", ")
+          set_clause = record.map { |key, value| "#{key} = '#{Utils.serialize_value(value)}'" }.join(", ")
           where_clause = "#{primary_key} = '#{primary_key_value}'"
           "UPDATE #{table} SET #{set_clause} WHERE #{where_clause};"
         else

data/lib/multiwoven/integrations/core/utils.rb

@@ -3,6 +3,11 @@
 module Multiwoven
   module Integrations::Core
     module Utils
+      def serialize_value(value)
+        value.is_a?(Hash) || value.is_a?(Array) ? value.to_json : value
+      end
+      module_function :serialize_value
+
       def keys_to_symbols(hash)
         if hash.is_a?(Hash)
           hash.each_with_object({}) do |(key, value), result|

data/lib/multiwoven/integrations/destination/amazon_s3/client.rb

@@ -96,7 +96,9 @@ module Multiwoven::Integrations::Destination
         CSV.generate do |csv|
           headers = records.first.keys
           csv << headers
-          records.each { |record| csv << record.values_at(*headers) }
+          records.each do |record|
+            csv << record.values_at(*headers).map { |v| serialize_value(v) }
+          end
         end
       end
 

data/lib/multiwoven/integrations/destination/postgresql/client.rb

@@ -153,7 +153,7 @@ module Multiwoven::Integrations::Destination
       def escape_value(db, value)
         return "NULL" if value.nil?
 
-        "'#{db.escape_string(value.to_s)}'"
+        "'#{db.escape_string(serialize_value(value).to_s)}'"
       end
 
       def quote_ident(name)

data/lib/multiwoven/integrations/destination/sftp/client.rb

@@ -149,7 +149,9 @@ module Multiwoven::Integrations::Destination
         CSV.generate do |csv|
           headers = records.first.keys
           csv << headers
-          records.each { |record| csv << record.values_at(*headers) }
+          records.each do |record|
+            csv << record.values_at(*headers).map { |v| serialize_value(v) }
+          end
         end
       end
 

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.42.0"
+    VERSION = "0.43.1"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -45,6 +45,7 @@ module Multiwoven
       OneDrive
       MicrosoftDynamics
       EpicFhir
+      SqlServer
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/sql_server/client.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+require "tiny_tds"
+
+module Multiwoven::Integrations::Source
+  module SqlServer
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        db = create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue TinyTds::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"],
+          message: e.message
+        ).to_multiwoven_message
+      ensure
+        db&.close
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        schema = connection_config[:schema].presence || "dbo"
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{escape(schema)}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+        records = db.execute(query).map do |row|
+          row.transform_keys { |key| key.to_s.downcase }
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SQLSERVER:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db&.close
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        @pagination_primary_key = sync_config.model.primary_key
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        db = create_connection(connection_config)
+
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SQLSERVER:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      ensure
+        @pagination_primary_key = nil
+        db&.close
+      end
+
+      def search(vector_search_config)
+        connection_config = vector_search_config.source.connection_specification.with_indifferent_access
+        query = vector_search_config[:vector]
+        limit = vector_search_config[:limit]
+        query = batched_query(query, limit, 0) unless limit.nil?
+
+        db = create_connection(connection_config)
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SQLSERVER:SEARCH:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db&.close
+      end
+
+      private
+
+      def query(connection, sql)
+        connection.execute(reformat_query(sql)).map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+
+      # SQL Server does not support LIMIT/OFFSET; use OFFSET/FETCH NEXT.
+      def batched_query(sql_query, limit, offset)
+        offset = offset.to_i
+        limit = limit&.to_i
+        raise ArgumentError, "Offset and limit must be non-negative" if offset.negative? || (!limit.nil? && limit.negative?)
+
+        sql_query = strip_trailing_terminator(sql_query)
+        raise ArgumentError, "Query already contains a LIMIT clause" if clause_outside_literals?(sql_query, /\bLIMIT\s+\d+\b/i)
+        raise ArgumentError, "Query already contains an OFFSET clause" if clause_outside_literals?(sql_query, /\bOFFSET\s+\d+\b/i)
+
+        apply_pagination(sql_query, limit, offset)
+      end
+
+      # Convert Postgres/MySQL-style LIMIT/OFFSET (from upstream, including query_source) into SQL Server pagination.
+      def reformat_query(sql_query)
+        sql_query = strip_trailing_terminator(sql_query)
+
+        return sql_query if clause_outside_literals?(sql_query, /\bOFFSET\s+\d+\s+ROWS\b/i)
+
+        limit = nil
+        offset = nil
+
+        normalized = with_masked_literals(sql_query) do |masked|
+          if (match = masked.match(/\bLIMIT\s+(\d+)\b/i))
+            limit = match[1].to_i
+            masked = masked.sub(/\bLIMIT\s+\d+\b/i, "")
+          end
+
+          if (match = masked.match(/\bOFFSET\s+(\d+)\b(?!\s+ROWS)/i))
+            offset = match[1].to_i
+            masked = masked.sub(/\bOFFSET\s+\d+\b(?!\s+ROWS)/i, "")
+          end
+
+          masked.gsub(/[^\S\n]+/, " ").strip
+        end
+
+        return normalized if limit.nil? && offset.nil?
+
+        apply_pagination(normalized, limit, offset || 0)
+      end
+
+      def apply_pagination(sql_query, limit, offset)
+        raise ArgumentError, "Limit must be at least 1" if !limit.nil? && limit.to_i < 1
+
+        keys = normalize_primary_keys(@pagination_primary_key)
+        has_order = clause_outside_literals?(sql_query, /\bORDER\s+BY\b/i)
+
+        # query_source appends LIMIT without ORDER BY or primary_key metadata.
+        # TOP does not require ORDER BY; OFFSET/FETCH does.
+        return apply_top(sql_query, limit) if offset.to_i.zero? && !limit.nil? && !has_order && keys.empty?
+
+        sql_query = ensure_stable_order(sql_query, keys: keys, has_order: has_order)
+
+        clause = "OFFSET #{offset.to_i} ROWS"
+        clause = "#{clause} FETCH NEXT #{limit.to_i} ROWS ONLY" unless limit.nil?
+
+        "#{sql_query} #{clause}"
+      end
+
+      def apply_top(sql_query, limit)
+        with_masked_literals(sql_query) do |masked|
+          raise ArgumentError, "Query already contains TOP" if masked.match?(/\A\s*SELECT\s+(?:(?:DISTINCT|ALL)\s+)?TOP\b/i)
+
+          pattern = /\A\s*SELECT\s+((?:DISTINCT|ALL)\s+)?/i
+          raise ArgumentError, "Cannot apply TOP to this query; add ORDER BY" unless masked.match?(pattern)
+
+          masked.sub(pattern) { "SELECT #{Regexp.last_match(1)}TOP #{limit.to_i} " }
+        end
+      end
+
+      def ensure_stable_order(sql_query, keys:, has_order:)
+        return append_order_tie_breakers(sql_query, keys) if has_order
+
+        if keys.empty?
+          raise ArgumentError,
+                "Paginated SQL Server queries require an ORDER BY with a unique key, or a model primary_key"
+        end
+
+        "#{sql_query} ORDER BY #{order_by_list(keys)}"
+      end
+
+      def append_order_tie_breakers(sql_query, keys)
+        missing = keys.reject { |key| order_by_includes_key?(sql_query, key) }
+        return sql_query if missing.empty?
+
+        "#{sql_query}, #{order_by_list(missing)}"
+      end
+
+      def order_by_includes_key?(sql_query, key)
+        included = false
+        with_masked_literals(sql_query) do |masked|
+          order_clause = masked[/\bORDER\s+BY\b(.+)\z/im, 1]
+          if order_clause
+            quoted = Regexp.escape(quote_identifier(key))
+            bare = Regexp.escape(key.to_s)
+            included = order_clause.match?(/#{quoted}|\b#{bare}\b/i)
+          end
+          masked
+        end
+        included
+      end
+
+      def normalize_primary_keys(primary_key)
+        Array(primary_key).flat_map { |key| key.to_s.split(",") }.map(&:strip).reject(&:empty?).uniq
+      end
+
+      def order_by_list(keys)
+        keys.map { |key| quote_identifier(key) }.join(", ")
+      end
+
+      def quote_identifier(name)
+        "[#{name.to_s.gsub("]", "]]")}]"
+      end
+
+      def strip_trailing_terminator(sql_query)
+        with_masked_literals(sql_query.to_s.strip) do |masked|
+          masked = masked.rstrip
+          masked = masked.chomp(";") while masked.end_with?(";")
+          masked
+        end
+      end
+
+      def clause_outside_literals?(sql_query, pattern)
+        matched = false
+        with_masked_literals(sql_query) do |masked|
+          matched = masked.match?(pattern)
+          masked
+        end
+        matched
+      end
+
+      # Mask single-quoted SQL literals (including N'...' and escaped '') so rewrites
+      # do not touch values like 'a;b' or 'LIMIT 1'.
+      def with_masked_literals(sql_query)
+        literals = []
+        masked = sql_query.to_s.gsub(/(?:N)?'(?:''|[^'])*'/i) do |literal|
+          literals << literal
+          "__SQL_LITERAL_#{literals.length - 1}__"
+        end
+
+        result = yield(masked)
+        literals.each_with_index do |literal, index|
+          result = result.gsub("__SQL_LITERAL_#{index}__", literal)
+        end
+        result
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        TinyTds::Client.new(
+          username: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          host: connection_config[:host],
+          port: connection_config[:port].presence || 1433,
+          database: connection_config[:database],
+          timeout: 10,
+          azure: connection_config[:azure].presence || false
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+
+      def escape(value)
+        value.to_s.gsub("'", "''")
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/sql_server/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "SqlServer",
+    "title": "SQL Server",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sql-server",
+    "github_issue_label": "source-sql-server",
+    "icon": "https://res.cloudinary.com/dspflukeu/image/upload/v1790358609/Multiwoven/connectors/sql_server/icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/sql_server/config/spec.json

@@ -0,0 +1,94 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sql-server",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "SqlServer",
+    "type": "object",
+    "required": ["data_type", "credentials", "host", "database", "schema"],
+    "properties": {
+      "data_type": {
+        "description": "Type of data in the database",
+        "type": "string",
+        "title": "Data Format Type",
+        "oneOf": [
+          {
+            "const": "structured",
+            "title": "Tables & Records (Structured)"
+          },
+          {
+            "const": "vector",
+            "title": "Tables & Records containing Vector fields"
+          }
+        ],
+        "default": "structured",
+        "order": 0
+      },
+      "credentials": {
+        "title": "",
+        "type": "object",
+        "required": ["auth_type", "username", "password"],
+        "properties": {
+          "auth_type": {
+            "type": "string",
+            "default": "username/password",
+            "order": 0,
+            "readOnly": true
+          },
+          "username": {
+            "description": "Username refers to your individual SQL Server login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
+            "examples": ["SQLSERVER_USER"],
+            "type": "string",
+            "title": "Username",
+            "order": 1
+          },
+          "password": {
+            "description": "This field requires the password associated with the SQL Server login specified in the preceding section.",
+            "type": "string",
+            "multiwoven_secret": true,
+            "title": "Password",
+            "order": 2
+          }
+        },
+        "order": 1
+      },
+      "host": {
+        "description": "The hostname or IP address of your SQL Server instance.",
+        "examples": ["127.0.0.1"],
+        "type": "string",
+        "title": "Host",
+        "order": 2
+      },
+      "port": {
+        "description": "The port number for your SQL Server instance, which defaults to 1433 and may vary based on your configuration.",
+        "examples": ["1433"],
+        "type": "string",
+        "title": "Port",
+        "order": 3
+      },
+      "database": {
+        "description": "The specific SQL Server database to connect to.",
+        "examples": ["SQLSERVER_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 4
+      },
+      "schema": {
+        "description": "The schema within the SQL Server database, typically 'dbo'.",
+        "examples": ["dbo"],
+        "default": "dbo",
+        "type": "string",
+        "title": "Schema",
+        "order": 5
+      },
+      "azure": {
+        "description": "Enable when connecting to Azure SQL Database (required for Azure encryption/login behaviour).",
+        "type": "boolean",
+        "title": "Azure SQL",
+        "default": false,
+        "order": 6
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/sql_server/icon.svg

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<svg width="800px" height="800px" viewBox="0 -141.54 1478.201 1478.201" xmlns="http://www.w3.org/2000/svg">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

data/lib/multiwoven/integrations.rb

@@ -113,6 +113,7 @@ require_relative "integrations/source/aisquared/client"
 require_relative "integrations/source/one_drive/client"
 require_relative "integrations/source/microsoft_dynamics/client"
 require_relative "integrations/source/epic_fhir/client"
+require_relative "integrations/source/sql_server/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

data/multiwoven-integrations.gemspec

@@ -39,6 +39,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "aws-sdk-s3"
   spec.add_runtime_dependency "aws-sdk-sts"
   spec.add_runtime_dependency "aws-sigv4"
+  spec.add_runtime_dependency "base64"
+  spec.add_runtime_dependency "bigdecimal"
   spec.add_runtime_dependency "csv"
   spec.add_runtime_dependency "dry-schema"
   spec.add_runtime_dependency "dry-struct"
@@ -52,6 +54,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "hubspot-api-client"
   spec.add_runtime_dependency "iterable-api-client"
   spec.add_runtime_dependency "jwt"
+  spec.add_runtime_dependency "logger"
   spec.add_runtime_dependency "MailchimpMarketing"
   spec.add_runtime_dependency "net-sftp"
   spec.add_runtime_dependency "pg"

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.42.0
+  version: 0.43.1
 platform: ruby
 authors:
 - Subin T P
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-24 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -94,6 +93,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bigdecimal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -276,6 +303,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: MailchimpMarketing
   requirement: !ruby/object:Gem::Requirement
@@ -913,6 +954,10 @@ files:
 - lib/multiwoven/integrations/source/snowflake/config/meta.json
 - lib/multiwoven/integrations/source/snowflake/config/spec.json
 - lib/multiwoven/integrations/source/snowflake/icon.svg
+- lib/multiwoven/integrations/source/sql_server/client.rb
+- lib/multiwoven/integrations/source/sql_server/config/meta.json
+- lib/multiwoven/integrations/source/sql_server/config/spec.json
+- lib/multiwoven/integrations/source/sql_server/icon.svg
 - lib/multiwoven/integrations/source/watsonx_ai/client.rb
 - lib/multiwoven/integrations/source/watsonx_ai/config/catalog.json
 - lib/multiwoven/integrations/source/watsonx_ai/config/meta.json
@@ -939,7 +984,6 @@ metadata:
   homepage_uri: https://www.multiwoven.com/
   source_code_uri: https://github.com/Multiwoven/multiwoven/tree/main/integrations
   changelog_uri: https://github.com/Multiwoven/multiwoven/blob/main/integrations/CHANGELOG.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -954,8 +998,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Integration suite for open source reverse ETL platform
 test_files: []