multiwoven-integrations 0.42.0 → 0.43.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2aafe56dba0839f408ce9b67bd4a5a3e0e7b06081cea437da55299fb83236aa5
-  data.tar.gz: 73de5d0ebc7c416a889f40c6e1131fb3b8d4fb15b8414b0ecfea0d068745dc52
+  metadata.gz: 62e0ad2ac6fc880f4b88dd4a511312b240645d5d1134966659a16225a927848d
+  data.tar.gz: a7368698f3dd5a28894549b635c136b348d43feb25513a1a93f8333a6f9c5ae7
 SHA512:
-  metadata.gz: 9fe49be29f5706af6f8cc6df96394082143c419c56d82cde9222969fd1b45b949d00e1023317fa54e8b3171c649967b0ca8aa70aff8515745ef6cd15d0f8e323
-  data.tar.gz: 80a9c368b79c7251c88692c30f17325690ebc6e96384ae4010ad8a8f878a091f523e50fe1d5a4a7ffaf6ca32922d20711609905b53fb70791ebda2fb5e2db2cc
+  metadata.gz: f0a58e7df26e7579cf5037d6b6605374695a07c6276279da1bd4b9582d7c9f797a001eb9ab7b52e41f402468c7e43c0958d1d1882f27f66aa92617e9a6bd8058
+  data.tar.gz: 8867babc43650be6e51e1a40c129089fa1b2ba46b561967b6d58bf251b3c48d3e42ef1d21ac46b5731d3a44012ffbd5edba298d76d621b3422db06f487b8eb5c

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.42.0"
+    VERSION = "0.43.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -45,6 +45,7 @@ module Multiwoven
       OneDrive
       MicrosoftDynamics
       EpicFhir
+      SqlServer
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/sql_server/client.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+require "tiny_tds"
+
+module Multiwoven::Integrations::Source
+  module SqlServer
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        db = create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue TinyTds::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"],
+          message: e.message
+        ).to_multiwoven_message
+      ensure
+        db&.close
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        schema = connection_config[:schema].presence || "dbo"
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{escape(schema)}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+        records = db.execute(query).map do |row|
+          row.transform_keys { |key| key.to_s.downcase }
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SQLSERVER:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db&.close
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        @pagination_primary_key = sync_config.model.primary_key
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        db = create_connection(connection_config)
+
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SQLSERVER:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      ensure
+        @pagination_primary_key = nil
+        db&.close
+      end
+
+      def search(vector_search_config)
+        connection_config = vector_search_config.source.connection_specification.with_indifferent_access
+        query = vector_search_config[:vector]
+        limit = vector_search_config[:limit]
+        query = batched_query(query, limit, 0) unless limit.nil?
+
+        db = create_connection(connection_config)
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SQLSERVER:SEARCH:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db&.close
+      end
+
+      private
+
+      def query(connection, sql)
+        connection.execute(reformat_query(sql)).map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+
+      # SQL Server does not support LIMIT/OFFSET; use OFFSET/FETCH NEXT.
+      def batched_query(sql_query, limit, offset)
+        offset = offset.to_i
+        limit = limit&.to_i
+        raise ArgumentError, "Offset and limit must be non-negative" if offset.negative? || (!limit.nil? && limit.negative?)
+
+        sql_query = strip_trailing_terminator(sql_query)
+        raise ArgumentError, "Query already contains a LIMIT clause" if clause_outside_literals?(sql_query, /\bLIMIT\s+\d+\b/i)
+        raise ArgumentError, "Query already contains an OFFSET clause" if clause_outside_literals?(sql_query, /\bOFFSET\s+\d+\b/i)
+
+        apply_pagination(sql_query, limit, offset)
+      end
+
+      # Convert Postgres/MySQL-style LIMIT/OFFSET (from upstream, including query_source) into SQL Server pagination.
+      def reformat_query(sql_query)
+        sql_query = strip_trailing_terminator(sql_query)
+
+        return sql_query if clause_outside_literals?(sql_query, /\bOFFSET\s+\d+\s+ROWS\b/i)
+
+        limit = nil
+        offset = nil
+
+        normalized = with_masked_literals(sql_query) do |masked|
+          if (match = masked.match(/\bLIMIT\s+(\d+)\b/i))
+            limit = match[1].to_i
+            masked = masked.sub(/\bLIMIT\s+\d+\b/i, "")
+          end
+
+          if (match = masked.match(/\bOFFSET\s+(\d+)\b(?!\s+ROWS)/i))
+            offset = match[1].to_i
+            masked = masked.sub(/\bOFFSET\s+\d+\b(?!\s+ROWS)/i, "")
+          end
+
+          masked.gsub(/[^\S\n]+/, " ").strip
+        end
+
+        return normalized if limit.nil? && offset.nil?
+
+        apply_pagination(normalized, limit, offset || 0)
+      end
+
+      def apply_pagination(sql_query, limit, offset)
+        raise ArgumentError, "Limit must be at least 1" if !limit.nil? && limit.to_i < 1
+
+        keys = normalize_primary_keys(@pagination_primary_key)
+        has_order = clause_outside_literals?(sql_query, /\bORDER\s+BY\b/i)
+
+        # query_source appends LIMIT without ORDER BY or primary_key metadata.
+        # TOP does not require ORDER BY; OFFSET/FETCH does.
+        return apply_top(sql_query, limit) if offset.to_i.zero? && !limit.nil? && !has_order && keys.empty?
+
+        sql_query = ensure_stable_order(sql_query, keys: keys, has_order: has_order)
+
+        clause = "OFFSET #{offset.to_i} ROWS"
+        clause = "#{clause} FETCH NEXT #{limit.to_i} ROWS ONLY" unless limit.nil?
+
+        "#{sql_query} #{clause}"
+      end
+
+      def apply_top(sql_query, limit)
+        with_masked_literals(sql_query) do |masked|
+          raise ArgumentError, "Query already contains TOP" if masked.match?(/\A\s*SELECT\s+(?:(?:DISTINCT|ALL)\s+)?TOP\b/i)
+
+          pattern = /\A\s*SELECT\s+((?:DISTINCT|ALL)\s+)?/i
+          raise ArgumentError, "Cannot apply TOP to this query; add ORDER BY" unless masked.match?(pattern)
+
+          masked.sub(pattern) { "SELECT #{Regexp.last_match(1)}TOP #{limit.to_i} " }
+        end
+      end
+
+      def ensure_stable_order(sql_query, keys:, has_order:)
+        return append_order_tie_breakers(sql_query, keys) if has_order
+
+        if keys.empty?
+          raise ArgumentError,
+                "Paginated SQL Server queries require an ORDER BY with a unique key, or a model primary_key"
+        end
+
+        "#{sql_query} ORDER BY #{order_by_list(keys)}"
+      end
+
+      def append_order_tie_breakers(sql_query, keys)
+        missing = keys.reject { |key| order_by_includes_key?(sql_query, key) }
+        return sql_query if missing.empty?
+
+        "#{sql_query}, #{order_by_list(missing)}"
+      end
+
+      def order_by_includes_key?(sql_query, key)
+        included = false
+        with_masked_literals(sql_query) do |masked|
+          order_clause = masked[/\bORDER\s+BY\b(.+)\z/im, 1]
+          if order_clause
+            quoted = Regexp.escape(quote_identifier(key))
+            bare = Regexp.escape(key.to_s)
+            included = order_clause.match?(/#{quoted}|\b#{bare}\b/i)
+          end
+          masked
+        end
+        included
+      end
+
+      def normalize_primary_keys(primary_key)
+        Array(primary_key).flat_map { |key| key.to_s.split(",") }.map(&:strip).reject(&:empty?).uniq
+      end
+
+      def order_by_list(keys)
+        keys.map { |key| quote_identifier(key) }.join(", ")
+      end
+
+      def quote_identifier(name)
+        "[#{name.to_s.gsub("]", "]]")}]"
+      end
+
+      def strip_trailing_terminator(sql_query)
+        with_masked_literals(sql_query.to_s.strip) do |masked|
+          masked = masked.rstrip
+          masked = masked.chomp(";") while masked.end_with?(";")
+          masked
+        end
+      end
+
+      def clause_outside_literals?(sql_query, pattern)
+        matched = false
+        with_masked_literals(sql_query) do |masked|
+          matched = masked.match?(pattern)
+          masked
+        end
+        matched
+      end
+
+      # Mask single-quoted SQL literals (including N'...' and escaped '') so rewrites
+      # do not touch values like 'a;b' or 'LIMIT 1'.
+      def with_masked_literals(sql_query)
+        literals = []
+        masked = sql_query.to_s.gsub(/(?:N)?'(?:''|[^'])*'/i) do |literal|
+          literals << literal
+          "__SQL_LITERAL_#{literals.length - 1}__"
+        end
+
+        result = yield(masked)
+        literals.each_with_index do |literal, index|
+          result = result.gsub("__SQL_LITERAL_#{index}__", literal)
+        end
+        result
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        TinyTds::Client.new(
+          username: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          host: connection_config[:host],
+          port: connection_config[:port].presence || 1433,
+          database: connection_config[:database],
+          timeout: 10,
+          azure: connection_config[:azure].presence || false
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+
+      def escape(value)
+        value.to_s.gsub("'", "''")
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/sql_server/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "SqlServer",
+    "title": "SQL Server",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sql-server",
+    "github_issue_label": "source-sql-server",
+    "icon": "https://res.cloudinary.com/dspflukeu/image/upload/v1790358609/Multiwoven/connectors/sql_server/icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/sql_server/config/spec.json

@@ -0,0 +1,94 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sql-server",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "SqlServer",
+    "type": "object",
+    "required": ["data_type", "credentials", "host", "database", "schema"],
+    "properties": {
+      "data_type": {
+        "description": "Type of data in the database",
+        "type": "string",
+        "title": "Data Format Type",
+        "oneOf": [
+          {
+            "const": "structured",
+            "title": "Tables & Records (Structured)"
+          },
+          {
+            "const": "vector",
+            "title": "Tables & Records containing Vector fields"
+          }
+        ],
+        "default": "structured",
+        "order": 0
+      },
+      "credentials": {
+        "title": "",
+        "type": "object",
+        "required": ["auth_type", "username", "password"],
+        "properties": {
+          "auth_type": {
+            "type": "string",
+            "default": "username/password",
+            "order": 0,
+            "readOnly": true
+          },
+          "username": {
+            "description": "Username refers to your individual SQL Server login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
+            "examples": ["SQLSERVER_USER"],
+            "type": "string",
+            "title": "Username",
+            "order": 1
+          },
+          "password": {
+            "description": "This field requires the password associated with the SQL Server login specified in the preceding section.",
+            "type": "string",
+            "multiwoven_secret": true,
+            "title": "Password",
+            "order": 2
+          }
+        },
+        "order": 1
+      },
+      "host": {
+        "description": "The hostname or IP address of your SQL Server instance.",
+        "examples": ["127.0.0.1"],
+        "type": "string",
+        "title": "Host",
+        "order": 2
+      },
+      "port": {
+        "description": "The port number for your SQL Server instance, which defaults to 1433 and may vary based on your configuration.",
+        "examples": ["1433"],
+        "type": "string",
+        "title": "Port",
+        "order": 3
+      },
+      "database": {
+        "description": "The specific SQL Server database to connect to.",
+        "examples": ["SQLSERVER_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 4
+      },
+      "schema": {
+        "description": "The schema within the SQL Server database, typically 'dbo'.",
+        "examples": ["dbo"],
+        "default": "dbo",
+        "type": "string",
+        "title": "Schema",
+        "order": 5
+      },
+      "azure": {
+        "description": "Enable when connecting to Azure SQL Database (required for Azure encryption/login behaviour).",
+        "type": "boolean",
+        "title": "Azure SQL",
+        "default": false,
+        "order": 6
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/sql_server/icon.svg

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<svg width="800px" height="800px" viewBox="0 -141.54 1478.201 1478.201" xmlns="http://www.w3.org/2000/svg">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

data/lib/multiwoven/integrations.rb

@@ -113,6 +113,7 @@ require_relative "integrations/source/aisquared/client"
 require_relative "integrations/source/one_drive/client"
 require_relative "integrations/source/microsoft_dynamics/client"
 require_relative "integrations/source/epic_fhir/client"
+require_relative "integrations/source/sql_server/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.42.0
+  version: 0.43.0
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-24 00:00:00.000000000 Z
+date: 2026-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -913,6 +913,10 @@ files:
 - lib/multiwoven/integrations/source/snowflake/config/meta.json
 - lib/multiwoven/integrations/source/snowflake/config/spec.json
 - lib/multiwoven/integrations/source/snowflake/icon.svg
+- lib/multiwoven/integrations/source/sql_server/client.rb
+- lib/multiwoven/integrations/source/sql_server/config/meta.json
+- lib/multiwoven/integrations/source/sql_server/config/spec.json
+- lib/multiwoven/integrations/source/sql_server/icon.svg
 - lib/multiwoven/integrations/source/watsonx_ai/client.rb
 - lib/multiwoven/integrations/source/watsonx_ai/config/catalog.json
 - lib/multiwoven/integrations/source/watsonx_ai/config/meta.json