multiwoven-integrations 0.33.6 → 0.34.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48b04e24af2a47fa9b506e924fa3b5540e2a8da35cd5abac444e495a68bd3440
-  data.tar.gz: b9d01d845e32acaa279aa02ccec2699900e36469640ae0f5b6d5ce8d42773077
+  metadata.gz: 3ebf8ef48419c9724217ce90888b0cea454b715dcaec7cab8903c3b994e0e23f
+  data.tar.gz: 94048a4a4d2794fdeee73b9b2426f18e77f5412d6af9eab4a3bbafcac934277d
 SHA512:
-  metadata.gz: 3568483d79a0260b4742b7042595a5ad27783bd70adf14f388b095a0fb37fd45d83d0250eb90cf2c1d3870a7ca7ed4955b39585ff54c5ffe580a8b3aced55d9b
-  data.tar.gz: 0d67ef9289ebf131ab54385adb81477574ac8c6ff207f110211d976f8090cc0d6f5abf85a9f6f3a1f075025301466e469d2e575a609c3218f8d249ed3393effd
+  metadata.gz: 5f02b62f2097263bac9db7cbf9ea3961d72816ca6af8628a8fb56fc54626c344647471d2774da61abcff0d7031f20fdc76fea2631651535435a797da3c9eea79
+  data.tar.gz: f7de34a831108a64f987f510882568c2ac19d58177be6178ffbe25b97f4baf80ca59bca3d62afb0055d2d0e86d97b98383dc8756ead177b63302fc8e424f2229

data/lib/multiwoven/integrations/core/rate_limiter.rb

@@ -14,6 +14,18 @@ module Multiwoven
 
         super(sync_config, records, action)
       end
+
+      def read(sync_config)
+        stream = sync_config.stream
+
+        @queue ||= Limiter::RateQueue.new(stream.request_rate_limit, interval: stream.rate_limit_unit_seconds) do
+          Integrations::Service.logger.info("Hit the limit for stream: #{stream.name}, waiting")
+        end
+
+        @queue.shift
+
+        super(sync_config)
+      end
     end
   end
 end

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -165,6 +165,12 @@ module Multiwoven
       end
     end
 
+    class IncrementStrategyConfig < ProtocolModel
+      attr_accessor :offset, :limit, :offset_variable, :limit_variable
+
+      attribute :increment_strategy, Types::String.default("page")
+    end
+
     class SyncConfig < ProtocolModel
       attr_accessor :offset, :limit, :sync_run_id
 
@@ -178,6 +184,8 @@ module Multiwoven
       attribute :destination_sync_mode, DestinationSyncMode
       # reference ids
       attribute :sync_id, Types::String.default("unknown")
+      # increment strategy
+      attribute? :increment_strategy_config, IncrementStrategyConfig.optional
     end
 
     class VectorConfig < ProtocolModel

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.33.6"
+    VERSION = "0.34.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -33,6 +33,7 @@ module Multiwoven
       Firecrawl
       Odoo
       GoogleDrive
+      Http
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/http/client.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module Http
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = prepare_config(connection_config)
+        create_connection(connection_config)
+        if connection_config[:sample_query].blank?
+          build_paginated_request(connection_config, {})
+        else
+          sample_query = JSON.parse(connection_config[:sample_query])
+          build_paginated_request(connection_config, sample_query.values.first)
+        end
+        response = send_request(
+          url: @url,
+          http_method: connection_config[:http_method],
+          payload: connection_config[:request_format],
+          headers: connection_config[:headers],
+          config: connection_config[:config],
+          params: connection_config[:params]
+        )
+        success?(response) ? success_status : failure_status(nil)
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        connection_config = prepare_config(connection_config)
+        create_connection(connection_config)
+        if connection_config[:sample_query].blank?
+          build_paginated_request(connection_config, {})
+        else
+          sample_query = JSON.parse(connection_config[:sample_query])
+          build_paginated_request(connection_config, sample_query.values.first)
+        end
+        response = send_request(
+          url: @url,
+          http_method: connection_config[:http_method],
+          payload: connection_config[:request_format],
+          headers: connection_config[:headers],
+          config: connection_config[:config],
+          params: connection_config[:params]
+        )
+        raise StandardError, "Response code: #{response.code}, Body: #{response.body}" unless success?(response)
+
+        catalog = Catalog.new(streams: create_streams(JSON.parse(response.body)))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        connection_config = create_connection(connection_config)
+        if sync_config.increment_strategy_config
+          @limit = sync_config.increment_strategy_config.limit
+          @offset = sync_config.increment_strategy_config.offset
+        else
+          @limit = sync_config.limit
+          @offset = sync_config.offset
+        end
+        query = sync_config.model.query
+        query(connection_config, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "HTTP:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def prepare_config(config)
+        config.with_indifferent_access.tap do |conf|
+          conf[:config][:timeout] ||= 30
+        end
+      end
+
+      def create_connection(connection_config)
+        @url = "#{connection_config[:base_url]}#{connection_config[:path]}"
+        connection_config
+      end
+
+      def build_paginated_request(connection_config, query)
+        connection_config[:request_format] = JSON.parse(connection_config[:request_format] || "{}")
+
+        apply_param_pagination(connection_config)
+        apply_batched_query(connection_config, query)
+      end
+
+      def apply_param_pagination(connection_config)
+        return unless connection_config[:limit_param].present? && connection_config[:offset_param].present?
+
+        connection_config[:params] = {} if connection_config[:params].nil?
+        connection_config[:params].merge!({ connection_config[:limit_param] => @limit }) if @limit.present?
+        connection_config[:params].merge!({ connection_config[:offset_param] => @offset }) if @offset.present?
+      end
+
+      def apply_batched_query(connection_config, query)
+        return unless connection_config[:sample_query].present?
+
+        sample_query = JSON.parse(connection_config[:sample_query])
+        query = batched_query(query, @limit, @offset) unless @limit.nil? && @offset.nil?
+        connection_config[:request_format].merge!({ sample_query.keys.first => query }) unless query.nil?
+      end
+
+      def query(connection_config, query)
+        connection_config = prepare_config(connection_config)
+        build_paginated_request(connection_config, query)
+        response = send_request(
+          url: @url,
+          http_method: connection_config[:http_method],
+          payload: connection_config[:request_format],
+          headers: connection_config[:headers],
+          config: connection_config[:config],
+          params: connection_config[:params] || {}
+        )
+        if success?(response)
+          response_body = JSON.parse(response.body)
+          parse_response = get_parse_response(connection_config[:parse_response])
+          parse_response(response_body, parse_response)
+        else
+          handle_exception("Failed to fetch data", { context: "HTTP:QUERY:EXCEPTION", type: "error" })
+        end
+      end
+
+      def create_streams(response_body)
+        group_by_table(response_body).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r["name"], action: StreamAction["fetch"], json_schema: r["schema"])
+        end
+      end
+
+      def get_parse_response(parse_response)
+        parse_response = JSON.parse(parse_response) if parse_response.is_a?(String) && parse_response.start_with?("[")
+        parse_response
+      end
+
+      def parse_response(response_body, parse_response)
+        case parse_response
+        when Array
+          records = []
+          parse_response.each do |path|
+            records << JsonPath.on(response_body, path)
+          end
+          records[1].each_slice(records[0].size).map do |row_values|
+            data = Hash[records[0].zip(row_values)]
+            RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message
+          end
+        else
+          records = JsonPath.on(response_body, parse_response)
+          records.map do |data|
+            RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message
+          end
+        end
+      end
+
+      def build_schema(record)
+        case record
+        when Hash
+          {
+            "type" => "object",
+            "properties" => record.transform_values { |value| build_schema(value) }
+          }
+        when Array
+          {
+            "type" => "array",
+            "items" => build_schema(record.first)
+          }
+        else
+          { "type" => %w[string null] }
+        end
+      end
+
+      def group_by_table(response_body)
+        schema = []
+        response_body.each do |key, values|
+          schema << {
+            "name" => key.to_s,
+            "schema" => {
+              "$schema" => "http://json-schema.org/draft-07/schema#",
+              "type" => "object",
+              "properties" => {
+                key.to_s => build_schema(values)
+              }
+            }
+          }
+        end
+        schema
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/http/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "Http",
+    "title": "HTTP",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/http",
+    "github_issue_label": "source-http",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/http/config/spec.json

@@ -0,0 +1,117 @@
+{
+  "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/http-model-endpoint",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "HTTP",
+    "type": "object",
+    "required": ["increment_type","base_url", "http_method"],
+    "properties": {
+      "increment_type": {
+        "type": "string",
+        "title": "Increment Type",
+        "enum": ["Page", "Offset"],
+        "default": "Page",
+        "order": 0
+      },
+      "http_method": {
+        "type": "string",
+        "title": "HTTP Method",
+        "enum": ["POST", "GET"],
+        "order": 1
+      },
+      "base_url": {
+        "type": "string",
+        "title": "URL",
+        "order": 2
+      },
+      "path": {
+        "type": "string",
+        "title": "Path",
+        "description": "The path to the resource on the server. This is appended to the base URL.",
+        "order": 3,
+        "default": "/"
+      },
+      "headers": {
+        "title": "HTTP Headers",
+        "description": "Custom headers to include in the HTTP request. Useful for authentication, content type specifications, and other request metadata.",
+        "order": 4,
+        "additionalProperties": {
+          "type": "string"
+        }
+      },
+      "config": {
+        "title": "",
+        "type": "object",
+        "properties": {
+          "timeout": {
+            "type": "string",
+            "default": "30",
+            "title": "HTTP Timeout",
+            "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
+            "order": 0
+          }
+        },
+        "order": 5
+      },
+      "request_format":{
+        "title": "Request Format",
+        "description": "Request payload format. This should be a valid JSON string that matches the expected input of the HTTP endpoint.",
+        "type": "string",
+        "x-request-format": true,
+        "order": 6
+      },
+      "sample_query": {
+        "title": "Sample Query",
+        "description": "A sample query to test the connection and retrieve a sample response from the HTTP endpoint. This should be a valid JSON string that matches the expected input of the HTTP endpoint.",
+        "type": "string",
+        "x-request-format": true,
+        "order": 7,
+        "default": "{ \"query\": \"Select * from table\" }"
+      },
+      "parse_response": {
+        "title": "Parse Response",
+        "description": "A JSONPath expression that points to a single record (usually the first row) in the response. The keys of this object are treated as the column names. Example: `$.data[0]`",
+        "type": "string",
+        "x-response-format": true,
+        "order": 8,
+        "default": "$.data[0]"
+      },
+      "params": {
+        "title": "Query Parameters",
+        "description": "Key-value pairs to be included as query parameters in the HTTP request URL.",
+        "order": 9,
+        "additionalProperties": {
+          "type": "string"
+        }
+      },
+      "offset_param": {
+        "type": "string",
+        "title": "Offset Parameter",
+        "description": "The name of the query parameter used to specify the offset for pagination.",
+        "order": 10,
+        "default": "offset"
+      },
+      "limit_param": {
+        "type": "string",
+        "title": "Limit Parameter",
+        "description": "The name of the query parameter used to specify the limit for pagination.",
+        "order": 11,
+        "default": "limit"
+      },
+      "page_size": {
+        "type": "string",
+        "title": "Page Size",
+        "description": "The number of records to fetch per page. This value is used in conjunction with the limit parameter for pagination.",
+        "order": 12
+      },
+      "page_start": {
+        "type": "string",
+        "title": "Page Start",
+        "description": "The starting page number for pagination. This value is used to calculate the offset for the initial request.",
+        "order": 13
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/http/icon.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<svg width="800px" height="800px" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none">
+
+
+
+
+
+

data/lib/multiwoven/integrations.rb

@@ -45,6 +45,7 @@ require "xmlrpc/client"
 require "googleauth"
 require "google/apis/drive_v3"
 require "aws-sdk-textract"
+require "jsonpath"
 
 # Service
 require_relative "integrations/config"
@@ -96,6 +97,7 @@ require_relative "integrations/source/qdrant/client"
 require_relative "integrations/source/firecrawl/client"
 require_relative "integrations/source/odoo/client"
 require_relative "integrations/source/google_drive/client"
+require_relative "integrations/source/http/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.33.6
+  version: 0.34.0
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-09-13 00:00:00.000000000 Z
+date: 2025-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -775,6 +775,10 @@ files:
 - lib/multiwoven/integrations/source/google_vertex_model/config/meta.json
 - lib/multiwoven/integrations/source/google_vertex_model/config/spec.json
 - lib/multiwoven/integrations/source/google_vertex_model/icon.svg
+- lib/multiwoven/integrations/source/http/client.rb
+- lib/multiwoven/integrations/source/http/config/meta.json
+- lib/multiwoven/integrations/source/http/config/spec.json
+- lib/multiwoven/integrations/source/http/icon.svg
 - lib/multiwoven/integrations/source/http_model/client.rb
 - lib/multiwoven/integrations/source/http_model/config/catalog.json
 - lib/multiwoven/integrations/source/http_model/config/meta.json