multiwoven-integrations 0.41.5 → 0.41.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c53fd23ba288ab3578c1c23581ceff400328eda9c010f05e1f5a60509fddf2f
-  data.tar.gz: 88d651895e365873ed1abdd6bcaa88190edd1aadf217dbf0c61640d7d3fff85e
+  metadata.gz: 9997be9180829657c2c1961957ac5af3ba80f9085ec53610b2dc9426105c5324
+  data.tar.gz: 0f92f5434027f6b19e0f83f785f3e791da95b42e22834c66e9fbe9011543d411
 SHA512:
-  metadata.gz: 0ff426493d73236e35bd1a989bb78816b3d330320f21b2c14fe933e289403115bf920fed3cd962099808dcd04747ab22b153690c184a5732d87b2e12dc2c9825
-  data.tar.gz: 55aa60ff60a337f75934c9c77410161c18e96589edeca97985e514089abd03022faf399b056dc0cfee255ba1c735ed2be7ad9bf9b7ea2d9faecc3f5b882e9a84
+  metadata.gz: 1ad58e7cbaefab9d122e5775404b3ac7095e1b6deaaea45093f8c23f46a2d6f2f5c9a98bd73df77b1c5d5e06211ae49bd1e50c114ce322b290e40b5a5995d564
+  data.tar.gz: c9ae0e891adbb23830fb35342f28f84186e433477e0eb488f49b99c45f5e365b1d3953f66a333b5be24db82f133f216ed1735de1d13696d7c5a227286076c619

data/lib/multiwoven/integrations/core/base_connector.rb

@@ -21,7 +21,12 @@ module Multiwoven
       end
 
       def model_catalog
-        @model_catalog ||= ModelCatalog.new(models: (live_models + curated_models).uniq { |model| model[:id] })
+        @model_catalog ||= begin
+          models = (live_models + curated_models)
+                   .select { |model| include_model?(model) }
+                   .uniq { |model| model[:id] }
+          ModelCatalog.new(models: models)
+        end
       rescue Dry::Struct::Error => e
         Integrations::Service.logger.error("#{self.class}: unusable model catalog: #{e.message}")
         @model_catalog = ModelCatalog.new(models: [])
@@ -72,6 +77,21 @@ module Multiwoven
 
       private
 
+      # Only list what the connector can actually serve. No connector has an
+      # image-generation payload path, so those are dropped everywhere.
+      def include_model?(model)
+        !image_output_model?(model)
+      end
+
+      def image_output_model?(model)
+        (model[:type] || model["type"]).to_s == "image"
+      end
+
+      def embedding_model?(model)
+        [model[:type] || model["type"], model[:model_type] || model["model_type"]]
+          .any? { |value| value.to_s == "embedding" }
+      end
+
       def read_json(file_path)
         path = Object.const_source_location(self.class.to_s)[0]
         connector_folder = File.dirname(path)

data/lib/multiwoven/integrations/core/constants.rb

@@ -75,6 +75,8 @@ module Multiwoven
       GOOGLE_SPREADSHEET_ID_REGEX = %r{/d/([-\w]{20,})/}.freeze
 
       OPEN_AI_URL = "https://api.openai.com/v1/chat/completions"
+      OPEN_AI_COMPLETIONS_URL = "https://api.openai.com/v1/completions"
+      OPEN_AI_RESPONSES_URL = "https://api.openai.com/v1/responses"
       ANTHROPIC_URL = "https://api.anthropic.com/v1/messages"
 
       # Bedrock Models

data/lib/multiwoven/integrations/core/streaming_http_client.rb

@@ -10,11 +10,22 @@ module Multiwoven
           http = configure_http(uri, config)
           request = build_request(method, uri, payload, headers)
           http.request(request) do |response|
+            raise_error_response(response) unless (200..299).cover?(response.code.to_i)
+
             response.read_body do |chunk|
               yield chunk if block_given? # Pass each response chunk
             end
           end
         end
+
+        private
+
+        # Without this a 4xx body is parsed as if it were an SSE stream.
+        def raise_error_response(response)
+          body = +""
+          response.read_body { |chunk| body << chunk }
+          raise "HTTP #{response.code}: #{body.strip}"
+        end
       end
     end
   end

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.41.5"
+    VERSION = "0.41.6"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/aisquared/client.rb

@@ -35,6 +35,11 @@ module Multiwoven::Integrations::Source
 
       private
 
+      # Every request goes to /chat, so embeddings cannot connect here.
+      def include_model?(model)
+        super && !embedding_model?(model)
+      end
+
       def lightning_embedding_url
         host = AISQUARED_BOLT_URL.to_s.strip
         raise AisquaredError, "AISQUARED_BOLT_URL is not configured" if host.empty?

data/lib/multiwoven/integrations/source/anthropic/client.rb

@@ -67,7 +67,7 @@ module Multiwoven::Integrations::Source
         send_request(
           url: url,
           http_method: http_method,
-          payload: JSON.parse(payload),
+          payload: normalize_payload(payload),
           headers: build_headers(connection_config, streaming: false),
           config: connection_config[:config]
         )
@@ -84,7 +84,7 @@ module Multiwoven::Integrations::Source
         send_streaming_request(
           url: ANTHROPIC_URL,
           http_method: HTTP_POST,
-          payload: JSON.parse(payload),
+          payload: normalize_payload(payload),
           headers: build_headers(connection_config, streaming: true),
           config: connection_config[:config]
         ) do |chunk|
@@ -94,6 +94,17 @@ module Multiwoven::Integrations::Source
         handle_exception(e, { context: "ANTHROPIC:RUN_STREAM_MODEL:EXCEPTION", type: "error" })
       end
 
+      def normalize_payload(payload)
+        parsed = payload.is_a?(String) ? JSON.parse(payload) : payload.deep_dup
+        return parsed unless parsed.respond_to?(:deep_stringify_keys)
+
+        parsed = parsed.deep_stringify_keys
+        model = parsed["model"].to_s
+        # OpenRouter ids carry a vendor prefix, dots and a routing suffix; Anthropic ids have none.
+        parsed["model"] = model.split(":").first.to_s.split("/").last.to_s.tr(".", "-") if model.present?
+        parsed
+      end
+
       def process_response(response)
         if success?(response)
           data = JSON.parse(response.body)

data/lib/multiwoven/integrations/source/open_ai/client.rb

@@ -6,12 +6,9 @@ module Multiwoven::Integrations::Source
     class Client < SourceConnector
       def check_connection(connection_config)
         connection_config = prepare_config(connection_config)
-        response = send_request(
-          url: OPEN_AI_URL,
-          http_method: HTTP_POST,
-          payload: JSON.parse(connection_config[:request_format]),
-          headers: auth_headers(connection_config[:api_key]),
-          config: connection_config[:config]
+        response = request_with_endpoint_fallback(
+          connection_config,
+          connection_config[:request_format]
         )
         success?(response) ? success_status : failure_status_from_response(response)
       rescue StandardError => e
@@ -60,21 +57,65 @@ module Multiwoven::Integrations::Source
       end
 
       def run_model(connection_config, payload)
-        response = send_request(
-          url: OPEN_AI_URL,
+        response = request_with_endpoint_fallback(connection_config, payload)
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, { context: "OPEN AI:RUN_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def run_model_stream(connection_config, payload)
+        stream_with_endpoint_fallback(connection_config, payload) do |message|
+          yield message if block_given?
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "OPEN AI:RUN_STREAM_MODEL:EXCEPTION", type: "error" })
+      end
+
+      # Some OpenAI models reject /v1/chat/completions and require /v1/completions or /v1/responses.
+      def request_with_endpoint_fallback(connection_config, payload, url: OPEN_AI_URL)
+        normalized = normalize_payload(payload)
+        response = post_openai(connection_config, adapt_payload_for_url(normalized, url), url)
+        return response if success?(response)
+
+        fallback_url = resolve_fallback_url(response_error_message(response))
+        return response if fallback_url.blank? || fallback_url == url
+
+        post_openai(connection_config, adapt_payload_for_url(normalized, fallback_url), fallback_url)
+      end
+
+      def stream_with_endpoint_fallback(connection_config, payload, url: OPEN_AI_URL)
+        normalized = normalize_payload(payload)
+        emitted = false
+
+        begin
+          post_openai_stream(connection_config, streaming_payload(normalized, url), url) do |message|
+            emitted = true
+            yield message if block_given?
+          end
+        rescue StandardError => e
+          fallback_url = resolve_fallback_url(e.message)
+          # Retrying after the consumer has seen chunks would deliver them twice.
+          raise if fallback_url.blank? || fallback_url == url || emitted
+
+          post_openai_stream(connection_config, streaming_payload(normalized, fallback_url), fallback_url) do |message|
+            yield message if block_given?
+          end
+        end
+      end
+
+      def post_openai(connection_config, payload, url)
+        send_request(
+          url: url,
           http_method: HTTP_POST,
           payload: payload,
           headers: auth_headers(connection_config[:api_key]),
           config: connection_config[:config]
         )
-        process_response(response)
-      rescue StandardError => e
-        handle_exception(e, { context: "OPEN AI:RUN_MODEL:EXCEPTION", type: "error" })
       end
 
-      def run_model_stream(connection_config, payload)
+      def post_openai_stream(connection_config, payload, url)
         send_streaming_request(
-          url: OPEN_AI_URL,
+          url: url,
           http_method: HTTP_POST,
           payload: payload,
           headers: auth_headers(connection_config[:api_key]),
@@ -82,13 +123,100 @@ module Multiwoven::Integrations::Source
         ) do |chunk|
           process_streaming_response(chunk) { |message| yield message if block_given? }
         end
-      rescue StandardError => e
-        handle_exception(e, { context: "OPEN AI:RUN_STREAM_MODEL:EXCEPTION", type: "error" })
+      end
+
+      # /v1/responses renames the token cap and rejects the chat-only keys.
+      RESPONSES_DROP_KEYS = %w[messages max_tokens n stop response_format
+                               frequency_penalty presence_penalty logprobs top_logprobs logit_bias].freeze
+      COMPLETIONS_DROP_KEYS = %w[messages response_format tools tool_choice].freeze
+
+      def resolve_fallback_url(message)
+        return if message.blank?
+
+        return OPEN_AI_RESPONSES_URL if message.match?(%r{v1/responses}i)
+        return OPEN_AI_COMPLETIONS_URL if message.match?(%r{v1/completions}i) || message.match?(/not a chat model/i)
+
+        nil
+      end
+
+      def response_error_message(response)
+        return if response.nil?
+
+        parsed = JSON.parse(response.body)
+        return parsed.dig("error", "message") || parsed["message"].to_s if parsed.is_a?(Hash)
+
+        parsed.to_s
+      rescue JSON::ParserError, TypeError
+        response&.body.to_s
+      end
+
+      def adapt_payload_for_url(payload, url)
+        case url
+        when OPEN_AI_RESPONSES_URL then responses_payload(payload)
+        when OPEN_AI_COMPLETIONS_URL then completions_payload(payload)
+        else payload
+        end
+      end
+
+      def responses_payload(payload)
+        payload.except(*RESPONSES_DROP_KEYS).tap do |adapted|
+          adapted["max_output_tokens"] ||= payload["max_tokens"] if payload["max_tokens"]
+          adapted["input"] ||= responses_input(payload)
+        end
+      end
+
+      def completions_payload(payload)
+        payload.except(*COMPLETIONS_DROP_KEYS).tap do |adapted|
+          adapted["prompt"] ||= extract_prompt_text(payload)
+        end
+      end
+
+      # Responses takes structured input, so roles survive instead of being flattened.
+      def responses_input(payload)
+        messages = payload["messages"] || payload[:messages]
+        return payload["input"] || payload["prompt"] || "" if messages.blank?
+
+        Array(messages).filter_map { |message| responses_input_message(message) }
+      end
+
+      def responses_input_message(message)
+        content = message["content"] || message[:content]
+        return if content.blank?
+
+        { "role" => message["role"] || message[:role] || "user", "content" => content }
+      end
+
+      def streaming_payload(payload, url)
+        adapt_payload_for_url(payload, url).merge("stream" => true)
+      end
+
+      def extract_prompt_text(payload)
+        messages = payload["messages"] || payload[:messages]
+        return payload["prompt"] || payload["input"] || "" if messages.blank?
+
+        Array(messages).filter_map { |message| message_text(message["content"] || message[:content]) }.join("\n")
+      end
+
+      def message_text(content)
+        return content if content.is_a?(String)
+        return nil if content.blank?
+
+        Array(content).filter_map { |part| part.is_a?(Hash) ? part["text"] || part[:text] : part }.join("\n").presence
+      end
+
+      # check_connection posts to a hardcoded chat endpoint, so embeddings cannot connect here.
+      def include_model?(model)
+        super && !embedding_model?(model)
+      end
+
+      def normalize_payload(payload)
+        parsed = payload.is_a?(String) ? JSON.parse(payload) : payload.deep_dup
+        parsed.respond_to?(:deep_stringify_keys) ? parsed.deep_stringify_keys : parsed
       end
 
       def process_response(response)
         if success?(response)
-          data = JSON.parse(response.body)
+          data = normalize_response_data(JSON.parse(response.body))
           [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
         else
           create_log_message("OPEN AI:RUN_MODEL", "error", "request failed: #{response.body}")
@@ -97,8 +225,80 @@ module Multiwoven::Integrations::Source
         handle_exception(e, { context: "OPEN AI:PROCESS_RESPONSE:EXCEPTION", type: "error" })
       end
 
+      # Completions and Responses APIs use different shapes; Model Hub catalogs and
+      # x-output-path expect the chat shape choices[].message.content.
+      def normalize_response_data(data)
+        return data unless data.is_a?(Hash)
+        return normalize_responses_api_data(data) if data["object"] == "response" || data["output"].is_a?(Array)
+
+        choices = data["choices"]
+        return data unless choices.is_a?(Array)
+
+        data.merge(
+          "choices" => choices.map { |choice| normalize_choice(choice) }
+        )
+      end
+
+      def normalize_responses_api_data(data)
+        return data if data.dig("choices", 0, "message", "content").present?
+
+        text = extract_responses_api_text(data["output"])
+        return data if text.nil?
+
+        data.merge(
+          "choices" => [
+            {
+              "index" => 0,
+              "message" => { "role" => "assistant", "content" => text },
+              "finish_reason" => data["status"] == "completed" ? "stop" : data["status"]
+            }
+          ]
+        )
+      end
+
+      def extract_responses_api_text(output)
+        return nil unless output.is_a?(Array)
+
+        texts = output.flat_map { |item| responses_api_message_texts(item) }
+        texts.presence&.join
+      end
+
+      def responses_api_message_texts(item)
+        return [] unless item.is_a?(Hash)
+        return [] unless item["type"] == "message" || item["role"] == "assistant"
+
+        Array(item["content"]).filter_map { |part| responses_api_text_part(part) }
+      end
+
+      def responses_api_text_part(part)
+        return unless part.is_a?(Hash)
+        return unless part["type"].nil? || part["type"].to_s.include?("text")
+
+        part["text"].presence || part["content"].presence
+      end
+
+      def normalize_choice(choice)
+        return choice unless choice.is_a?(Hash)
+        return choice if choice.dig("message", "content").present?
+
+        text = choice["text"]
+        return choice if text.nil?
+
+        choice.merge(
+          "message" => { "role" => choice.dig("message", "role") || "assistant", "content" => text }
+        )
+      end
+
+      # Responses SSE prefixes every frame with an "event:" line, which is not JSON.
       def extract_data_entries(chunk)
-        chunk.split(/^data: /).map(&:strip).reject(&:empty?)
+        entries = chunk.to_s.split("\n").filter_map do |line|
+          line = line.strip
+          line.delete_prefix("data:").strip.presence if line.start_with?("data:")
+        end
+        return entries if entries.any?
+
+        # A non-SSE body (a JSON error payload) arrives unframed and drives the fallback.
+        [chunk.to_s.strip].reject(&:empty?)
       end
 
       def process_streaming_response(chunk)
@@ -110,9 +310,43 @@ module Multiwoven::Integrations::Source
 
           raise StandardError, "Error: #{data["error"]["message"]}" if data["error"] && data["error"]["message"]
 
+          data = normalize_streaming_data(data)
+          next if data.nil?
+
           yield [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] if block_given?
         end
       end
+
+      def normalize_streaming_data(data)
+        return nil unless data.is_a?(Hash)
+
+        type = data["type"].to_s
+        return responses_streaming_delta(data) if type.start_with?("response.")
+
+        choices = data["choices"]
+        return data unless choices.is_a?(Array)
+
+        data.merge("choices" => choices.map { |choice| normalize_streaming_choice(choice) })
+      end
+
+      def responses_streaming_delta(data)
+        return nil unless data["type"].to_s.include?("output_text.delta")
+
+        text = data["delta"].presence || data["text"].presence
+        return nil if text.blank?
+
+        { "choices" => [{ "index" => 0, "delta" => { "content" => text } }] }
+      end
+
+      def normalize_streaming_choice(choice)
+        return choice unless choice.is_a?(Hash)
+        return choice if choice.dig("delta", "content").present? || choice.dig("message", "content").present?
+
+        text = choice["text"]
+        return choice if text.nil?
+
+        choice.merge("delta" => { "content" => text })
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.41.5
+  version: 0.41.6
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-02 00:00:00.000000000 Z
+date: 2026-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport