RubyGems - multiwoven-integrations - Versions diffs - 0.19.3 → 0.20.0 - Mend

multiwoven-integrations 0.19.3 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (10) hide show

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfc51c82e8c79ab8f6b275559a6b15167d23ee1f501c58ca257d3d748b2e9bf8
-  data.tar.gz: 27aaf1554717099d3485a1d230f6804117373ec74382f0faa78d748d16a14d44
+  metadata.gz: eb53fac768bc4dc4cf394e1414fa84762041448762d9e48517288ba7cbe9c56c
+  data.tar.gz: 6e1e7eb31a0174aeb981c1d27c62c30fc29db5b39de9a9dba3b79cb1cdbd6c5b
 SHA512:
-  metadata.gz: a03defe388d9c0a96b80b070201d34ee9520630c7c1adebf3ca7b0158316ab4af0dae524755c3eed2ecbdbb98dce16316ce3224bea9b874f42cd01c9a0c3c912
-  data.tar.gz: da3a1b3c468090850d9b4732a261dcc82f124325fe8db7f20a9577a8022395ecd30a5c55dddea272b61e5fc6a79acac0d77c1d88419c6960ef901b3a32293c59
+  metadata.gz: dd355e29baafc570c76e9165163a5020cbc491ed1aa1afc311f6bff049c24072065aad1c06cd974731c64a846803b144ad66de11e1e0d5c75de4ca0bbb83c931
+  data.tar.gz: 801eae3f6cb3f1e1ae6f54b1ba75e38a6b264b35680010d3657b299c4e7d97f216db854b824f3210de1dfa7757b46120a62fac3aff1346ba64e185a3f0a426e4

data/lib/multiwoven/integrations/core/constants.rb CHANGED Viewed

@@ -55,6 +55,11 @@ module Multiwoven
       GOOGLE_VERTEX_ENDPOINT_SERVICE_URL = "%<region>s-aiplatform.googleapis.com"
       GOOGLE_VERTEX_MODEL_NAME = "projects/%<project_id>s/locations/%<region>s/endpoints/%<endpoint_id>s"
+      WATSONX_HEALTH_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v4/deployments?version=%<version>s"
+      WATSONX_PREDICTION_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v4/deployments/%<deployment_id>s/predictions?version=%<version>s"
+      WATSONX_GENERATION_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v1/deployments/%<deployment_id>s/text/generation?version=%<version>s"
+      WATSONX_STREAM_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v1/deployments/%<deployment_id>s/text/generation_stream?version=%<version>s"
       # HTTP
       HTTP_GET = "GET"
       HTTP_POST = "POST"

data/lib/multiwoven/integrations/rollout.rb CHANGED Viewed

@@ -2,7 +2,7 @@
 module Multiwoven
   module Integrations
-    VERSION = "0.19.3"
+    VERSION = "0.20.0"
     ENABLED_SOURCES = %w[
       Snowflake
@@ -22,6 +22,7 @@ module Multiwoven
       HttpModel
       OpenAI
       Sftp
+      WatsonxAi
     ].freeze
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/watsonx_ai/client.rb ADDED Viewed

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+module Multiwoven::Integrations::Source
+  module WatsonxAi
+    include Multiwoven::Integrations::Core
+    API_VERSION = "2021-05-01"
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        get_access_token(connection_config[:api_key])
+        url = format(
+          WATSONX_HEALTH_DEPLOYMENT_URL,
+          region: connection_config[:region],
+          version: API_VERSION
+        )
+        response = send_request(
+          url: url,
+          http_method: HTTP_GET,
+          payload: {},
+          headers: auth_headers(@access_token),
+          config: connection_config[:config]
+        )
+        evaluate_deployment_status(response, connection_config[:deployment_id])
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+      def discover(_connection_config)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:DISCOVER:EXCEPTION", type: "error" })
+      end
+      def read(sync_config)
+        connection_config, payload = prepare_config_and_payload(sync_config)
+        process_model_request(connection_config, payload) { |message| yield message if block_given? }
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:READ:EXCEPTION", type: "error" })
+      end
+      private
+      def process_model_request(connection_config, payload)
+        if connection_config[:is_stream] && connection_config[:model_type] == "Prompt template"
+          run_model_stream(connection_config, payload) { |message| yield message if block_given? }
+        else
+          run_model(connection_config, payload)
+        end
+      end
+      def evaluate_deployment_status(response, deployment_id)
+        response_body = JSON.parse(response.body)
+        deployment_status = response_body["resources"]&.find { |res| res.dig("metadata", "id") == deployment_id }
+        return failure_status unless deployment_status
+        deployment_status.dig("entity", "status", "state") == "ready" ? success_status : failure_status
+      end
+      def prepare_config_and_payload(sync_config)
+        config = sync_config.source.connection_specification
+        connection_config = config.with_indifferent_access.tap do |conf|
+          conf[:config][:timeout] ||= 30
+          conf[:is_stream] ||= false
+        end
+        payload = sync_config.model.query
+        [connection_config, payload]
+      end
+      def get_access_token(api_key)
+        cache = defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : ActiveSupport::Cache::MemoryStore.new
+        cache_key = "watsonx_ai_#{api_key}"
+        cached_token = cache.read(cache_key)
+        if cached_token
+          @access_token = cached_token
+        else
+          new_token = get_iam_token(api_key)
+          # max expiration is 3 minutes. No way to make it higher
+          cache.write(cache_key, new_token, expires_in: 180)
+          @access_token = new_token
+        end
+      end
+      def get_iam_token(api_key)
+        uri = URI("https://iam.cloud.ibm.com/identity/token")
+        request = Net::HTTP::Post.new(uri)
+        request["Content-Type"] = "application/x-www-form-urlencoded"
+        request.body = "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=#{api_key}"
+        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+        raise "Failed to get IAM token: #{response.body}" unless response.is_a?(Net::HTTPSuccess)
+        JSON.parse(response.body)["access_token"]
+      end
+      def parse_json(json_string)
+        JSON.parse(json_string)
+      rescue JSON::ParserError => e
+        handle_exception(e, { context: "OPEN AI:PARSE_JSON:EXCEPTION", type: "error" })
+        {}
+      end
+      def run_model(connection_config, payload)
+        get_access_token(connection_config[:api_key])
+        url = format(
+          connection_config[:model_type] == "Machine learning model" ? WATSONX_PREDICTION_DEPLOYMENT_URL : WATSONX_GENERATION_DEPLOYMENT_URL,
+          region: connection_config[:region],
+          deployment_id: connection_config[:deployment_id],
+          version: API_VERSION
+        )
+        response = send_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: JSON.parse(payload),
+          headers: auth_headers(@access_token),
+          config: connection_config[:config]
+        )
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:RUN_MODEL:EXCEPTION", type: "error" })
+      end
+      def process_response(response)
+        if success?(response)
+          if response.body.start_with?("{") || response.body.start_with?("[")
+            data = JSON.parse(response.body)
+            [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
+          else
+            data = format_data(response.body)
+            RecordMessage.new(data: { responses: data }, emitted_at: Time.now.to_i).to_multiwoven_message
+          end
+        else
+          create_log_message("WATSONX AI:RUN_MODEL", "error", "request failed: #{response.body}")
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:PROCESS_RESPONSE:EXCEPTION", type: "error" })
+      end
+      def run_model_stream(connection_config, payload)
+        get_access_token(connection_config[:api_key])
+        url = format(
+          WATSONX_STREAM_DEPLOYMENT_URL,
+          region: connection_config[:region],
+          deployment_id: connection_config[:deployment_id],
+          version: API_VERSION
+        )
+        send_streaming_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: JSON.parse(payload),
+          headers: auth_headers(@access_token),
+          config: connection_config[:config]
+        ) do |chunk|
+          process_streaming_response(chunk) { |message| yield message if block_given? }
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:RUN_STREAM_MODEL:EXCEPTION", type: "error" })
+      end
+      def format_data(response_body)
+        messages = response_body.split("\n\n")
+        messages.map do |message|
+          match = message.match(/data:\s*(\{.*\})/)
+          match ? JSON.parse(match[1]) : nil
+        end.compact
+      end
+      def extract_data_entries(chunk)
+        chunk.split(/^data: /).map(&:strip).reject(&:empty?)
+      end
+      def process_streaming_response(chunk)
+        data_entries = extract_data_entries(chunk)
+        data_entries.each do |entry|
+          data, = entry.split("\n", 2)
+          next if data == "id: 1"
+          data = parse_json(data)
+          raise StandardError, "Error: #{data["errors"][0]["message"]}" if data["errors"] && data["errors"][0]["message"]
+          next if data["results"][0]["stop_reason"] != "not_finished"
+          yield [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/watsonx_ai/config/catalog.json ADDED Viewed

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 120,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/multiwoven/integrations/source/watsonx_ai/config/meta.json ADDED Viewed

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "WatsonxAi",
+    "title": "WatsonX AI Model Endpoint",
+    "connector_type": "source",
+    "category": "AI Model",
+    "documentation_url": "https://docs.mutltiwoven.com",
+    "github_issue_label": "source-watsonx-ai-model",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/watsonx_ai/config/spec.json ADDED Viewed

@@ -0,0 +1,74 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/source/watsonx-ai-endpoint",
+  "stream_type": "user_defined",
+  "connector_query_type": "ai_ml",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "WatsonX AI Endpoint",
+    "type": "object",
+    "required": ["api_key", "region", "deployment_id", "request_format", "response_format"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "API Key",
+        "order": 0
+      },
+      "is_stream": {
+        "type": "boolean",
+        "title": "Enable streaming",
+        "description": "Enables data streaming for such as chat, when supported by the model. When true, messages and model data are processed in chunks for immediate delivery, enhancing responsiveness. Default is false, processing only after the entire response is received.",
+        "default": false,
+        "order": 1
+      },
+      "region": {
+        "description": "WatsonX AI region",
+        "type": "string",
+        "title": "Region",
+        "order": 2
+      },
+      "deployment_id": {
+        "description": "WatsonX AI online deployment id",
+        "type": "string",
+        "title": "Deployment ID",
+        "multiwoven_secret": true,
+        "order": 3
+      },
+      "config": {
+        "title": "",
+        "type": "object",
+        "properties": {
+          "timeout": {
+            "type": "string",
+            "default": "30",
+            "title": "HTTP Timeout",
+            "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
+            "order": 0
+          }
+        },
+        "order": 4
+      },
+      "request_format": {
+        "title": "Request Format",
+        "description": "Sample request format",
+        "type": "string",
+        "order": 5
+      },
+      "response_format": {
+        "title": "Response Format",
+        "description": "Sample response format",
+        "type": "string",
+        "x-response-format": true,
+        "order": 6
+      },
+      "model_type": {
+        "title": "Model Type",
+        "type": "string",
+        "description": "Deployed model type.",
+        "enum": ["Machine learning model", "Prompt template"],
+        "default": "Machine learning model",
+        "order": 7
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/watsonx_ai/icon.svg ADDED Viewed

@@ -0,0 +1 @@

+ <svg id="Watsonx-Data--Streamline-Carbon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" height="16" width="16"><desc>Watsonx Data Streamline Icon: https://streamlinehq.com</desc><defs></defs><path d="M26 24c-1.1046 0-2 .8954-2 2 0 .0764.0142.1488.0225.2229C21.7417 28.0192 18.9433 29 16 29c-2.7746 0-5.3432-.881-7.4566-2.3676.2576.0261.517.0444.7798.0444C13.5561 26.6768 17 23.233 17 19h-2c0 3.1304-2.5464 5.6768-5.6768 5.6768-2.2111 0-4.1977-1.2816-5.1318-3.2725-.1365-.2972-.2595-.6007-.3738-.9094C4.4778 20.8169 5.2174 21 6 21c2.7568 0 5-2.2432 5-5v-2H9v2c0 1.6543-1.3457 3-3 3s-3-1.3457-3-3c0-2.1152.4917-4.1328 1.4619-5.9956l-1.7744-.9238C1.5835 11.2017 1 13.5943 1 16c0 8.271 6.729 15 15 15 3.3744 0 6.5818-1.1193 9.2048-3.1662.244.106.5123.1662.7952.1662 1.1046 0 2-.8954 2-2s-.8954-2-2-2Z" fill="#000000"></path><path transform="rotate(90 22 22)" d="M21 21h2v2h-2Z" fill="#000000"></path><path transform="rotate(90 16 16)" d="M15 15h2v2h-2Z" fill="#000000"></path><path transform="rotate(-90 10 10)" d="M9 9h2v2H9Z" fill="#000000"></path><path d="M16 1c-3.3744 0-6.5818 1.1193-9.2048 3.1662C6.5512 4.0602 6.2829 4 6 4c-1.1046 0-2 .8954-2 2s.8954 2 2 2 2-.8954 2-2c0-.0764-.0142-.1488-.0225-.2229C10.2583 3.9808 13.0567 3 16 3c2.7708 0 5.3363.8784 7.4481 2.3613-.249-.0242-.5005-.038-.7547-.038-4.2329 0-7.6768 3.4438-7.6768 7.6768h2c0-3.1304 2.5464-5.6768 5.6768-5.6768 2.0554 0 3.9068 1.0953 4.9186 2.8651.2153.4283.4053.8701.5729 1.3237C27.5234 11.1887 26.7844 11 26 11c-2.7568 0-5 2.2432-5 5v2h2v-2c0-1.6543 1.3457-3 3-3s3 1.3457 3 3c0 2.1152-.4917 4.1328-1.4619 5.9956l1.7744.9238C30.4165 20.7983 31 18.4057 31 16c0-8.271-6.729-15-15-15Z" fill="#000000"></path><path id="_Transparent_Rectangle_" d="M0 0h32v32H0Z" fill="none"></path></svg>

data/lib/multiwoven/integrations.rb CHANGED Viewed

@@ -75,6 +75,7 @@ require_relative "integrations/source/google_vertex_model/client"
 require_relative "integrations/source/http_model/client"
 require_relative "integrations/source/open_ai/client"
 require_relative "integrations/source/sftp/client"
+require_relative "integrations/source/watsonx_ai/client"
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.19.3
+  version: 0.20.0
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-12 00:00:00.000000000 Z
+date: 2025-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -776,6 +776,11 @@ files:
 - lib/multiwoven/integrations/source/snowflake/config/meta.json
 - lib/multiwoven/integrations/source/snowflake/config/spec.json
 - lib/multiwoven/integrations/source/snowflake/icon.svg
+- lib/multiwoven/integrations/source/watsonx_ai/client.rb
+- lib/multiwoven/integrations/source/watsonx_ai/config/catalog.json
+- lib/multiwoven/integrations/source/watsonx_ai/config/meta.json
+- lib/multiwoven/integrations/source/watsonx_ai/config/spec.json
+- lib/multiwoven/integrations/source/watsonx_ai/icon.svg
 - multiwoven-integrations.gemspec
 - sig/multiwoven/integrations.rbs
 homepage: https://www.multiwoven.com/