outhad-integrations 0.32.0

data/lib/outhad/integrations/source/sftp/config/spec.json

@@ -0,0 +1,59 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sftp",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "SFTP",
+    "required": ["host", "username", "password", "file_path", "file_name", "format_type" ],
+    "properties": {
+      "host": {
+        "title": "Host",
+        "description": "Hostname of the SFTP server.",
+        "type": "string",
+        "order": 0
+      },
+      "port": {
+        "title": "Port",
+        "description": "Port of the SFTP server.",
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 65536,
+        "default": 22,
+        "order": 1
+      },
+      "username": {
+        "title": "User Name",
+        "description": "Username to use to access the SFTP server.",
+        "type": "string",
+        "order": 2
+      },
+      "password": {
+        "title": "Password",
+        "description": "Password associated with the username.",
+        "type": "string",
+        "outhad_secret": true,
+        "order": 3
+      },
+      "file_path": {
+        "title": "File path",
+        "type": "string",
+        "description": "Path to the directory where file is stored.",
+        "order": 4
+      },
+      "file_name": {
+        "title": "File Name",
+        "type": "string",
+        "description": "Name of the file to be written.",
+        "order": 5
+      },
+      "format_type": {
+        "title": "File Format Type",
+        "type": "string",
+        "description": "Format of the data output.",
+        "enum": ["csv"],
+        "default": "csv",
+        "order": 6
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/sftp/icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="250" height="250" fill="none"><g clip-path="url(#a)"><path fill="#615EFF" d="M200.745 218.706a12.051 12.051 0 0 1-3.483 8.475 11.837 11.837 0 0 1-8.4 3.514H62.072c-3.15 0-6.173-1.261-8.403-3.507a12.053 12.053 0 0 1-3.49-8.472V31.485a12.08 12.08 0 0 1 .903-4.593 11.996 11.996 0 0 1 2.578-3.894 11.875 11.875 0 0 1 3.86-2.6 11.791 11.791 0 0 1 4.552-.912h81.371l57.302 58.96v140.26Z"/><path fill="#fff" d="M83.868 156.832c-.142-1.429-.75-2.539-1.822-3.33-1.074-.792-2.53-1.187-4.37-1.187-1.25 0-2.305.177-3.166.531-.861.343-1.522.821-1.981 1.435a3.457 3.457 0 0 0-.673 2.09c-.023.65.112 1.217.407 1.701.307.484.725.903 1.256 1.258.53.342 1.144.643 1.84.903a19.02 19.02 0 0 0 2.229.638l3.255.779c1.58.354 3.03.827 4.352 1.417 1.32.591 2.464 1.317 3.431 2.179a9.057 9.057 0 0 1 2.247 3.047c.542 1.169.82 2.509.831 4.021-.011 2.22-.578 4.145-1.698 5.775-1.109 1.617-2.712 2.875-4.811 3.773-2.088.885-4.606 1.328-7.554 1.328-2.925 0-5.472-.449-7.642-1.346-2.158-.898-3.845-2.226-5.06-3.986-1.202-1.771-1.833-3.962-1.892-6.572h7.412c.082 1.217.43 2.232 1.044 3.047.625.803 1.456 1.411 2.494 1.825 1.05.401 2.235.602 3.555.602 1.298 0 2.424-.189 3.38-.567.966-.378 1.715-.903 2.246-1.576.53-.673.796-1.447.796-2.321 0-.815-.242-1.5-.725-2.055-.472-.555-1.168-1.027-2.088-1.417-.908-.39-2.022-.744-3.343-1.063l-3.945-.992c-3.054-.744-5.466-1.907-7.235-3.489-1.77-1.583-2.648-3.714-2.636-6.395-.012-2.196.572-4.116 1.751-5.757 1.192-1.641 2.825-2.923 4.9-3.844 2.076-.921 4.435-1.382 7.076-1.382 2.69 0 5.036.461 7.04 1.382 2.018.921 3.586 2.203 4.706 3.844 1.12 1.641 1.699 3.543 1.734 5.704h-7.341Zm12.988 25.844v-36.278h23.988v6.324h-16.328v8.645h14.736v6.324h-14.736v14.985h-7.66Zm27.76-29.954v-6.324h29.754v6.324h-11.091v29.954h-7.571v-29.954h-11.092Zm34.654 29.954v-36.278h14.294c2.748 0 5.089.526 7.023 1.577 1.934 1.039 3.408 2.486 4.422 4.34 1.026 1.842 1.539 3.968 1.539 6.377s-.519 4.535-1.557 6.377c-1.037 1.842-2.541 3.277-4.51 4.304-1.958 1.028-4.329 1.541-7.112 1.541h-9.11v-6.146h7.872c1.474 0 2.689-.254 3.644-.762.967-.52 1.686-1.234 2.158-2.143.484-.921.725-1.978.725-3.171 0-1.205-.241-2.256-.725-3.153-.472-.91-1.191-1.612-2.158-2.108-.967-.508-2.194-.762-3.679-.762h-5.166v30.007h-7.66Z"/><path fill="#1A194D" d="m149.75 76.907 50.996 49.177V78.101h-45.173a11.33 11.33 0 0 1-5.643-1.433l-.18.24Z" opacity=".3"/><path fill="#C5C4FF" d="M200.746 78.35h-45.297a11.836 11.836 0 0 1-8.399-3.514 12.055 12.055 0 0 1-3.484-8.475v-47.2l57.18 59.189Z"/></g><defs><clipPath id="a"><path fill="#fff" d="M50 19h151v212H50z"/></clipPath></defs></svg>

data/lib/outhad/integrations/source/snowflake/client.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module Snowflake
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_outhad_message
+      rescue Sequel::DatabaseConnectionError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                FROM information_schema.columns
+                WHERE table_schema = \'#{connection_config[:schema]}\' AND table_catalog = \'#{connection_config[:database]}\'
+                ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+
+        records = []
+        db.fetch(query.gsub("\n", "")) do |row|
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SNOWFLAKE:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        db = create_connection(connection_config)
+
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "SNOWFLAKE:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def query(connection, query)
+        records = []
+        connection.fetch(query) do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+        end
+        records
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" if connection_config[:credentials][:auth_type] != "username/password"
+
+        Sequel.odbc(drvconnect: generate_drvconnect(connection_config))
+      end
+
+      def generate_drvconnect(connection_config)
+        c = connection_config[:credentials]
+        "driver=#{SNOWFLAKE_DRIVER_PATH};server=#{connection_config[:host]};uid=#{c[:username]};pwd=#{c[:password]};schema=#{connection_config[:schema]};database=#{connection_config[:database]};warehouse=#{connection_config[:warehouse]};"
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/snowflake/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "Snowflake",
+    "title": "Snowflake",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/snowflake",
+    "github_issue_label": "source-snowflake",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/snowflake/config/spec.json

@@ -0,0 +1,82 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/snowflake",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Snowflake",
+    "type": "object",
+    "required": ["host", "role", "warehouse", "database"],
+    "properties": {
+      "credentials": {
+        "title": "",
+        "type": "object",
+        "required": ["auth_type", "username", "password"],
+        "properties": {
+          "auth_type": {
+            "type": "string",
+            "default": "username/password",
+            "order": 0,
+            "readOnly": true
+          },
+          "username": {
+            "description": "The username you created to allow outhad to access the database.",
+            "examples": ["OUTHAD_USER"],
+            "type": "string",
+            "title": "Username",
+            "order": 1
+          },
+          "password": {
+            "description": "The password associated with the username.",
+            "type": "string",
+            "outhad_secret": true,
+            "title": "Password",
+            "order": 2
+          }
+        },
+        "order": 0
+      },
+      "host": {
+        "description": "The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com).",
+        "examples": ["accountname.us-east-2.aws.snowflakecomputing.com"],
+        "type": "string",
+        "title": "Account Name",
+        "order": 1
+      },
+      "role": {
+        "description": "The role you created for outhad to access Snowflake.",
+        "examples": ["OUTHAD_ROLE"],
+        "type": "string",
+        "title": "Role",
+        "order": 2
+      },
+      "warehouse": {
+        "description": "The warehouse you created for outhad to access data.",
+        "examples": ["OUTHAD_WAREHOUSE"],
+        "type": "string",
+        "title": "Warehouse",
+        "order": 3
+      },
+      "database": {
+        "description": "The database you created for outhad to access data.",
+        "examples": ["OUTHAD_DATABASE"],
+        "type": "string",
+        "title": "Database",
+        "order": 4
+      },
+      "schema": {
+        "description": "The source Snowflake schema tables. Leave empty to access tables from multiple schemas.",
+        "examples": ["OUTHAD_SCHEMA"],
+        "type": "string",
+        "title": "Schema",
+        "order": 5
+      },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "title": "JDBC URL Params",
+        "type": "string",
+        "order": 6
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/snowflake/icon.svg

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg viewBox="0 0 44 44" stroke="#249edc" fill="#249edc" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <path d="M37.2637465,33.128906 L28.0879655,27.828125 C26.7989025,27.085938 25.1504655,27.527344 24.4043715,28.816406 C24.1153085,29.324219 24.0020275,29.882812 24.0567155,30.425781 L24.0567155,40.785156 C24.0567155,42.265625 25.2598395,43.46875 26.7442155,43.46875 C28.2246835,43.46875 29.4278085,42.265625 29.4278085,40.785156 L29.4278085,34.828125 L34.5684335,37.796875 C35.8574965,38.542969 37.5098395,38.097656 38.2520275,36.808594 C38.9981215,35.519531 38.5567155,33.871094 37.2637465,33.128906" id="Path"></path>
+    <path d="M14.4434335,21.769531 C14.4590585,20.8125 13.9551525,19.921875 13.1270275,19.441406 L3.95124649,14.144531 C3.55280849,13.914062 3.09577749,13.792969 2.63874649,13.792969 C1.69733949,13.792969 0.822339495,14.296875 0.353589495,15.109375 C-0.372972505,16.367188 0.060621495,17.980469 1.31843349,18.707031 L6.60749649,21.757812 L1.31843349,24.8125 C0.709058495,25.164062 0.271558495,25.730469 0.091871495,26.410156 C-0.091722505,27.089844 0.00202749496,27.800781 0.353589495,28.410156 C0.822339495,29.222656 1.69733949,29.726562 2.63483949,29.726562 C3.09577749,29.726562 3.55280849,29.605469 3.95124649,29.375 L13.1270275,24.078125 C13.9473395,23.601562 14.4512465,22.71875 14.4434335,21.769531" id="Path2"></path>
+    <path d="M6.03327749,10.390625 L15.2090585,15.6875 C16.2793715,16.308594 17.5996835,16.105469 18.4434335,15.28125 C18.9785895,14.789062 19.3106215,14.085938 19.3106215,13.304688 L19.3106215,2.6875 C19.3106215,1.203125 18.1074965,0 16.6270275,0 C15.1426525,0 13.9395275,1.203125 13.9395275,2.6875 L13.9395275,8.730469 L8.72858949,5.722656 C7.43952749,4.976562 5.79108949,5.417969 5.04499649,6.707031 C4.29890249,7.996094 4.74421549,9.644531 6.03327749,10.390625" id="Path3"></path>
+    <path d="M26.6660895,22.199219 C26.6660895,22.402344 26.5489025,22.683594 26.4043715,22.832031 L22.7676525,26.46875 C22.6231215,26.613281 22.3379655,26.730469 22.1348395,26.730469 L21.2090585,26.730469 C21.0059335,26.730469 20.7207775,26.613281 20.5762465,26.46875 L16.9356215,22.832031 C16.7910895,22.683594 16.6739025,22.402344 16.6739025,22.199219 L16.6739025,21.273438 C16.6739025,21.066406 16.7910895,20.785156 16.9356215,20.640625 L20.5762465,17 C20.7207775,16.855469 21.0059335,16.738281 21.2090585,16.738281 L22.1348395,16.738281 C22.3379655,16.738281 22.6231215,16.855469 22.7676525,17 L26.4043715,20.640625 C26.5489025,20.785156 26.6660895,21.066406 26.6660895,21.273438 L26.6660895,22.199219 Z M23.4199965,21.753906 L23.4199965,21.714844 C23.4199965,21.566406 23.3340585,21.359375 23.2285895,21.25 L22.1543715,20.179688 C22.0489025,20.070312 21.8418715,19.984375 21.6895275,19.984375 L21.6504655,19.984375 C21.5020275,19.984375 21.2949965,20.070312 21.1856215,20.179688 L20.1153085,21.25 C20.0098395,21.355469 19.9239025,21.5625 19.9239025,21.714844 L19.9239025,21.753906 C19.9239025,21.90625 20.0098395,22.113281 20.1153085,22.21875 L21.1856215,23.292969 C21.2949965,23.398438 21.5020275,23.484375 21.6504655,23.484375 L21.6895275,23.484375 C21.8418715,23.484375 22.0489025,23.398438 22.1543715,23.292969 L23.2285895,22.21875 C23.3340585,22.113281 23.4199965,21.90625 23.4199965,21.753906 Z" id="Combined-Shape"></path>
+    <path d="M28.0879655,15.6875 L37.2637465,10.390625 C38.5528085,9.648438 38.9981215,7.996094 38.2520275,6.707031 C37.5059335,5.417969 35.8574965,4.976562 34.5684335,5.722656 L29.4278085,8.691406 L29.4278085,2.6875 C29.4278085,1.203125 28.2246835,-5.68434189e-14 26.7442155,-5.68434189e-14 C25.2598395,-5.68434189e-14 24.0567155,1.203125 24.0567155,2.6875 L24.0567155,13.09375 C24.0059335,13.632812 24.1114025,14.195312 24.4043715,14.703125 C25.1504655,15.992188 26.7989025,16.433594 28.0879655,15.6875" id="Path4"></path>
+    <path d="M17.0489025,27.515625 C16.4395275,27.398438 15.7871835,27.496094 15.2090585,27.828125 L6.03327749,33.128906 C4.74421549,33.871094 4.29890249,35.519531 5.04499649,36.808594 C5.79108949,38.101562 7.43952749,38.542969 8.72858949,37.796875 L13.9395275,34.789062 L13.9395275,40.785156 C13.9395275,42.265625 15.1426525,43.46875 16.6270275,43.46875 C18.1074965,43.46875 19.3106215,42.265625 19.3106215,40.785156 L19.3106215,30.167969 C19.3106215,28.828125 18.3301525,27.71875 17.0489025,27.515625" id="Path5"></path>
+    <path d="M42.9981215,15.078125 C42.2559335,13.785156 40.6035895,13.34375 39.3145275,14.089844 L30.1387465,19.386719 C29.2598395,19.894531 28.7754655,20.824219 28.7910895,21.769531 C28.7832775,22.710938 29.2676525,23.628906 30.1387465,24.128906 L39.3145275,29.429688 C40.6035895,30.171875 42.2520275,29.730469 42.9981215,28.441406 C43.7442155,27.152344 43.2989025,25.503906 42.0098395,24.757812 L36.8145275,21.757812 L42.0098395,18.757812 C43.3028085,18.015625 43.7442155,16.367188 42.9981215,15.078125" id="Path6"></path>
+</svg>

data/lib/outhad/integrations/source/watsonx_ai/client.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module WatsonxAi
+    include Outhad::Integrations::Core
+    API_VERSION = "2021-05-01"
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        get_access_token(connection_config[:api_key])
+        url = format(
+          WATSONX_HEALTH_DEPLOYMENT_URL,
+          region: connection_config[:region],
+          version: API_VERSION
+        )
+        response = send_request(
+          url: url,
+          http_method: HTTP_GET,
+          payload: {},
+          headers: auth_headers(@access_token),
+          config: connection_config[:config]
+        )
+        evaluate_deployment_status(response, connection_config[:deployment_id])
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(_connection_config)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config, payload = prepare_config_and_payload(sync_config)
+        process_model_request(connection_config, payload) { |message| yield message if block_given? }
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:READ:EXCEPTION", type: "error" })
+      end
+
+      private
+
+      def process_model_request(connection_config, payload)
+        if connection_config[:is_stream] && connection_config[:model_type] == "Prompt template"
+          run_model_stream(connection_config, payload) { |message| yield message if block_given? }
+        else
+          run_model(connection_config, payload)
+        end
+      end
+
+      def evaluate_deployment_status(response, deployment_id)
+        response_body = JSON.parse(response.body)
+        deployment_status = response_body["resources"]&.find { |res| res.dig("metadata", "id") == deployment_id }
+
+        return failure_status unless deployment_status
+
+        deployment_status.dig("entity", "status", "state") == "ready" ? success_status : failure_status
+      end
+
+      def prepare_config_and_payload(sync_config)
+        config = sync_config.source.connection_specification
+        connection_config = config.with_indifferent_access.tap do |conf|
+          conf[:config][:timeout] ||= 30
+          conf[:is_stream] ||= false
+        end
+        payload = sync_config.model.query
+        [connection_config, payload]
+      end
+
+      def get_access_token(api_key)
+        cache = defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : ActiveSupport::Cache::MemoryStore.new
+        cache_key = "watsonx_ai_#{api_key}"
+        cached_token = cache.read(cache_key)
+        if cached_token
+          @access_token = cached_token
+        else
+          new_token = get_iam_token(api_key)
+          # max expiration is 3 minutes. No way to make it higher
+          cache.write(cache_key, new_token, expires_in: 180)
+          @access_token = new_token
+        end
+      end
+
+      def get_iam_token(api_key)
+        uri = URI("https://iam.cloud.ibm.com/identity/token")
+        request = Net::HTTP::Post.new(uri)
+        request["Content-Type"] = "application/x-www-form-urlencoded"
+        request.body = "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=#{api_key}"
+        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+
+        raise "Failed to get IAM token: #{response.body}" unless response.is_a?(Net::HTTPSuccess)
+
+        JSON.parse(response.body)["access_token"]
+      end
+
+      def parse_json(json_string)
+        JSON.parse(json_string)
+      rescue JSON::ParserError => e
+        handle_exception(e, { context: "OPEN AI:PARSE_JSON:EXCEPTION", type: "error" })
+        {}
+      end
+
+      def run_model(connection_config, payload)
+        get_access_token(connection_config[:api_key])
+        url = format(
+          connection_config[:model_type] == "Machine learning model" ? WATSONX_PREDICTION_DEPLOYMENT_URL : WATSONX_GENERATION_DEPLOYMENT_URL,
+          region: connection_config[:region],
+          deployment_id: connection_config[:deployment_id],
+          version: API_VERSION
+        )
+        response = send_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: JSON.parse(payload),
+          headers: auth_headers(@access_token),
+          config: connection_config[:config]
+        )
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:RUN_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def process_response(response)
+        if success?(response)
+          if response.body.start_with?("{") || response.body.start_with?("[")
+            data = JSON.parse(response.body)
+            [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_outhad_message]
+          else
+            data = format_data(response.body)
+            RecordMessage.new(data: { responses: data }, emitted_at: Time.now.to_i).to_outhad_message
+          end
+        else
+          create_log_message("WATSONX AI:RUN_MODEL", "error", "request failed: #{response.body}")
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:PROCESS_RESPONSE:EXCEPTION", type: "error" })
+      end
+
+      def run_model_stream(connection_config, payload)
+        get_access_token(connection_config[:api_key])
+        url = format(
+          WATSONX_STREAM_DEPLOYMENT_URL,
+          region: connection_config[:region],
+          deployment_id: connection_config[:deployment_id],
+          version: API_VERSION
+        )
+        send_streaming_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: JSON.parse(payload),
+          headers: auth_headers(@access_token),
+          config: connection_config[:config]
+        ) do |chunk|
+          process_streaming_response(chunk) { |message| yield message if block_given? }
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX AI:RUN_STREAM_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def format_data(response_body)
+        messages = response_body.split("\n\n")
+        messages.map do |message|
+          match = message.match(/data:\s*(\{.*\})/)
+          match ? JSON.parse(match[1]) : nil
+        end.compact
+      end
+
+      def extract_data_entries(chunk)
+        chunk.split(/^data: /).map(&:strip).reject(&:empty?)
+      end
+
+      def process_streaming_response(chunk)
+        data_entries = extract_data_entries(chunk)
+        data_entries.each do |entry|
+          data, = entry.split("\n", 2)
+
+          next if data == "id: 1"
+
+          data = parse_json(data)
+
+          raise StandardError, "Error: #{data["errors"][0]["message"]}" if data["errors"] && data["errors"][0]["message"]
+
+          next if data["results"][0]["stop_reason"] != "not_finished"
+
+          yield [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_outhad_message] if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/watsonx_ai/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 120,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/outhad/integrations/source/watsonx_ai/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "WatsonxAi",
+    "title": "WatsonX AI Model Endpoint",
+    "connector_type": "source",
+    "category": "AI Model",
+    "sub_category": "AI_ML Service",
+    "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/watsonx_ai-model",
+    "github_issue_label": "source-watsonx-ai-model",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/watsonx_ai/config/spec.json

@@ -0,0 +1,74 @@
+{
+  "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/watsonx_ai-model",
+  "stream_type": "user_defined",
+  "connector_query_type": "ai_ml",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "WatsonX AI Endpoint",
+    "type": "object",
+    "required": ["api_key", "region", "deployment_id", "request_format", "response_format"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "outhad_secret": true,
+        "title": "API Key",
+        "order": 0
+      },
+      "is_stream": {
+        "type": "boolean",
+        "title": "Enable streaming",
+        "description": "Enables data streaming for such as chat, when supported by the model. When true, messages and model data are processed in chunks for immediate delivery, enhancing responsiveness. Default is false, processing only after the entire response is received.",
+        "default": false,
+        "order": 1
+      },
+      "region": {
+        "description": "WatsonX AI region",
+        "type": "string",
+        "title": "Region",
+        "order": 2
+      },
+      "deployment_id": {
+        "description": "WatsonX AI online deployment id",
+        "type": "string",
+        "title": "Deployment ID",
+        "outhad_secret": true,
+        "order": 3
+      },
+      "config": {
+        "title": "",
+        "type": "object",
+        "properties": {
+          "timeout": {
+            "type": "string",
+            "default": "30",
+            "title": "HTTP Timeout",
+            "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
+            "order": 0
+          }
+        },
+        "order": 4
+      },
+      "request_format": {
+        "title": "Request Format",
+        "description": "Sample request format",
+        "type": "string",
+        "order": 5
+      },
+      "response_format": {
+        "title": "Response Format",
+        "description": "Sample response format",
+        "type": "string",
+        "x-response-format": true,
+        "order": 6
+      },
+      "model_type": {
+        "title": "Model Type",
+        "type": "string",
+        "description": "Deployed model type.",
+        "enum": ["Machine learning model", "Prompt template"],
+        "default": "Machine learning model",
+        "order": 7
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/watsonx_ai/icon.svg

@@ -0,0 +1 @@
+<svg id="Watsonx-Data--Streamline-Carbon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" height="16" width="16"><desc>Watsonx Data Streamline Icon: https://streamlinehq.com</desc><defs></defs><path d="M26 24c-1.1046 0-2 .8954-2 2 0 .0764.0142.1488.0225.2229C21.7417 28.0192 18.9433 29 16 29c-2.7746 0-5.3432-.881-7.4566-2.3676.2576.0261.517.0444.7798.0444C13.5561 26.6768 17 23.233 17 19h-2c0 3.1304-2.5464 5.6768-5.6768 5.6768-2.2111 0-4.1977-1.2816-5.1318-3.2725-.1365-.2972-.2595-.6007-.3738-.9094C4.4778 20.8169 5.2174 21 6 21c2.7568 0 5-2.2432 5-5v-2H9v2c0 1.6543-1.3457 3-3 3s-3-1.3457-3-3c0-2.1152.4917-4.1328 1.4619-5.9956l-1.7744-.9238C1.5835 11.2017 1 13.5943 1 16c0 8.271 6.729 15 15 15 3.3744 0 6.5818-1.1193 9.2048-3.1662.244.106.5123.1662.7952.1662 1.1046 0 2-.8954 2-2s-.8954-2-2-2Z" fill="#000000"></path><path transform="rotate(90 22 22)" d="M21 21h2v2h-2Z" fill="#000000"></path><path transform="rotate(90 16 16)" d="M15 15h2v2h-2Z" fill="#000000"></path><path transform="rotate(-90 10 10)" d="M9 9h2v2H9Z" fill="#000000"></path><path d="M16 1c-3.3744 0-6.5818 1.1193-9.2048 3.1662C6.5512 4.0602 6.2829 4 6 4c-1.1046 0-2 .8954-2 2s.8954 2 2 2 2-.8954 2-2c0-.0764-.0142-.1488-.0225-.2229C10.2583 3.9808 13.0567 3 16 3c2.7708 0 5.3363.8784 7.4481 2.3613-.249-.0242-.5005-.038-.7547-.038-4.2329 0-7.6768 3.4438-7.6768 7.6768h2c0-3.1304 2.5464-5.6768 5.6768-5.6768 2.0554 0 3.9068 1.0953 4.9186 2.8651.2153.4283.4053.8701.5729 1.3237C27.5234 11.1887 26.7844 11 26 11c-2.7568 0-5 2.2432-5 5v2h2v-2c0-1.6543 1.3457-3 3-3s3 1.3457 3 3c0 2.1152-.4917 4.1328-1.4619 5.9956l1.7744.9238C30.4165 20.7983 31 18.4057 31 16c0-8.271-6.729-15-15-15Z" fill="#000000"></path><path id="_Transparent_Rectangle_" d="M0 0h32v32H0Z" fill="none"></path></svg>