outhad-integrations 0.32.0

data/lib/outhad/integrations/source/clickhouse/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "Clickhouse",
+    "title": "ClickHouse",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/clickhouse",
+    "github_issue_label": "source-clickhouse",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/clickhouse/config/spec.json

@@ -0,0 +1,42 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/clickhouse",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "ClickHouse",
+    "type": "object",
+    "required": ["url", "username", "password", "database"],
+    "properties": {
+      "url": {
+        "description": "The ClickHouse host url to connect.",
+        "examples": ["tu61szglca.us-west-2.aws.clickhouse.cloud"],
+        "type": "string",
+        "title": "Personal URL",
+        "order": 0
+      },
+      "username": {
+        "description": "The username for ClickHouse.",
+        "examples": ["Default"],
+        "type": "string",
+        "title": "Username",
+        "order": 1
+      },
+      "password": {
+        "description": "The password for ClickHouse.",
+        "examples": ["Default"],
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Password",
+        "order": 2
+      },
+      "database": {
+        "description": "The ClickHouse database.",
+        "examples": ["default"],
+        "type": "string",
+        "title": "Database",
+        "order": 3
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/clickhouse/icon.svg

@@ -0,0 +1,25 @@
+<svg height="100" width="100" version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 50.6 50.6" style="enable-background:new 0 0 50.6 50.6;" xml:space="preserve">
+ <metadata>
+  <sfw xmlns="ns_sfw;">
+   <slices>
+   </slices>
+      <sliceSourceBounds bottomLeftOrigin="true" height="50.6" width="50.6" x="0" y="0">
+   </sliceSourceBounds>
+  </sfw>
+ </metadata>
+    <g>
+  <g>
+   <path d="M0.6,0H5c0.3,0,0.6,0.3,0.6,0.6V50c0,0.3-0.3,0.6-0.6,0.6H0.6C0.3,50.6,0,50.4,0,50V0.6C0,0.3,0.3,0,0.6,0z" fill="yellow">
+   </path>
+      <path d="M11.8,0h4.4c0.3,0,0.6,0.3,0.6,0.6V50c0,0.3-0.3,0.6-0.6,0.6h-4.4c-0.3,0-0.6-0.3-0.6-0.6V0.6C11.3,0.3,11.5,0,11.8,0z" fill="yellow">
+   </path>
+      <path d="M23.1,0h4.4c0.3,0,0.6,0.3,0.6,0.6V50c0,0.3-0.3,0.6-0.6,0.6h-4.4c-0.3,0-0.6-0.3-0.6-0.6V0.6C22.5,0.3,22.8,0,23.1,0z" fill="yellow">
+   </path>
+      <path d="M34.3,0h4.4c0.3,0,0.6,0.3,0.6,0.6V50c0,0.3-0.3,0.6-0.6,0.6h-4.4c-0.3,0-0.6-0.3-0.6-0.6V0.6C33.7,0.3,34,0,34.3,0z" fill="yellow">
+   </path>
+      <path d="M45.6,19.7H50c0.3,0,0.6,0.3,0.6,0.6v10.1c0,0.3-0.3,0.6-0.6,0.6h-4.4c-0.3,0-0.6-0.3-0.6-0.6V20.3
+            C45,20,45.3,19.7,45.6,19.7z" fill="yellow">
+   </path>
+  </g>
+ </g>
+</svg>

data/lib/outhad/integrations/source/databricks/client.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module Databricks
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_outhad_message
+      rescue Sequel::DatabaseConnectionError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                FROM system.information_schema.columns
+                WHERE table_schema = \'#{connection_config[:schema]}\' AND table_catalog = \'#{connection_config[:catalog]}\'
+                ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+
+        records = []
+        db.fetch(query.gsub("\n", "")) do |row|
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "DATABRICKS:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        db = create_connection(connection_config)
+
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "DATABRICKS:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def query(connection, query)
+        records = []
+        connection.fetch(query) do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+        end
+        records
+      end
+
+      def create_connection(connection_config)
+        Sequel.odbc(drvconnect: generate_drvconnect(connection_config))
+      end
+
+      def generate_drvconnect(connection_config)
+        "Driver=#{DATABRICKS_DRIVER_PATH};
+        Host=#{connection_config[:host]};
+        PORT=#{connection_config[:port]};
+        SSL=1;
+        HTTPPath=#{connection_config[:http_path]};
+        PWD=#{connection_config[:access_token]};
+        UID=token;
+        ThriftTransport=2;AuthMech=3;
+        AllowSelfSignedServerCert=1;
+        CAIssuedCertNamesMismatch=1"
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/databricks/config/meta.json

@@ -0,0 +1,17 @@
+{
+    "data": {
+      "name": "Databricks",
+      "title": "Databricks Data Warehouse",
+      "connector_type": "source",
+      "category": "Data Warehouse",
+      "sub_category": "Relational Database",
+      "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/databricks",
+      "github_issue_label": "source-databricks",
+      "icon": "icon.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "outhad"]
+    }
+  }
+  

data/lib/outhad/integrations/source/databricks/config/spec.json

@@ -0,0 +1,56 @@
+{
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/databricks",
+    "stream_type": "dynamic",
+    "connector_query_type": "raw_sql",
+    "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "Databricks Datawarehouse",
+      "type": "object",
+      "required": ["host", "port", "http_path", "catalog", "schema"],
+      "properties": {
+        "host": {
+          "title": "Server Hostname",
+          "description": "Server host name for the Databricks Cluster. It is different from the SQL Endpoint Cluster.",
+          "type": "string",
+          "examples": ["abc-12345678-wxyz.cloud.databricks.com"],
+          "order": 0
+        },
+        "port": {
+          "title": "Port",
+          "description": "",
+          "type": "string",
+          "default": "443",
+          "order": 1
+        },
+        "access_token": {
+          "title": "Personal Access Token",
+          "description": "",
+          "type": "string",
+          "outhad_secret": true,
+          "order": 2
+        },
+        "http_path": {
+          "title": "HTTP Path",
+          "description": "",
+          "examples": ["sql/protocolvx/o/1234567489/0000-1111111-abcd90"],
+          "type": "string",
+          "order": 3
+        },
+        "catalog": {
+          "description": "The name of the catalog",
+          "default": "hive_metastore",
+          "type": "string",
+          "title": "Databricks catalog",
+          "order": 4
+        },
+        "schema": {
+          "description": "The default schema tables are written.",
+          "default": "default",
+          "type": "string",
+          "title": "Database schema",
+          "order": 5
+        }
+      }
+    }
+  }
+  

data/lib/outhad/integrations/source/databricks/icon.svg

@@ -0,0 +1,19 @@
+<svg version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 40.1 42" style="enable-background:new 0 0 40.1 42;" xml:space="preserve">
+ <style type="text/css">
+  .st0{fill:#FF3621;}
+ </style>
+ <metadata>
+  <sfw xmlns="ns_sfw;">
+   <slices>
+   </slices>
+   <sliceSourceBounds bottomLeftOrigin="true" height="42" width="40.1" x="-69.1" y="-10.5">
+   </sliceSourceBounds>
+  </sfw>
+ </metadata>
+ <g>
+  <path class="st0" d="M40.1,31.1v-7.4l-0.8-0.5L20.1,33.7l-18.2-10l0-4.3l18.2,9.9l20.1-10.9v-7.3l-0.8-0.5L20.1,21.2L2.6,11.6
+		L20.1,2l14.1,7.7l1.1-0.6V8.3L20.1,0L0,10.9V12L20.1,23l18.2-10v4.4l-18.2,10L0.8,16.8L0,17.3v7.4l20.1,10.9l18.2-9.9v4.3l-18.2,10
+		L0.8,29.5L0,30v1.1L20.1,42L40.1,31.1z">
+  </path>
+ </g>
+</svg>

data/lib/outhad/integrations/source/databrics_model/client.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module DatabricksModel
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        url = build_url(DATABRICKS_HEALTH_URL, connection_config)
+        response = Outhad::Integrations::Core::HttpClient.request(
+          url,
+          HTTP_GET,
+          headers: auth_headers(connection_config[:token])
+        )
+        if success?(response)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_outhad_message
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "DATABRICKS MODEL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        # The server checks the ConnectorQueryType.
+        # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
+        # This query is then sent to the AI/ML model.
+        payload = JSON.parse(sync_config.model.query)
+        run_model(connection_config, payload)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "DATABRICKS MODEL:READ:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      private
+
+      def run_model(connection_config, payload)
+        connection_config = connection_config.with_indifferent_access
+
+        url = build_url(DATABRICKS_SERVING_URL, connection_config)
+        token = connection_config[:token]
+
+        response = send_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: payload,
+          headers: auth_headers(token),
+          config: connection_config[:config]
+        )
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, context: "DATABRICKS MODEL:RUN_MODEL:EXCEPTION", type: "error")
+      end
+
+      def process_response(response)
+        if success?(response)
+          begin
+            data = JSON.parse(response.body)
+            [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_outhad_message]
+          rescue JSON::ParserError
+            create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "parsing failed: please send a valid payload")
+          end
+        else
+          create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "request failed: #{response.body}")
+        end
+      end
+
+      def build_url(url, connection_config)
+        format(url, databricks_host: connection_config[:databricks_host],
+                    endpoint_name: connection_config[:endpoint])
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/databrics_model/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/outhad/integrations/source/databrics_model/config/meta.json

@@ -0,0 +1,17 @@
+{
+    "data": {
+      "name": "DatabricksModel",
+      "title": "Databricks Model",
+      "connector_type": "source",
+      "category": "AI Model",
+      "sub_category": "AI_ML Service",
+      "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/databricks-model",
+      "github_issue_label": "source-databricks-foundation",
+      "icon": "icon.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "outhad"]
+    }
+  }
+  

data/lib/outhad/integrations/source/databrics_model/config/spec.json

@@ -0,0 +1,63 @@
+{
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/databricks-model",
+    "stream_type": "user_defined",
+    "connector_query_type": "ai_ml",
+    "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "Databricks Model",
+      "type": "object",
+      "required": ["databricks_host", "token", "endpoint", "request_format", "response_format"],
+      "properties": {
+        "databricks_host": {
+          "title": "Databricks Host URL",
+          "description": "Endpoint host URL for the Databricks serving endpoint.",
+          "type": "string",
+          "examples": ["app.databricks.com"],
+          "order": 0
+        },
+        "token": {
+          "title": "Databricks Token",
+          "description": "personal access token",
+          "type": "string",
+          "outhad_secret": true,
+          "order": 1
+        },
+        "endpoint": {
+          "title": "Endpoint name",
+          "description": "Endpoint name",
+          "examples": ["databricks-dbrx-instruct"],
+          "type": "string",
+          "order": 2
+        },
+        "config": {
+          "title": "",
+          "type": "object",
+          "properties": {
+            "timeout": {
+              "type": "string",
+              "default": "30",
+              "title": "HTTP Timeout",
+              "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
+              "order": 0
+            }
+          },
+          "order": 3
+        },
+        "request_format": {
+          "title": "Request Format",
+          "description": "Sample Request Format",
+          "type": "string",
+          "x-request-format": true,
+          "order": 4
+        },
+        "response_format": {
+          "title": "Response Format",
+          "description": "Sample Response Format",
+          "type": "string",
+          "x-response-format": true,
+          "order": 5
+        }
+      }
+    }
+  }
+  

data/lib/outhad/integrations/source/databrics_model/icon.svg

@@ -0,0 +1,19 @@
+<svg version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 40.1 42" style="enable-background:new 0 0 40.1 42;" xml:space="preserve">
+ <style type="text/css">
+  .st0{fill:#FF3621;}
+ </style>
+ <metadata>
+  <sfw xmlns="ns_sfw;">
+   <slices>
+   </slices>
+   <sliceSourceBounds bottomLeftOrigin="true" height="42" width="40.1" x="-69.1" y="-10.5">
+   </sliceSourceBounds>
+  </sfw>
+ </metadata>
+ <g>
+  <path class="st0" d="M40.1,31.1v-7.4l-0.8-0.5L20.1,33.7l-18.2-10l0-4.3l18.2,9.9l20.1-10.9v-7.3l-0.8-0.5L20.1,21.2L2.6,11.6
+		L20.1,2l14.1,7.7l1.1-0.6V8.3L20.1,0L0,10.9V12L20.1,23l18.2-10v4.4l-18.2,10L0.8,16.8L0,17.3v7.4l20.1,10.9l18.2-9.9v4.3l-18.2,10
+		L0.8,29.5L0,30v1.1L20.1,42L40.1,31.1z">
+  </path>
+ </g>
+</svg>

data/lib/outhad/integrations/source/firecrawl/client.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module Firecrawl
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        request = crawl_activity
+        if success?(request)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "FIRECRAWL:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, { context: "FIRECRAWL:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        url = create_connection(connection_config)
+        query(url, nil, nil)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "FIRECRAWL:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        @base_url = connection_config[:base_url]
+        @api_key = connection_config[:api_key]
+        @config = if connection_config[:config].present?
+                    connection_config[:config].transform_values do |value|
+                      JSON.parse(value)
+                    rescue JSON::ParserError
+                      value
+                    end
+                  else
+                    {}
+                  end
+        @config[:url] ||= connection_config[:base_url]
+        FIRECRAWL_CRAWL_URL
+      end
+
+      def query(url, _query, limit = 1)
+        if limit.present?
+          if @config["includePaths"]&.any?
+            path = @config["includePaths"].first
+            @config["url"] = URI.join(@config["url"], path).to_s
+          end
+          @config.delete("includePaths")
+          @config[:limit] = limit
+        end
+        request = execute_crawl(url)
+        request = JSON.parse(request.body)
+        crawl_url = get_request_url(request)
+        response = get_crawl_result(crawl_url)
+        response["data"].map do |row|
+          metadata_json = row["metadata"].to_json if row["metadata"]
+          metadata_url = row["metadata"]["url"]
+          data = {
+            "metadata": metadata_json,
+            "markdown": row["markdown"],
+            "url": metadata_url,
+            "markdown_hash": Digest::MD5.hexdigest(row["markdown"]) # Placeholder for webscraping extractor use
+          }
+          RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_outhad_message
+        end
+      end
+
+      def execute_crawl(url)
+        send_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: JSON.parse(@config.to_json),
+          headers: auth_headers(@api_key),
+          config: {}
+        )
+      end
+
+      def crawl_activity
+        send_request(
+          url: FIRECRAWL_CRAWL_ACTIVE_URL,
+          http_method: HTTP_GET,
+          payload: {},
+          headers: auth_headers(@api_key),
+          config: {}
+        )
+      end
+
+      # This is to make sure the /crawl/{id} was returned in request.
+      # If not use /crawl/active to retrieve it.
+      def get_request_url(request)
+        if request["url"].blank?
+          if request["error"].present?
+            time = request["error"][/retry after (\d+)s/, 1].to_i
+            sleep(time)
+            execute_crawl(FIRECRAWL_CRAWL_URL)
+          end
+          active = crawl_activity
+          crawl_active = JSON.parse(active.body)
+
+          raise "Missing crawl result URL and no active crawl ID available." unless crawl_active["crawls"][-1]["id"].present?
+
+          crawl_id = crawl_active["crawls"][-1]["id"]
+          build_url(FIRECRAWL_GET_CRAWL_URL, crawl_id.to_s)
+        else
+          request["url"]
+        end
+      end
+
+      # Crawl job needs time to finish task. This method will check if the job is complete.
+      # If not sleep for 5 seconds and check again.
+      def get_crawl_result(url)
+        loop do
+          response = send_request(
+            url: url,
+            http_method: HTTP_GET,
+            payload: {},
+            headers: auth_headers(@api_key),
+            config: {}
+          )
+          response = JSON.parse(response.body)
+          return response if response["status"] != "scraping"
+
+          sleep(FIRECRAWL_REQUEST_RATE_LIMIT)
+        end
+      end
+
+      def build_url(url, id)
+        format(url, id: id)
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/firecrawl/config/catalog.json

@@ -0,0 +1,29 @@
+{
+  "request_rate_limit": 15,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 5,
+  "streams": [
+    {
+      "name": "scrape",
+      "action": "fetch",
+      "json_schema": {
+        "type": "object",
+        "properties": {
+          "markdown": {
+            "type": "string"
+          },
+          "metadata": {
+            "type": "string"
+          },
+          "url": {
+            "type": "string"
+          },
+          "markdown_hash": {
+            "type": "string"
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"]
+    }
+  ]
+}