outhad-integrations 0.32.0

data/lib/outhad/integrations/source/amazon_s3/config/spec.json

@@ -0,0 +1,119 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/amazon_s3",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "AmazonS3",
+    "type": "object",
+    "if": {
+      "properties": {
+        "auth_type": {
+          "enum": ["user"]
+        }
+      }
+    },
+    "then": {
+      "required": [
+        "auth_type",
+        "region",
+        "bucket",
+        "access_id",
+        "secret_access"
+      ]
+    },
+    "else": {
+      "required": [
+        "auth_type",
+        "region",
+        "bucket",
+        "arn",
+        "external_id"
+      ]
+    },
+    "properties": {
+      "data_type": {
+        "description": "Type of data in the files",
+        "type": "string",
+        "title": "Data Format Type",
+        "oneOf": [
+          { "const": "structured", "title": "Tables & Records (Structured)" },
+          { "const": "unstructured", "title": "Documents & Files (Unstructured)" }
+        ],
+        "default": "structured",
+        "order": 9
+      },
+      "auth_type": {
+        "title": "Authentication type",
+        "type": "string",
+        "default": "user",
+        "description": "Authenticate either by using an IAM User (Access Key ID & Secret Access Key) or an IAM Role (ARN)",
+        "enum": [
+          "user",
+          "role"
+        ],
+        "enumNames": [
+          "IAM User",
+          "IAM Role"
+        ],
+        "order": 0
+      },
+      "arn": {
+        "type": "string",
+        "title": "IAM Role ARN",
+        "order": 1
+      },
+      "external_id": {
+        "type": "string",
+        "title": "External Id",
+        "description": "Unique ID that allows handshake between AWS accounts.",
+        "order": 2
+      },
+      "access_id": {
+        "type": "string",
+        "title": "Access Id",
+        "order": 3
+      },
+      "secret_access": {
+        "type": "string",
+        "title": "Secret Access",
+        "outhad_secret": true,
+        "order": 4
+      },
+      "region": {
+        "description": "AWS region",
+        "examples": [
+          "us-east-2"
+        ],
+        "type": "string",
+        "title": "Region",
+        "order": 5
+      },
+      "bucket": {
+        "description": "Bucket Name",
+        "type": "string",
+        "title": "Bucket",
+        "order": 6
+      },
+      "path": {
+        "description": "Path to csv or parquet files",
+        "examples": [
+          "/path/to/files"
+        ],
+        "type": "string",
+        "title": "Path",
+        "order": 7
+      },
+      "file_type": {
+        "description": "The type of file to read",
+        "type": "string",
+        "title": "File Type",
+        "enum": [
+          "csv",
+          "parquet"
+        ],
+        "order": 8
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/amazon_s3/icon.svg

@@ -0,0 +1,34 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="428" height="512" viewBox="0 0 428 512">
+  <defs>
+    <style>
+      .cls-1 {
+        fill: #e25444;
+      }
+
+      .cls-1, .cls-2, .cls-3 {
+        fill-rule: evenodd;
+      }
+
+      .cls-2 {
+        fill: #7b1d13;
+      }
+
+      .cls-3 {
+        fill: #58150d;
+      }
+    </style>
+  </defs>
+  <path class="cls-1" d="M378,99L295,257l83,158,34-19V118Z"/>
+  <path class="cls-2" d="M378,99L212,118,127.5,257,212,396l166,19V99Z"/>
+  <path class="cls-3" d="M43,99L16,111V403l27,12L212,257Z"/>
+  <path class="cls-1" d="M42.637,98.667l169.587,47.111V372.444L42.637,415.111V98.667Z"/>
+  <path class="cls-3" d="M212.313,170.667l-72.008-11.556,72.008-81.778,71.83,81.778Z"/>
+  <path class="cls-3" d="M284.143,159.111l-71.919,11.733-71.919-11.733V77.333"/>
+  <path class="cls-3" d="M212.313,342.222l-72.008,13.334,72.008,70.222,71.83-70.222Z"/>
+  <path class="cls-2" d="M212,16L140,54V159l72.224-20.333Z"/>
+  <path class="cls-2" d="M212.224,196.444l-71.919,7.823V309.105l71.919,8.228V196.444Z"/>
+  <path class="cls-2" d="M212.224,373.333L140.305,355.3V458.363L212.224,496V373.333Z"/>
+  <path class="cls-1" d="M284.143,355.3l-71.919,18.038V496l71.919-37.637V355.3Z"/>
+  <path class="cls-1" d="M212.224,196.444l71.919,7.823V309.105l-71.919,8.228V196.444Z"/>
+  <path class="cls-1" d="M212,16l72,38V159l-72-20V16Z"/>
+</svg>

data/lib/outhad/integrations/source/anthropic/client.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module Anthropic
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      API_VERSION = "2023-06-01"
+      def check_connection(connection_config)
+        connection_config = prepare_config(connection_config)
+        response = make_request(ANTHROPIC_URL, HTTP_POST, connection_config[:request_format], connection_config)
+        success?(response) ? success_status : failure_status(nil)
+      rescue StandardError => e
+        handle_exception(e, { context: "ANTHROPIC:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, { context: "ANTHROPIC:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        # The server checks the ConnectorQueryType.
+        # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
+        # This query is then sent to the AI/ML model.
+        connection_config = prepare_config(sync_config.source.connection_specification)
+        stream = connection_config[:is_stream] ||= false
+        payload = sync_config.model.query
+        if stream
+          run_model_stream(connection_config, payload) { |message| yield message if block_given? }
+        else
+          run_model(connection_config, payload)
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "ANTHROPIC:READ:EXCEPTION", type: "error" })
+      end
+
+      private
+
+      def prepare_config(config)
+        config.with_indifferent_access.tap do |conf|
+          conf[:config][:timeout] ||= 30
+        end
+      end
+
+      def parse_json(json_string)
+        JSON.parse(json_string)
+      rescue JSON::ParserError => e
+        handle_exception(e, { context: "ANTHROPIC:PARSE_JSON:EXCEPTION", type: "error" })
+        {}
+      end
+
+      def build_headers(connection_config, streaming: false)
+        {
+          "x-api-key" => connection_config[:api_key],
+          "anthropic-version" => API_VERSION,
+          "content-type" => "application/json"
+        }.tap do |headers|
+          headers["transfer-encoding"] = "chunked" if streaming
+        end
+      end
+
+      def make_request(url, http_method, payload, connection_config)
+        send_request(
+          url: url,
+          http_method: http_method,
+          payload: JSON.parse(payload),
+          headers: build_headers(connection_config, streaming: false),
+          config: connection_config[:config]
+        )
+      end
+
+      def run_model(connection_config, payload)
+        response = make_request(ANTHROPIC_URL, HTTP_POST, payload, connection_config)
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, { context: "ANTHROPIC:RUN_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def run_model_stream(connection_config, payload)
+        send_streaming_request(
+          url: ANTHROPIC_URL,
+          http_method: HTTP_POST,
+          payload: JSON.parse(payload),
+          headers: build_headers(connection_config, streaming: true),
+          config: connection_config[:config]
+        ) do |chunk|
+          process_streaming_response(chunk) { |message| yield message if block_given? }
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "ANTHROPIC:RUN_STREAM_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def process_response(response)
+        if success?(response)
+          data = JSON.parse(response.body)
+          [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_outhad_message]
+        else
+          create_log_message("ANTHROPIC:RUN_MODEL", "error", "request failed: #{response.body}")
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "ANTHROPIC:PROCESS_RESPONSE:EXCEPTION", type: "error" })
+      end
+
+      def check_chunk_error(chunk)
+        return unless chunk.include?("{\"type\":\"error\"")
+
+        data = JSON.parse(chunk)
+        raise StandardError, "Error: #{data["error"]}" if data["error"] && data["error"]["message"]
+      end
+
+      def extract_content_event(chunk)
+        events = chunk.split("\n\n")
+        events.find { |e| e.include?("event: content_block_delta") }
+      end
+
+      def process_streaming_response(chunk)
+        check_chunk_error(chunk)
+
+        chunk.each_line do |event|
+          next unless event.include?("\"type\":\"content_block_delta\"")
+
+          json_string = event.split("\n").find { |line| line.start_with?("data: ") }&.sub(/^data: /, "")
+          next unless json_string
+
+          parsed_data = JSON.parse(json_string)
+          yield [RecordMessage.new(data: parsed_data, emitted_at: Time.now.to_i).to_outhad_message] if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/anthropic/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 50,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/outhad/integrations/source/anthropic/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "Anthropic",
+    "title": "Anthropic Model Endpoint",
+    "connector_type": "source",
+    "category": "AI Model",
+    "sub_category": "LLM",
+    "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/anthropic-model",
+    "github_issue_label": "source-anthropic-model",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/anthropic/config/spec.json

@@ -0,0 +1,56 @@
+{
+  "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/anthropic-model",
+  "stream_type": "user_defined",
+  "connector_query_type": "ai_ml",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Anthropic Endpoint",
+    "type": "object",
+    "required": ["api_key", "request_format", "response_format"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "outhad_secret": true,
+        "title": "API Key",
+        "order": 0
+      },
+      "is_stream": {
+        "type": "boolean",
+        "title": "Enable streaming",
+        "description": "Enables data streaming for such as chat, when supported by the model. When true, messages and model data are processed in chunks for immediate delivery, enhancing responsiveness. Default is false, processing only after the entire response is received.",
+        "default": false,
+        "order": 1
+      },
+      "config": {
+        "title": "",
+        "type": "object",
+        "properties": {
+          "timeout": {
+            "type": "string",
+            "default": "30",
+            "title": "HTTP Timeout",
+            "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
+            "order": 0
+          }
+        },
+        "order": 2
+      },
+      "request_format": {
+        "title": "Request Format",
+        "description": "Sample Request Format",
+        "type": "string",
+        "default": "{\"model\":\"claude-3-opus-20240229\",\"max_tokens\": 256, \"messages\":[{\"role\": \"user\", \"content\": \"Hi.\"}], \"stream\": false}",
+        "x-request-format": true,
+        "order": 3
+      },
+      "response_format": {
+        "title": "Response Format",
+        "description": "Sample Response Format",
+        "type": "string",
+        "default": "{\"id\":\"msg_0123ABC\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-3-7-sonnet-20250219\",\"content\":[{\"type\":\"text\",\"text\":\"Hello there! How can I assist you today? Whether you have a question, need some information, or just want to chat, I'm here to help. What's on your mind?\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":10,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":41}}",
+        "x-response-format": true,
+        "order": 4
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/anthropic/icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" id="Anthropic-Icon--Streamline-Svg-Logos" height="24" width="24"><desc>Anthropic Icon Streamline Icon: https://streamlinehq.com</desc><path fill="#181818" d="m13.788825 3.932 6.43325 16.136075h3.5279L17.316725 3.932H13.788825Z" stroke-width="0.25"></path><path fill="#181818" d="m6.325375 13.682775 2.20125 -5.67065 2.201275 5.67065H6.325375ZM6.68225 3.932 0.25 20.068075h3.596525l1.3155 -3.3886h6.729425l1.315275 3.3886h3.59655L10.371 3.932H6.68225Z" stroke-width="0.25"></path></svg>

data/lib/outhad/integrations/source/aws_athena/client.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module AwsAthena
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        athena_client = create_connection(connection_config)
+        athena_client.list_work_groups
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_outhad_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:schema]}' ORDER BY table_name, ordinal_position;"
+        db = create_connection(connection_config)
+        results = query_execution(db, query)
+        catalog = Catalog.new(streams: create_streams(results))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AWS:ATHENA:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        db = create_connection(connection_config)
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AWS:ATHENA:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        Aws.config.update({ credentials: Aws::Credentials.new(connection_config[:access_key], connection_config[:secret_access_key]), region: connection_config[:region] })
+        @database = connection_config[:schema]
+        @output_location = connection_config[:output_location]
+        Aws::Athena::Client.new
+      end
+
+      def query_execution(db, query)
+        response = db.start_query_execution(
+          query_string: query,
+          query_execution_context: { database: @database },
+          result_configuration: { output_location: @output_location }
+        )
+        query_execution_id = response[:query_execution_id]
+        loop do
+          response = db.get_query_execution(query_execution_id: query_execution_id)
+          status = response.query_execution.status.state
+          break if %w[SUCCEEDED FAILED CANCELLED].include?(status)
+        end
+        transform_results(db.get_query_results(query_execution_id: query_execution_id))
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def transform_results(results)
+        columns = results.result_set.result_set_metadata.column_info.map(&:name)
+        rows = results.result_set.rows.map do |row|
+          row.data.map(&:var_char_value)
+        end
+        rows.map { |row| columns.zip(row).to_h }
+      end
+
+      def query(db, query)
+        records = []
+        query_execution(db, query).map do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+        end
+        records
+      end
+
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry["table_name"]
+          column_data = {
+            column_name: entry["column_name"],
+            data_type: entry["data_type"],
+            is_nullable: entry["is_nullable"] == "YES"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/aws_athena/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "AwsAthena",
+    "title": "AWS Athena",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/aws_athena",
+    "github_issue_label": "source-aws-athena",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/aws_athena/config/spec.json

@@ -0,0 +1,63 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/aws_athena",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "AWS Athena",
+    "type": "object",
+    "required": ["access_key", "secret_access_key", "region", "workgroup", "catalog", "output_location"],
+    "properties": {
+      "access_key": {
+        "description": "The AWS Access Key ID to use for authentication.",
+        "examples": ["AWSATHENAACCESS"],
+        "type": "string",
+        "title": "Personal Access Key",
+        "order": 0
+      },
+      "secret_access_key": {
+        "description": "The AWS Secret Access Key to use for authentication.",
+        "examples": ["AWSATHENASECRET"],
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Secret Access Key",
+        "order": 1
+      },
+      "region": {
+        "description": "AWS region where Athena is located.",
+        "examples": ["ATHENA_REGION"],
+        "type": "string",
+        "title": "Secret Access Key",
+        "order": 2
+      },
+      "workgroup": {
+        "description": "The Athena workgroup you previously set up in AWS.",
+        "examples": ["ATHENA_WORKGROUP"],
+        "type": "string",
+        "title": "Workgroup",
+        "order": 3
+      },
+      "catalog": {
+        "description": "The Data catalog name within Athena.",
+        "examples": ["ATHENA_CATALOG"],
+        "type": "string",
+        "title": "Catalog",
+        "order": 4
+      },
+      "schema": {
+        "description": "The specific Athena database/schema to connect to.",
+        "examples": ["ATHENA_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 5
+      },
+      "output_location": {
+        "description": "S3 path for query output.",
+        "examples": ["s3://example-bucket-name/query-results/"],
+        "type": "string",
+        "title": "Query",
+        "order": 6
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/aws_athena/icon.svg

@@ -0,0 +1,22 @@
+<svg height="100" viewBox="0 0 100 100" width="100" xmlns="http://www.w3.org/2000/svg">
+    <path d="m84 64.68-34-4.19-34 4.19 34 26.62z" fill="#fcbf92"/>
+    <path d="m16 64.68 34 10.17v16.45l-34-17.05z" fill="#9d5025"/>
+    <path d="m84 64.68-34 10.17v16.45l34-17.05z" fill="#f58534"/>
+    <path d="m20.39 50.22h-4.39v9.67l4.39.8 4.88-5.18z" fill="#9d5025"/>
+    <path d="m25.27 60.29-4.88.4v-10.47h4.88z" fill="#f58534"/>
+    <path d="m28.56 38.16-5.78 1.09v21.94l5.78 1.2 5.29-12.17z" fill="#9d5025"/>
+    <path d="m33.85 61.79-5.29.6v-24.23l5.29.6z" fill="#f58534"/>
+    <path d="m39.83 43.04-7.98.8v19.24l7.98 1.6 8.47-10.87z" fill="#9d5025"/>
+    <path d="m45.21 63.78-5.38.9v-21.64l5.38.4z" fill="#f58534"/>
+    <g fill="#9d5025">
+        <path d="m50 25.59-5.38 1.7v38.29l5.38 1.09 5.38-20.54z"/>
+        <path d="m54.98 63.78 5.19.9 7.98-14.46-7.98-14.36-5.19.8z"/>
+        <path d="m68.15 50.22-7.98-14.36-5.19.8"/>
+        <path d="m66.05 61.79 5.39.6 5.78-18.25-5.78-18.15-5.39 1.3z"/>
+        <path d="m74.73 60.29 4.88.4 4.39-18.05-4.39-18.64-4.88 1.2z"/>
+    </g>
+    <path d="m50 25.59 5.38 1.7v38.29l-5.38 1.09z" fill="#f58534"/>
+    <path d="m68.15 37.46-7.98-1.6v28.82l7.98-1.6z" fill="#f58534"/>
+    <path d="m77.22 28.29-5.78-2.3v36.4l5.78-1.2z" fill="#f58534"/>
+    <path d="m84 26.19-4.39-2.19v36.69l4.39-.8z" fill="#f58534"/>
+</svg>