outhad-integrations 0.32.0

data/lib/outhad/integrations/source/pinecone_db/client.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module PineconeDB
+    include Outhad::Integrations::Core
+    class Client < VectorSourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        pinecone = create_connection(connection_config)
+        result = pinecone.describe_index(@index_name)
+        if success?(result)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "PINECONE:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "PINECONE:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def search(vector_search_config)
+        connection_config = vector_search_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        connection = create_connection(connection_config)
+        pinecone_index = connection.index(@index_name)
+        response = pinecone_index.query(vector: vector_search_config[:vector],
+                                        namespace: @namespace,
+                                        top_k: vector_search_config[:limit],
+                                        include_values: true,
+                                        include_metadata: true)
+        result = JSON.parse(response.body).with_indifferent_access
+        records = result["matches"]
+        records.map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "PINECONE:SEARCH:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        initialize_client(connection_config)
+        Pinecone.configure do |config|
+          config.api_key = @api_key
+          config.environment = @region
+        end
+        Pinecone::Client.new
+      end
+
+      def initialize_client(connection_config)
+        @api_key = connection_config["api_key"]
+        @region = connection_config["region"]
+        @index_name = connection_config["index_name"]
+        @namespace = connection_config["namespace"]
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/pinecone_db/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 120000,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/outhad/integrations/source/pinecone_db/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "PineconeDB",
+    "title": "Pinecone DB",
+    "connector_type": "source",
+    "category": "Database",
+    "sub_category": "Vector Database",
+    "documentation_url": "https://docs.squared.ai/activation/vector_search/pinecone_db",
+    "github_issue_label": "source-pinecone-db",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/pinecone_db/config/spec.json

@@ -0,0 +1,34 @@
+{
+  "documentation_url": "https://docs.squared.ai/activation/vector_search/pinecone_db",
+  "stream_type": "user_defined",
+  "connector_query_type": "vector_search",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Pinecone DB",
+    "type": "object",
+    "required": ["api_key", "region", "index_name", "namespace"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "outhad_secret": true,
+        "title": "API Key",
+        "order": 0
+      },
+      "region": {
+        "type": "string",
+        "title": "Region",
+        "order": 1
+      },
+      "index_name": {
+        "type": "string",
+        "title": "Index Name",
+        "order": 2
+      },
+      "namespace": {
+        "type": "string",
+        "title": "Namespace",
+        "order": 3
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/pinecone_db/icon.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg fill="#000000" width="800px" height="800px" viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg"><title>Dynamics 365 icon</title><path d="M4.59 7.41l4.94 3.54L4.59 24zm0-7.41v6.36l9.53 5.29 4.59-3.52zm0 24l14.82-8.47v-6.7Z"/></svg>

data/lib/outhad/integrations/source/postgresql/client.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Outhad::Integrations::Source
+  module Postgresql
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_outhad_message
+      rescue PG::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name,
+                 CASE WHEN data_type = 'USER-DEFINED' THEN udt_name ELSE data_type END
+                 AS data_type,
+                 is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+        records = db.exec(query) do |result|
+          result.map do |row|
+            row
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "POSTGRESQL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db&.close
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        db = create_connection(connection_config)
+
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "POSTGRESQL:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      ensure
+        db&.close
+      end
+
+      private
+
+      def query(connection, query)
+        connection.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+          end
+        end
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        PG.connect(
+          host: connection_config[:host],
+          dbname: connection_config[:database],
+          user: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          port: connection_config[:port]
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/postgresql/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "Postgresql",
+    "title": "PostgreSQL",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/postgresql",
+    "github_issue_label": "source-postgresql",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/postgresql/config/spec.json

@@ -0,0 +1,86 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/postgresql",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Postgresql",
+    "type": "object",
+    "required": ["data_type", "host", "port", "database", "schema"],
+    "properties": {
+      "data_type": {
+        "description": "Type of data in the database",
+        "type": "string",
+        "title": "Data Format Type",
+        "oneOf": [
+          {
+            "const": "structured",
+            "title": "Tables & Records (Structured)"
+          },
+          {
+            "const": "vector",
+            "title": "Tables & Records containing Vector fields"
+          }
+        ],
+        "default": "structured",
+        "order": 0
+      },
+      "credentials": {
+        "title": "",
+        "type": "object",
+        "required": ["auth_type", "username", "password"],
+        "properties": {
+          "auth_type": {
+            "type": "string",
+            "default": "username/password",
+            "order": 0,
+            "readOnly": true
+          },
+          "username": {
+            "description": "Username refers to your individual PostgreSQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
+            "examples": ["POSTGRESQL_USER"],
+            "type": "string",
+            "title": "Username",
+            "order": 1
+          },
+          "password": {
+            "description": "This field requires the password associated with the user account specified in the preceding section.",
+            "type": "string",
+            "outhad_secret": true,
+            "title": "Password",
+            "order": 2
+          }
+        },
+        "order": 1
+      },
+      "host": {
+        "description": "The hostname or IP address of your PostgreSQL server.",
+        "examples": ["127.0.0.1"],
+        "type": "string",
+        "title": "Host",
+        "order": 2
+      },
+      "port": {
+        "description": "The port number for your PostgreSQL server, which defaults to 5432, may vary based on your configuration. ",
+        "examples": ["5432"],
+        "type": "string",
+        "title": "Port",
+        "order": 3
+      },
+      "database": {
+        "description": "The specific PostgreSQL database to connect to.",
+        "examples": ["POSTGRESQL_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 4
+      },
+      "schema": {
+        "description": "The schema within the PostgreSQL database.",
+        "examples": ["POSTGRESQL_SCHEMA"],
+        "type": "string",
+        "title": "Schema",
+        "order": 5
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/postgresql/icon.svg

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg width="432.071pt" height="445.383pt" viewBox="0 0 432.071 445.383" xml:space="preserve" xmlns="http://www.w3.org/2000/svg">
+<g id="orginal" style="fill-rule:nonzero;clip-rule:nonzero;stroke:#000000;stroke-miterlimit:4;">
+	</g>
+<g id="Layer_x0020_3" style="fill-rule:nonzero;clip-rule:nonzero;fill:none;stroke:#FFFFFF;stroke-width:12.4651;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;">
+<path style="fill:#000000;stroke:#000000;stroke-width:37.3953;stroke-linecap:butt;stroke-linejoin:miter;" d="M323.205,324.227c2.833-23.601,1.984-27.062,19.563-23.239l4.463,0.392c13.517,0.615,31.199-2.174,41.587-7c22.362-10.376,35.622-27.7,13.572-23.148c-50.297,10.376-53.755-6.655-53.755-6.655c53.111-78.803,75.313-178.836,56.149-203.322    C352.514-5.534,262.036,26.049,260.522,26.869l-0.482,0.089c-9.938-2.062-21.06-3.294-33.554-3.496c-22.761-0.374-40.032,5.967-53.133,15.904c0,0-161.408-66.498-153.899,83.628c1.597,31.936,45.777,241.655,98.47,178.31    c19.259-23.163,37.871-42.748,37.871-42.748c9.242,6.14,20.307,9.272,31.912,8.147l0.897-0.765c-0.281,2.876-0.157,5.689,0.359,9.019c-13.572,15.167-9.584,17.83-36.723,23.416c-27.457,5.659-11.326,15.734-0.797,18.367c12.768,3.193,42.305,7.716,62.268-20.224    l-0.795,3.188c5.325,4.26,4.965,30.619,5.72,49.452c0.756,18.834,2.017,36.409,5.856,46.771c3.839,10.36,8.369,37.05,44.036,29.406c29.809-6.388,52.6-15.582,54.677-101.107"/>
+<path style="fill:#336791;stroke:none;" d="M402.395,271.23c-50.302,10.376-53.76-6.655-53.76-6.655c53.111-78.808,75.313-178.843,56.153-203.326c-52.27-66.785-142.752-35.2-144.262-34.38l-0.486,0.087c-9.938-2.063-21.06-3.292-33.56-3.496c-22.761-0.373-40.026,5.967-53.127,15.902    c0,0-161.411-66.495-153.904,83.63c1.597,31.938,45.776,241.657,98.471,178.312c19.26-23.163,37.869-42.748,37.869-42.748c9.243,6.14,20.308,9.272,31.908,8.147l0.901-0.765c-0.28,2.876-0.152,5.689,0.361,9.019c-13.575,15.167-9.586,17.83-36.723,23.416    c-27.459,5.659-11.328,15.734-0.796,18.367c12.768,3.193,42.307,7.716,62.266-20.224l-0.796,3.188c5.319,4.26,9.054,27.711,8.428,48.969c-0.626,21.259-1.044,35.854,3.147,47.254c4.191,11.4,8.368,37.05,44.042,29.406c29.809-6.388,45.256-22.942,47.405-50.555    c1.525-19.631,4.976-16.729,5.194-34.28l2.768-8.309c3.192-26.611,0.507-35.196,18.872-31.203l4.463,0.392c13.517,0.615,31.208-2.174,41.591-7c22.358-10.376,35.618-27.7,13.573-23.148z"/>
+<path d="M215.866,286.484c-1.385,49.516,0.348,99.377,5.193,111.495c4.848,12.118,15.223,35.688,50.9,28.045c29.806-6.39,40.651-18.756,45.357-46.051c3.466-20.082,10.148-75.854,11.005-87.281"/>
+<path d="M173.104,38.256c0,0-161.521-66.016-154.012,84.109c1.597,31.938,45.779,241.664,98.473,178.316c19.256-23.166,36.671-41.335,36.671-41.335"/>
+<path d="M260.349,26.207c-5.591,1.753,89.848-34.889,144.087,34.417c19.159,24.484-3.043,124.519-56.153,203.329"/>
+<path style="stroke-linejoin:bevel;" d="M348.282,263.953c0,0,3.461,17.036,53.764,6.653c22.04-4.552,8.776,12.774-13.577,23.155c-18.345,8.514-59.474,10.696-60.146-1.069c-1.729-30.355,21.647-21.133,19.96-28.739c-1.525-6.85-11.979-13.573-18.894-30.338    c-6.037-14.633-82.796-126.849,21.287-110.183c3.813-0.789-27.146-99.002-124.553-100.599c-97.385-1.597-94.19,119.762-94.19,119.762"/>
+<path d="M188.604,274.334c-13.577,15.166-9.584,17.829-36.723,23.417c-27.459,5.66-11.326,15.733-0.797,18.365c12.768,3.195,42.307,7.718,62.266-20.229c6.078-8.509-0.036-22.086-8.385-25.547c-4.034-1.671-9.428-3.765-16.361,3.994z"/>
+<path d="M187.715,274.069c-1.368-8.917,2.93-19.528,7.536-31.942c6.922-18.626,22.893-37.255,10.117-96.339c-9.523-44.029-73.396-9.163-73.436-3.193c-0.039,5.968,2.889,30.26-1.067,58.548c-5.162,36.913,23.488,68.132,56.479,64.938"/>
+<path style="fill:#FFFFFF;stroke-width:4.155;stroke-linecap:butt;stroke-linejoin:miter;" d="M172.517,141.7c-0.288,2.039,3.733,7.48,8.976,8.207c5.234,0.73,9.714-3.522,9.998-5.559c0.284-2.039-3.732-4.285-8.977-5.015c-5.237-0.731-9.719,0.333-9.996,2.367z"/>
+<path style="fill:#FFFFFF;stroke-width:2.0775;stroke-linecap:butt;stroke-linejoin:miter;" d="M331.941,137.543c0.284,2.039-3.732,7.48-8.976,8.207c-5.238,0.73-9.718-3.522-10.005-5.559c-0.277-2.039,3.74-4.285,8.979-5.015c5.239-0.73,9.718,0.333,10.002,2.368z"/>
+<path d="M350.676,123.432c0.863,15.994-3.445,26.888-3.988,43.914c-0.804,24.748,11.799,53.074-7.191,81.435"/>
+<path style="stroke-width:3;" d="M0,60.232"/>
+</g>
+</svg>

data/lib/outhad/integrations/source/qdrant/client.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module Qdrant
+    include Outhad::Integrations::Core
+    class Client < VectorSourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        response = Outhad::Integrations::Core::HttpClient.request(
+          @host,
+          HTTP_GET,
+          headers: auth_headers(@api_key)
+        )
+        if success?(response)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "QDRANT:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "QDRANT:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def search(vector_search_config)
+        connection_config = vector_search_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        url = build_url(QDRANT_SEARCH_URL)
+
+        body = {
+          vector: vector_search_config[:vector],
+          top: vector_search_config[:limit]
+        }
+
+        response = Outhad::Integrations::Core::HttpClient.request(
+          url,
+          HTTP_POST,
+          headers: {
+            "Content-Type" => "application/json",
+            "api-key" => @api_key
+          },
+          payload: body
+        )
+
+        response = JSON.parse(response.body).with_indifferent_access
+        records = response[:result] || []
+
+        records.map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "QDRANT:SEARCH:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        @api_key = connection_config[:api_key]
+        @host = connection_config[:host]
+        @collection_name = connection_config[:collection_name]
+      end
+
+      def build_url(url)
+        format(url, host: @host, collection_name: @collection_name)
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/qdrant/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 1200,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/outhad/integrations/source/qdrant/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "Qdrant",
+    "title": "Qdrant",
+    "connector_type": "source",
+    "category": "Database",
+    "sub_category": "Vector Database",
+    "documentation_url": "https://docs.squared.ai/activation/vector_search/qdrant",
+    "github_issue_label": "source-qdrant",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]        
+  }
+}

data/lib/outhad/integrations/source/qdrant/config/spec.json

@@ -0,0 +1,29 @@
+{
+  "documentation_url": "https://docs.squared.ai/activation/vector_search/qdrant",
+  "stream_type": "user_defined",
+  "connector_query_type": "vector_search",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Qdrant",
+    "type": "object",
+    "required": ["host", "api_key", "collection_name"],
+    "properties": {
+      "host": {
+        "type": "string",
+        "title": "Host",
+        "order": 0
+      },
+      "api_key": {
+        "type": "string",
+        "outhad_secret": true,
+        "title": "API Key",
+        "order": 1
+      },
+      "collection_name": {
+        "type": "string",
+        "title": "Collection Name",
+        "order": 2
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/qdrant/icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 451.84 152.4" width="451.84" height="152.4" fill="none"><g fill="#dc244c"><path d="M212.69 116c0 5.523-4.477 10.001-10 10.001h-6.836v-7.808h-.244c-.732 1.057-1.708 2.155-2.928 3.293-1.139 1.058-2.521 2.034-4.148 2.929-1.545.894-3.294 1.626-5.246 2.196a20.015 20.015 0 0 1-5.856.853c-4.392 0-8.377-.732-11.956-2.196-3.579-1.545-6.669-3.66-9.272-6.344-2.521-2.765-4.473-6.018-5.856-9.759-1.383-3.742-2.074-7.849-2.074-12.322 0-4.148.61-8.093 1.83-11.835 1.301-3.822 3.091-7.198 5.368-10.126a26.176 26.176 0 0 1 8.54-6.954c3.335-1.789 7.117-2.683 11.346-2.683 3.823 0 7.361.61 10.614 1.83 3.335 1.138 6.059 3.13 8.174 5.978h.244V43.769c0-5.523 4.477-10 10-10h8.3zm-16.836-19.646c0-4.473-1.301-8.092-3.904-10.858-2.521-2.765-6.1-4.148-10.736-4.148s-8.255 1.383-10.858 4.148c-2.521 2.766-3.782 6.385-3.782 10.858 0 4.474 1.261 8.093 3.782 10.858 2.603 2.766 6.222 4.149 10.858 4.149s8.215-1.383 10.736-4.149c2.603-2.765 3.904-6.384 3.904-10.858zM224.53 76.708c0-5.522 4.477-10 10-10h8.3v9.516h.244c1.952-3.66 4.27-6.384 6.954-8.174 2.684-1.87 6.059-2.805 10.126-2.805 1.057 0 2.114.04 3.172.121 1.057.082 2.033.245 2.928.489v16.713a22.4 22.4 0 0 0-3.904-.854 19.895 19.895 0 0 0-3.904-.367c-3.498 0-6.263.489-8.296 1.465-2.034.975-3.62 2.358-4.758 4.147-1.058 1.708-1.749 3.782-2.074 6.222-.326 2.441-.488 5.124-.488 8.052V116c0 5.524-4.477 10.002-10 10.002h-8.3zM310.64 118.56h-.244c-2.033 3.172-4.758 5.449-8.174 6.832-3.334 1.382-6.872 2.073-10.614 2.073-2.765 0-5.449-.406-8.052-1.219-2.521-.732-4.758-1.871-6.71-3.416-1.952-1.546-3.497-3.457-4.636-5.735-1.138-2.277-1.708-4.92-1.708-7.929 0-3.416.61-6.304 1.83-8.662 1.302-2.359 3.01-4.311 5.124-5.856 2.196-1.546 4.677-2.725 7.442-3.538a50.076 50.076 0 0 1 8.54-1.953c3.01-.406 5.978-.65 8.906-.732 3.01-.081 5.775-.121 8.296-.121 0-3.254-1.179-5.816-3.538-7.687-2.277-1.952-5.002-2.928-8.174-2.928-3.009 0-5.774.651-8.296 1.952-2.44 1.221-4.636 2.929-6.588 5.124l-9.76-10.004c3.416-3.171 7.402-5.53 11.956-7.075a41.766 41.766 0 0 1 14.152-2.44c5.368 0 9.76.69 13.176 2.073 3.498 1.302 6.263 3.254 8.296 5.856 2.115 2.603 3.579 5.816 4.392 9.638.814 3.742 1.22 8.093 1.22 13.054v20.135c0 5.522-4.477 10-10 10h-6.836zm-4.514-18.545c-1.382 0-3.131.082-5.246.244a24.865 24.865 0 0 0-5.978.976c-1.87.57-3.497 1.424-4.88 2.562-1.301 1.139-1.952 2.725-1.952 4.759 0 2.196.936 3.822 2.806 4.879 1.871 1.058 3.823 1.586 5.856 1.586 1.79 0 3.498-.244 5.124-.732 1.708-.488 3.213-1.179 4.514-2.074a9.45 9.45 0 0 0 3.05-3.416c.814-1.382 1.22-3.009 1.22-4.879v-3.905zM340.18 76.708c0-5.522 4.477-10 10-10h7.568v8.052h.244c.569-1.138 1.382-2.277 2.44-3.416 1.057-1.138 2.318-2.155 3.782-3.05s3.131-1.626 5.002-2.195c1.87-.57 3.904-.854 6.1-.854 4.636 0 8.377.732 11.224 2.196 2.846 1.382 5.042 3.334 6.588 5.855 1.626 2.522 2.724 5.49 3.294 8.906.569 3.416.854 7.117.854 11.103V116c0 5.523-4.477 10.001-10 10.001h-8.3V96.964a58.2 58.2 0 0 0-.244-5.246c-.082-1.87-.448-3.578-1.098-5.123a8.154 8.154 0 0 0-2.806-3.783c-1.22-.976-3.01-1.464-5.368-1.464-2.359 0-4.27.448-5.734 1.342a8.629 8.629 0 0 0-3.416 3.416c-.732 1.383-1.22 2.969-1.464 4.758a41.535 41.535 0 0 0-.366 5.612V116c0 5.523-4.477 10.001-10 10.001h-8.3zM451.84 71.348c0 5.522-4.477 10-10 10h-6.104v19.764c0 1.627.082 3.132.244 4.514.163 1.301.529 2.44 1.098 3.415.57.977 1.424 1.75 2.562 2.32 1.22.487 2.806.73 4.758.73.976 0 2.237-.08 3.782-.243 1.627-.245 2.847-.732 3.66-1.464v8.725c0 3.915-2.452 7.557-6.344 7.989a58.528 58.528 0 0 1-6.466.366c-3.09 0-5.937-.325-8.54-.976-2.602-.65-4.88-1.667-6.832-3.05-1.952-1.463-3.497-3.334-4.636-5.612-1.057-2.277-1.586-5.042-1.586-8.295V81.347h-11.712v-4.64c0-5.522 4.478-9.999 10-9.999h1.712V59.14c0-5.523 4.478-10 10-10h8.3v17.568h16.104z"/></g><g fill-rule="evenodd" clip-rule="evenodd"><path fill="#24386c" d="m103.79 140.09-3.039-83.784-5.503-22.089 36.734 3.889v101.35l-22.44 12.95z"/><path fill="#7589be" d="m131.98 38.1-22.44 12.96-46.308-10.158L9.029 62.971-.001 38.1l32.99-19.05 33-19.05 32.99 19.05z"/><path fill="#b2bfe8" d="m0 38.1 22.44 12.96 13.008 38.686 43.921 35.142L65.991 152.4l-33-19.051L0 114.299v-76.2"/><path fill="#24386c" d="m80.868 104.56-14.877 21.932v25.91l21.11-12.18 10.877-16.242"/><path fill="#7589be" d="M66 100.59 44.881 64.025l4.549-12.119 17.293-8.384L87.1 64.026z"/><path fill="#b2bfe8" d="m44.881 64.022 21.11 12.18v24.38l-19.524.84-11.81-15.08 10.224-22.32"/><path fill="#24386c" d="m65.991 76.2 21.11-12.179 14.367 23.922-17.386 14.365-18.091-1.727z"/><path fill="#dc244c" d="m87.101 140.22 22.44 12.181V51.061l-21.78-12.57-21.77-12.57-21.78 12.57-21.77 12.57v50.289l21.77 12.57 21.78 12.571 21.11-12.191zm0-51.83-21.11 12.19-21.11-12.19V64.02l21.11-12.19 21.11 12.19v24.37"/></g><path fill="url(#a)" d="M66 126.5v-25.914L45 88.5v25.871z"/><defs><linearGradient id="a" x1="62.128" x2="41.202" y1="105.54" y2="105.54" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#FF3364"/><stop offset="1" stop-color="#C91540" stop-opacity="0"/></linearGradient></defs></svg>

data/lib/outhad/integrations/source/redshift/client.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Outhad::Integrations::Source
+  module Redshift
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_outhad_message
+      rescue PG::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+        records = db.exec(query) do |result|
+          result.map do |row|
+            row
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "REDSHIFT:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db&.close
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        db = create_connection(connection_config)
+
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "REDSHIFT:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      ensure
+        db&.close
+      end
+
+      private
+
+      def query(connection, query)
+        connection.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+          end
+        end
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        PG.connect(
+          host: connection_config[:host],
+          dbname: connection_config[:database],
+          user: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          port: connection_config[:port]
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/redshift/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "Redshift",
+    "title": "Amazon Redshift",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/redshift",
+    "github_issue_label": "source-redshift",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}