outhad-integrations 0.32.0

data/lib/outhad/integrations/core/utils.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    module Utils
+      def keys_to_symbols(hash)
+        if hash.is_a?(Hash)
+          hash.each_with_object({}) do |(key, value), result|
+            result[key.to_sym] = keys_to_symbols(value)
+          end
+        elsif hash.is_a?(Array)
+          hash.map { |item| keys_to_symbols(item) }
+        else
+          hash
+        end
+      end
+
+      def convert_to_json_schema(column_definitions)
+        json_schema = {
+          "type" => "object",
+          "properties" => {}
+        }
+
+        column_definitions.each do |column|
+          column_name = column[:column_name]
+          type = column[:type]
+          optional = column[:optional]
+          json_type = map_type_to_json_schema(type)
+          json_schema["properties"][column_name] = {
+            "type" => json_type
+          }
+          json_schema["properties"][column_name]["type"] = [json_type, "null"] if optional
+        end
+
+        json_schema
+      end
+
+      def map_type_to_json_schema(type)
+        case type
+        when "NUMBER"
+          "integer"
+        when "vector"
+          "vector"
+        else
+          "string" # Default type
+        end
+      end
+
+      def logger
+        Integrations::Service.logger
+      end
+
+      def report_exception(exception, meta = {})
+        reporter = Integrations::Service.exception_reporter
+        reporter&.report(exception, meta)
+      end
+
+      def log_request_response(level, request, response)
+        Integrations::Protocol::LogMessage.new(
+          name: self.class.name,
+          level: level,
+          message: { request: request.to_s, response: response.to_s, level: level }.to_json
+        )
+      end
+
+      def create_log_message(context, type, message)
+        Integrations::Protocol::LogMessage.new(
+          name: context,
+          level: type,
+          message: message
+        ).to_outhad_message
+      end
+
+      def handle_exception(exception, meta = {})
+        logger.error(
+          "#{hash_to_string(meta)}: #{exception.message}"
+        )
+        report_exception(exception, meta)
+        create_log_message(meta[:context], meta[:type], exception.message)
+      end
+
+      def hash_to_string(hash)
+        hash.map { |key, value| "#{key} = #{value}" }.join(", ")
+      end
+
+      def extract_data(record_object, properties)
+        data_attributes = record_object.with_indifferent_access
+        data_attributes.select { |key, _| properties.key?(key.to_sym) }
+      end
+
+      def success?(response)
+        response && %w[200 201].include?(response.code.to_s)
+      end
+
+      def build_catalog(catalog_json)
+        streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
+        Outhad::Integrations::Protocol::Catalog.new(
+          streams: streams,
+          request_rate_limit: catalog_json["request_rate_limit"] || 60,
+          request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
+          request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10,
+          schema_mode: catalog_json["schema_mode"] || "schema"
+        )
+      end
+
+      def build_stream(stream_json)
+        Outhad::Integrations::Protocol::Stream.new(
+          name: stream_json["name"],
+          url: stream_json["url"],
+          action: stream_json["action"],
+          request_method: stream_json["method"],
+          batch_support: stream_json["batch_support"] || false,
+          batch_size: stream_json["batch_size"] || 1,
+          json_schema: stream_json["json_schema"],
+          request_rate_limit: stream_json["request_rate_limit"].to_i,
+          request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
+          request_rate_concurrency: stream_json["request_rate_concurrency"].to_i,
+          supported_sync_modes: stream_json["supported_sync_modes"]
+        )
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/vector_source_connector.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    class VectorSourceConnector < SourceConnector
+      # This needs to be implemented
+      # for all vector database sources
+      # that will be used in RAG workflows
+      def search(_vector_search_config)
+        raise "Not implemented"
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/airtable/client.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require_relative "schema_helper"
+module Outhad
+  module Integrations
+    module Destination
+      module Airtable
+        include Outhad::Integrations::Core
+        class Client < DestinationConnector
+          prepend Outhad::Integrations::Core::RateLimiter
+          MAX_CHUNK_SIZE = 10
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            bases = Outhad::Integrations::Core::HttpClient.request(
+              AIRTABLE_BASES_ENDPOINT,
+              HTTP_GET,
+              headers: auth_headers(connection_config[:api_key])
+            )
+            if success?(bases)
+              base_id_exists?(bases, connection_config[:base_id])
+              success_status
+            else
+              failure_status(nil)
+            end
+          rescue StandardError => e
+            failure_status(e)
+          end
+
+          def discover(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            base_id = connection_config[:base_id]
+            api_key = connection_config[:api_key]
+
+            bases = Outhad::Integrations::Core::HttpClient.request(
+              AIRTABLE_BASES_ENDPOINT,
+              HTTP_GET,
+              headers: auth_headers(api_key)
+            )
+
+            base = extract_bases(bases).find { |b| b["id"] == base_id }
+            base_name = base["name"]
+
+            schema = Outhad::Integrations::Core::HttpClient.request(
+              AIRTABLE_GET_BASE_SCHEMA_ENDPOINT.gsub("{baseId}", base_id),
+              HTTP_GET,
+              headers: auth_headers(api_key)
+            )
+
+            catalog = build_catalog_from_schema(extract_body(schema), base_id, base_name)
+            catalog.to_outhad_message
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "AIRTABLE:DISCOVER:EXCEPTION",
+                               type: "error"
+                             })
+          end
+
+          def write(sync_config, records, _action = "create")
+            connection_config = sync_config.destination.connection_specification.with_indifferent_access
+            api_key = connection_config[:api_key]
+            url = sync_config.stream.url
+            log_message_array = []
+            write_success = 0
+            write_failure = 0
+            records.each_slice(MAX_CHUNK_SIZE) do |chunk|
+              payload = create_payload(chunk)
+              args = [sync_config.stream.request_method, url, payload]
+              response = Outhad::Integrations::Core::HttpClient.request(
+                url,
+                sync_config.stream.request_method,
+                payload: payload,
+                headers: auth_headers(api_key)
+              )
+              if success?(response)
+                write_success += chunk.size
+              else
+                write_failure += chunk.size
+              end
+              log_message_array << log_request_response("info", args, response)
+            rescue StandardError => e
+              handle_exception(e, {
+                                 context: "AIRTABLE:RECORD:WRITE:EXCEPTION",
+                                 type: "error",
+                                 sync_id: sync_config.sync_id,
+                                 sync_run_id: sync_config.sync_run_id
+                               })
+              write_failure += chunk.size
+              log_message_array << log_request_response("error", args, e.message)
+            end
+            tracking_message(write_success, write_failure, log_message_array)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "AIRTABLE:RECORD:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+          end
+
+          private
+
+          def create_payload(records)
+            {
+              "records" => records.map do |record|
+                {
+                  "fields" => record
+                }
+              end
+            }
+          end
+
+          def base_id_exists?(bases, base_id)
+            return if extract_bases(bases).any? { |base| base["id"] == base_id }
+
+            raise ArgumentError, "base_id not found"
+          end
+
+          def extract_bases(response)
+            response_body = extract_body(response)
+            response_body["bases"] if response_body
+          end
+
+          def extract_body(response)
+            response_body = response.body
+            JSON.parse(response_body) if response_body
+          end
+
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
+          end
+
+          def create_stream(table, base_id, base_name)
+            {
+              name: "#{base_name}/#{SchemaHelper.clean_name(table["name"])}",
+              action: "create",
+              method: HTTP_POST,
+              url: "#{AIRTABLE_URL_BASE}#{base_id}/#{table["id"]}",
+              json_schema: SchemaHelper.get_json_schema(table),
+              supported_sync_modes: %w[incremental],
+              batch_support: true,
+              batch_size: 10
+
+            }.with_indifferent_access
+          end
+
+          def build_catalog_from_schema(schema, base_id, base_name)
+            catalog = build_catalog(load_catalog)
+            schema["tables"].each do |table|
+              catalog.streams << build_stream(create_stream(table, base_id, base_name))
+            end
+            catalog
+          end
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/airtable/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 300,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/outhad/integrations/destination/airtable/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Airtable",
+    "title": "Airtable",
+    "connector_type": "destination",
+    "category": "Productivity Tools",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/productivity-tools/airtable",
+    "github_issue_label": "destination-airtable",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/destination/airtable/config/spec.json

@@ -0,0 +1,23 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/productivity-tools/airtable",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Airtable",
+    "type": "object",
+    "required": ["api_key", "base_id"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Personal access token",
+        "order": 0
+      },
+      "base_id": {
+        "type": "string",
+        "title": "Airtable base id",
+        "order": 1
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/airtable/icon.svg

@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="27" shape-rendering="geometricPrecision" viewBox="0 0 200 170">
+  <path fill="#FCB400" d="M90.039 12.367L24.079 39.66c-3.667 1.519-3.63 6.729.062 8.192l66.235 26.266a24.575 24.575 0 0018.12 0l66.236-26.266c3.69-1.463 3.729-6.673.06-8.191l-65.958-27.294a24.578 24.578 0 00-18.795 0"></path>
+  <path fill="#18BFFF" d="M105.312 88.46v65.617c0 3.12 3.147 5.258 6.048 4.108l73.806-28.648a4.418 4.418 0 002.79-4.108V59.813c0-3.121-3.147-5.258-6.048-4.108l-73.806 28.648a4.42 4.42 0 00-2.79 4.108"></path>
+  <path fill="#F82B60" d="M88.078 91.846l-21.904 10.576-2.224 1.075-46.238 22.155c-2.93 1.414-6.672-.722-6.672-3.978V60.088c0-1.178.604-2.195 1.414-2.96a5.024 5.024 0 011.12-.84c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
+  <path fill="rgba(0, 0, 0, 0.25)" d="M88.078 91.846l-21.904 10.576-53.72-45.295a5.024 5.024 0 011.12-.839c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
+</svg>

data/lib/outhad/integrations/destination/airtable/schema_helper.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations
+    module Destination
+      module Airtable
+        module SchemaHelper
+          include Core::Constants
+
+          module_function
+
+          def clean_name(name_str)
+            name_str.strip.gsub(" ", "_")
+          end
+
+          def get_json_schema(table)
+            fields = table["fields"] || {}
+            properties = fields.each_with_object({}) do |field, props|
+              name, schema = process_field(field)
+              props[name] = schema
+            end
+
+            build_schema(properties)
+          end
+
+          def process_field(field)
+            name = clean_name(field.fetch("name", ""))
+            original_type = field.fetch("type", "")
+            options = field.fetch("options", {})
+
+            schema = determine_schema(original_type, options)
+            [name, schema]
+          end
+
+          def determine_schema(original_type, options)
+            if COMPLEX_AIRTABLE_TYPES.keys.include?(original_type)
+              complex_type = deep_copy(COMPLEX_AIRTABLE_TYPES[original_type])
+              adjust_complex_type(original_type, complex_type, options)
+            elsif SIMPLE_AIRTABLE_TYPES.keys.include?(original_type)
+              simple_type_schema(original_type, options)
+            else
+              SCHEMA_TYPES[:STRING]
+            end
+          end
+
+          def adjust_complex_type(original_type, complex_type, options)
+            exec_type = options.dig("result", "type") || "simpleText"
+            if complex_type == SCHEMA_TYPES[:ARRAY_WITH_ANY]
+              adjust_array_with_any(original_type, complex_type, exec_type, options)
+            else
+              complex_type
+            end
+          end
+
+          def adjust_array_with_any(original_type, complex_type, exec_type, options)
+            if original_type == "formula" && %w[number currency percent duration].include?(exec_type)
+              complex_type = SCHEMA_TYPES[:NUMBER]
+            elsif original_type == "formula" && ARRAY_FORMULAS.none? { |x| options.fetch("formula", "").start_with?(x) }
+              complex_type = SCHEMA_TYPES[:STRING]
+            elsif SIMPLE_AIRTABLE_TYPES.keys.include?(exec_type)
+              complex_type["items"] = deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
+            else
+              complex_type["items"] = SCHEMA_TYPES[:STRING]
+            end
+            complex_type
+          end
+
+          def simple_type_schema(original_type, options)
+            exec_type = options.dig("result", "type") || original_type
+            deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
+          end
+
+          def build_schema(properties)
+            {
+              "$schema" => JSON_SCHEMA_URL,
+              "type" => "object",
+              "additionalProperties" => true,
+              "properties" => properties
+            }
+          end
+
+          def deep_copy(object)
+            Marshal.load(Marshal.dump(object))
+          end
+
+          SCHEMA_TYPES = {
+            STRING: { "type": %w[null string] },
+            NUMBER: { "type": %w[null number] },
+            BOOLEAN: { "type": %w[null boolean] },
+            DATE: { "type": %w[null string], "format": "date" },
+            DATETIME: { "type": %w[null string], "format": "date-time" },
+            ARRAY_WITH_STRINGS: { "type": %w[null array], "items": { "type": %w[null string] } },
+            ARRAY_WITH_ANY: { "type": %w[null array], "items": {} }
+          }.freeze.with_indifferent_access
+
+          SIMPLE_AIRTABLE_TYPES = {
+            "multipleAttachments" => SCHEMA_TYPES[:STRING],
+            "autoNumber" => SCHEMA_TYPES[:NUMBER],
+            "barcode" => SCHEMA_TYPES[:STRING],
+            "button" => SCHEMA_TYPES[:STRING],
+            "checkbox" => :BOOLEAN,
+            "singleCollaborator" => SCHEMA_TYPES[:STRING],
+            "count" => SCHEMA_TYPES[:NUMBER],
+            "createdBy" => SCHEMA_TYPES[:STRING],
+            "createdTime" => SCHEMA_TYPES[:DATETIME],
+            "currency" => SCHEMA_TYPES[:NUMBER],
+            "email" => SCHEMA_TYPES[:STRING],
+            "date" => SCHEMA_TYPES[:DATE],
+            "dateTime" => SCHEMA_TYPES[:DATETIME],
+            "duration" => SCHEMA_TYPES[:NUMBER],
+            "lastModifiedBy" => SCHEMA_TYPES[:STRING],
+            "lastModifiedTime" => SCHEMA_TYPES[:DATETIME],
+            "multipleRecordLinks" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
+            "multilineText" => SCHEMA_TYPES[:STRING],
+            "multipleCollaborators" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
+            "multipleSelects" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
+            "number" => SCHEMA_TYPES[:NUMBER],
+            "percent" => SCHEMA_TYPES[:NUMBER],
+            "phoneNumber" => SCHEMA_TYPES[:STRING],
+            "rating" => SCHEMA_TYPES[:NUMBER],
+            "richText" => SCHEMA_TYPES[:STRING],
+            "singleLineText" => SCHEMA_TYPES[:STRING],
+            "singleSelect" => SCHEMA_TYPES[:STRING],
+            "externalSyncSource" => SCHEMA_TYPES[:STRING],
+            "url" => SCHEMA_TYPES[:STRING],
+            "simpleText" => SCHEMA_TYPES[:STRING]
+          }.freeze
+
+          COMPLEX_AIRTABLE_TYPES = {
+            "formula" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
+            "lookup" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
+            "multipleLookupValues" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
+            "rollup" => SCHEMA_TYPES[:ARRAY_WITH_ANY]
+          }.freeze.with_indifferent_access
+
+          ARRAY_FORMULAS = %w[ARRAYCOMPACT ARRAYFLATTEN ARRAYUNIQUE ARRAYSLICE].freeze
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/ais_data_store/client.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Outhad::Integrations::Destination
+  module AISDataStore
+    include Outhad::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_outhad_message
+      rescue PG::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name,
+                 CASE WHEN data_type = 'USER-DEFINED' THEN udt_name ELSE data_type END
+                 AS data_type,
+                 is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+        records = db.exec(query) do |result|
+          result.map do |row|
+            row
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AIS:DATA:STORE:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db&.close
+      end
+
+      def write(sync_config, records, action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        table_name = sync_config.stream.name
+        primary_key = sync_config.model.primary_key
+        log_message_array = []
+        db = create_connection(connection_config)
+
+        write_success = 0
+        write_failure = 0
+
+        records.each do |record|
+          query = Outhad::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
+          logger.debug("AIS:DATA:STORE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+          begin
+            response = db.exec(query)
+            write_success += 1
+            log_message_array << log_request_response("info", query, response)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "AIS:DATA:STORE:RECORD:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+            write_failure += 1
+            log_message_array << log_request_response("error", query, e.message)
+          end
+        end
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AIS:DATA:STORE:RECORD:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def query(connection, query)
+        connection.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+          end
+        end
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        PG.connect(
+          host: connection_config[:host],
+          dbname: connection_config[:database],
+          user: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          port: connection_config[:port]
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/ais_data_store/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "AISDataStore",
+    "title": "AIS Data Store",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/ais_data_store",
+    "github_issue_label": "destination-ais-data-store",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}