outhad-integrations 0.32.0

data/lib/outhad/integrations/protocol/protocol.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Protocol
+    module Types
+      include Dry.Types()
+    end
+
+    SyncMode = Types::String.enum("full_refresh", "incremental")
+    SyncStatus = Types::String.enum("started", "running", "complete", "incomplete")
+    DestinationSyncMode = Types::String.enum("insert", "upsert")
+    ConnectorType = Types::String.enum("source", "destination")
+    ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml", "vector_search")
+    ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml", "dynamic_sql", "unstructured", "vector_search")
+    ConnectionStatusType = Types::String.enum("succeeded", "failed")
+    StreamType = Types::String.enum("static", "dynamic", "user_defined")
+    StreamAction = Types::String.enum("fetch", "create", "update", "delete")
+    OuthadMessageType = Types::String.enum(
+      "record", "log", "connector_spec",
+      "connection_status", "catalog", "control",
+      "tracking"
+    )
+    ControlMessageType = Types::String.enum(
+      "rate_limit", "connection_config", "full_refresh"
+    )
+    LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
+    RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
+    SchemaMode = Types::String.enum("schema", "schemaless")
+    FileFormatType = Types::String.enum("csv")
+    CompressionType = Types::String.enum("un_compressed", "zip")
+
+    class ProtocolModel < Dry::Struct
+      extend Outhad::Integrations::Core::Utils
+      class << self
+        def from_json(json_string)
+          data = JSON.parse(json_string)
+          new(keys_to_symbols(data))
+        end
+      end
+    end
+
+    class ConnectionStatus < ProtocolModel
+      attribute :status, ConnectionStatusType
+      attribute? :message, Types::String.optional
+
+      def to_outhad_message
+        OuthadMessage.new(
+          type: OuthadMessageType["connection_status"],
+          connection_status: self
+        )
+      end
+    end
+
+    class ConnectorSpecification < ProtocolModel
+      attribute? :documentation_url, Types::String.optional
+      attribute? :changelog_url, Types::String.optional
+      attribute :connection_specification, Types::Hash
+      attribute :supports_normalization, Types::Bool.default(false)
+      attribute :supports_dbt, Types::Bool.default(false)
+      attribute :stream_type, StreamType
+      attribute? :supported_destination_sync_modes, Types::Array.of(DestinationSyncMode).optional
+      attribute? :connector_query_type, ConnectorQueryType
+
+      def to_outhad_message
+        OuthadMessage.new(
+          type: OuthadMessageType["connector_spec"],
+          connector_spec: self
+        )
+      end
+    end
+
+    class Connector < ProtocolModel
+      attribute :name, Types::String
+      attribute :type, ConnectorType
+      attribute :connection_specification, Types::Hash
+      attribute :connector_instance, Types::Any.optional.default(nil)
+      attribute :query_type, Types::String.default("raw_sql").enum(*ConnectorQueryType.values)
+    end
+
+    class LogMessage < ProtocolModel
+      attribute :level, LogLevel
+      attribute :message, Types::String
+      attribute? :name, Types::String.optional
+      attribute? :stack_trace, Types::String.optional
+
+      def to_outhad_message
+        OuthadMessage.new(
+          type: OuthadMessageType["log"],
+          log: self
+        )
+      end
+    end
+
+    class Model < ProtocolModel
+      attribute? :name, Types::String.optional
+      attribute :query, Types::String
+      attribute :query_type, ModelQueryType
+      attribute :primary_key, Types::String
+    end
+
+    class RecordMessage < ProtocolModel
+      attribute :data, Types::Hash
+      attribute :emitted_at, Types::Integer
+
+      def to_outhad_message
+        OuthadMessage.new(
+          type: OuthadMessageType["record"],
+          record: self
+        )
+      end
+    end
+
+    class Stream < ProtocolModel
+      # Common
+      attribute :name, Types::String
+      attribute? :action, StreamAction
+      attribute :json_schema, Types::Hash
+      attribute? :supported_sync_modes, Types::Array.of(SyncMode).optional.default(["incremental"])
+
+      # Applicable for database streams
+      attribute :source_defined_cursor, Types::Bool.default(false)
+      attribute? :default_cursor_field, Types::Array.of(Types::String).optional
+      attribute? :source_defined_primary_key, Types::Array.of(Types::Array.of(Types::String)).optional
+      attribute? :namespace, Types::String.optional
+      # Applicable for API streams
+      attribute? :url, Types::String.optional
+      attribute? :request_method, Types::String.optional
+      attribute :batch_support, Types::Bool.default(false)
+      attribute :batch_size, Types::Integer.default(1)
+      # Rate limits
+      attribute? :request_rate_limit, Types::Integer
+      attribute? :request_rate_limit_unit, RequestRateLimitingUnit
+      attribute? :request_rate_concurrency, Types::Integer
+
+      def rate_limit_unit_seconds
+        case request_rate_limit_unit
+        when "minute"
+          60 # Seconds in a minute
+        when "hour"
+          3600 # Seconds in an hour
+        when "day"
+          86_400 # Seconds in a day
+        else
+          1 # Default case, consider as seconds or handle as error
+        end
+      end
+    end
+
+    class Catalog < ProtocolModel
+      attribute :streams, Types::Array.of(Stream)
+
+      # Rate limits
+      attribute? :request_rate_limit, Types::Integer.default(60)
+      attribute? :request_rate_limit_unit, RequestRateLimitingUnit
+      attribute? :request_rate_concurrency, Types::Integer.default(10)
+      attribute? :schema_mode, Types::String.optional.default("schema")
+      attribute :source_defined_cursor, Types::Bool.default(false)
+      attribute? :default_cursor_field, Types::Array.of(Types::String).optional
+
+      def to_outhad_message
+        OuthadMessage.new(
+          type: OuthadMessageType["catalog"],
+          catalog: self
+        )
+      end
+    end
+
+    class SyncConfig < ProtocolModel
+      attr_accessor :offset, :limit, :sync_run_id
+
+      attribute :source, Connector
+      attribute :destination, Connector
+      attribute :model, Model
+      attribute :stream, Stream
+      attribute :sync_mode, SyncMode
+      attribute? :cursor_field, Types::String.optional
+      attribute? :current_cursor_field, Types::String.optional
+      attribute :destination_sync_mode, DestinationSyncMode
+      # reference ids
+      attribute :sync_id, Types::String.default("unknown")
+    end
+
+    class VectorConfig < ProtocolModel
+      attribute :source, Connector
+      attribute :vector, Types::Array.of(Types::Float)
+      attribute :limit, Types::Integer.default(1)
+    end
+
+    class ControlMessage < ProtocolModel
+      attribute :type, ControlMessageType
+      attribute :emitted_at, Types::Integer
+      attribute? :status, ConnectionStatusType.optional
+      attribute? :meta, Types::Hash
+
+      def to_outhad_message
+        OuthadMessage.new(
+          type: OuthadMessageType["control"],
+          control: self
+        )
+      end
+    end
+
+    class TrackingMessage < ProtocolModel
+      attribute :success, Types::Integer.default(0)
+      attribute :failed, Types::Integer.default(0)
+      attribute? :meta, Types::Hash
+      attribute? :logs, Types::Array.of(LogMessage)
+
+      def to_outhad_message
+        OuthadMessage.new(
+          type: OuthadMessageType["tracking"],
+          tracking: self
+        )
+      end
+    end
+
+    class OuthadMessage < ProtocolModel
+      attribute :type, OuthadMessageType
+      attribute? :log, LogMessage.optional
+      attribute? :connection_status, ConnectionStatus.optional
+      attribute? :connector_spec, ConnectorSpecification.optional
+      attribute? :catalog, Catalog.optional
+      attribute? :record, RecordMessage.optional
+      attribute? :control, ControlMessage.optional
+      attribute? :tracking, TrackingMessage.optional
+    end
+  end
+end

data/lib/outhad/integrations/rollout.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations
+    VERSION = "0.32.0"
+
+    ENABLED_SOURCES = %w[
+      Snowflake
+      Redshift
+      Bigquery
+      Postgresql
+      Databricks
+      SalesforceConsumerGoodsCloud
+      AwsAthena
+      Clickhouse
+      AmazonS3
+      MariaDB
+      Oracle
+      DatabricksModel
+      AwsSagemakerModel
+      VertexModel
+      HttpModel
+      OpenAI
+      Sftp
+      WatsonxAi
+      WatsonxData
+      Anthropic
+      AwsBedrockModel
+      GenericOpenAI
+      IntuitQuickBooks
+      PineconeDB
+      Qdrant
+      Firecrawl
+      Odoo
+    ].freeze
+
+    ENABLED_DESTINATIONS = %w[
+      Klaviyo
+      SalesforceCrm
+      FacebookCustomAudience
+      Slack
+      Hubspot
+      GoogleSheets
+      Airtable
+      Stripe
+      SalesforceConsumerGoodsCloud
+      Sftp
+      Postgresql
+      Zendesk
+      Http
+      Iterable
+      MariaDB
+      DatabricksLakehouse
+      Oracle
+      MicrosoftExcel
+      MicrosoftSql
+      Mailchimp
+      AISDataStore
+      AmazonS3
+      MicrosoftDynamics
+      Qdrant
+      PineconeDB
+      Odoo
+    ].freeze
+  end
+end

data/lib/outhad/integrations/service.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations
+    class Service
+      def initialize
+        yield(self.class.config) if block_given?
+      end
+      class << self
+        def connectors
+          {
+            source: build_connectors(
+              ENABLED_SOURCES, "Source"
+            ),
+            destination: build_connectors(
+              ENABLED_DESTINATIONS, "Destination"
+            )
+          }
+        end
+
+        def connector_class(connector_type, connector_name)
+          Object.const_get(
+            "Outhad::Integrations::#{connector_type}::#{connector_name}::Client"
+          )
+        end
+
+        def logger
+          config.logger || default_logger
+        end
+
+        def exception_reporter
+          config.exception_reporter
+        end
+
+        def config
+          @config ||= Config.new
+        end
+
+        private
+
+        def build_connectors(enabled_connectors, type)
+          enabled_connectors.map do |connector|
+            client = connector_class(type, connector).new
+            client.meta_data[:data][:connector_spec] = client.connector_spec.to_h
+            client.meta_data[:data]
+          end
+        end
+
+        def default_logger
+          @default_logger ||= Logger.new($stdout)
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/amazon_s3/client.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module AmazonS3
+    include Outhad::Integrations::Core
+    class Client < UnstructuredSourceConnector
+      @session_name = ""
+
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        @session_name = "connection-#{connection_config[:region]}-#{connection_config[:bucket]}"
+
+        if unstructured_data?(connection_config)
+          create_s3_connection(connection_config)
+          @s3_resource.bucket(connection_config[:bucket]).objects.limit(1).first
+        else
+          conn = create_connection(connection_config)
+          path = build_path(connection_config)
+          get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
+        end
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_outhad_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        @session_name = "discover-#{connection_config[:region]}-#{connection_config[:bucket]}"
+
+        streams = if unstructured_data?(connection_config)
+                    [create_unstructured_stream]
+                  else
+                    conn = create_connection(connection_config)
+                    # If pulling from multiple files, all files must have the same schema
+                    path = build_path(connection_config)
+                    records = get_results(conn, "DESCRIBE SELECT * FROM '#{path}';")
+                    columns = build_discover_columns(records)
+                    [Outhad::Integrations::Protocol::Stream.new(name: path, action: StreamAction["fetch"], json_schema: convert_to_json_schema(columns))]
+                  end
+        catalog = Catalog.new(streams: streams)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, { context: "AMAZONS3:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        @session_name = "#{sync_config.sync_id}-#{sync_config.source.name}-#{sync_config.destination.name}"
+
+        return handle_unstructured_data(sync_config) if unstructured_data?(connection_config)
+
+        conn = create_connection(connection_config)
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        query(conn, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AMAZONS3:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def get_auth_data(connection_config)
+        session = @session_name.gsub(/\s+/, "-")
+        @session_name = ""
+        if connection_config[:auth_type] == "user"
+          Aws::Credentials.new(connection_config[:access_id], connection_config[:secret_access])
+        elsif connection_config[:auth_type] == "role"
+          sts_client = Aws::STS::Client.new(region: connection_config[:region])
+          resp = sts_client.assume_role({
+                                          role_arn: connection_config[:arn],
+                                          role_session_name: session,
+                                          external_id: connection_config[:external_id]
+                                        })
+          Aws::Credentials.new(
+            resp.credentials.access_key_id,
+            resp.credentials.secret_access_key,
+            resp.credentials.session_token
+          )
+        end
+      end
+
+      def create_s3_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+
+        # Get authentication credentials
+        auth_data = get_auth_data(connection_config)
+
+        # Create S3 resource for easier operations
+        @s3_resource = Aws::S3::Resource.new(
+          region: connection_config[:region],
+          credentials: auth_data
+        )
+      end
+
+      def create_connection(connection_config)
+        # In the case when previewing a query
+        @session_name = "preview-#{connection_config[:region]}-#{connection_config[:bucket]}" if @session_name.to_s.empty?
+        auth_data = get_auth_data(connection_config)
+        conn = DuckDB::Database.open.connect
+        # Install and/or Load the HTTPFS extension
+        conn.execute(INSTALL_HTTPFS_QUERY)
+        # Set up S3 configuration
+        secret_query = "
+              CREATE SECRET amazons3_source (
+              TYPE S3,
+              KEY_ID '#{auth_data.credentials.access_key_id}',
+              SECRET '#{auth_data.credentials.secret_access_key}',
+              REGION '#{connection_config[:region]}',
+              SESSION_TOKEN '#{auth_data.credentials.session_token}'
+          );
+        "
+        get_results(conn, secret_query)
+        conn
+      end
+
+      def build_path(connection_config)
+        path = connection_config[:path]
+        path = "#{path}/" if path.to_s.strip.empty? || path[-1] != "/"
+        "s3://#{connection_config[:bucket]}#{path}*.#{connection_config[:file_type]}"
+      end
+
+      def get_results(conn, query)
+        results = conn.query(query)
+        hash_array_values(results)
+      end
+
+      def query(conn, query)
+        records = get_results(conn, query)
+        records.map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+        end
+      end
+
+      def hash_array_values(describe)
+        keys = describe.columns.map(&:name)
+        describe.map do |row|
+          Hash[keys.zip(row)]
+        end
+      end
+
+      def build_discover_columns(describe_results)
+        describe_results.map do |row|
+          type = column_schema_helper(row["column_type"])
+          {
+            column_name: row["column_name"],
+            type: type
+          }
+        end
+      end
+
+      def column_schema_helper(column_type)
+        case column_type
+        when "VARCHAR", "BIT", "DATE", "TIME", "TIMESTAMP", "UUID"
+          "string"
+        when "DOUBLE"
+          "number"
+        when "BIGINT", "HUGEINT", "INTEGER", "SMALLINT"
+          "integer"
+        when "BOOLEAN"
+          "boolean"
+        end
+      end
+
+      def handle_unstructured_data(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        bucket_name = connection_config[:bucket]
+        command = sync_config.model.query.strip
+        create_s3_connection(connection_config)
+
+        case command
+        when LIST_FILES_CMD
+          list_files_in_folder(bucket_name, connection_config[:path] || "")
+        when /^#{DOWNLOAD_FILE_CMD}\s+(.+)$/
+          # Extract the file path and remove surrounding quotes if present
+          file_path = ::Regexp.last_match(1).strip
+          file_path = file_path.gsub(/^["']|["']$/, "") # Remove leading/trailing quotes
+          download_file_to_local(bucket_name, file_path, sync_config.sync_id)
+        else
+          raise "Invalid command. Supported commands: #{LIST_FILES_CMD}, #{DOWNLOAD_FILE_CMD} <file_path>"
+        end
+      end
+
+      def list_files_in_folder(bucket_name, folder_path)
+        folder_path = folder_path.end_with?("/") ? folder_path : "#{folder_path}/"
+        bucket = @s3_resource.bucket(bucket_name)
+
+        bucket.objects(prefix: folder_path).reject { |object| object.key == folder_path }.map do |object|
+          RecordMessage.new(
+            data: {
+              file_name: File.basename(object.key),
+              file_path: object.key,
+              size: object.content_length,
+              file_type: File.extname(object.key).sub(".", ""),
+              created_date: object.last_modified.to_s,
+              modified_date: object.last_modified.to_s
+            },
+            emitted_at: Time.now.to_i
+          ).to_outhad_message
+        end
+      end
+
+      def download_file_to_local(bucket_name, file_path, sync_id)
+        download_path = ENV["FILE_DOWNLOAD_PATH"]
+        file = if download_path
+                 File.join(download_path, "syncs", sync_id, File.basename(file_path))
+               else
+                 Tempfile.new(["s3_file", "syncs", sync_id, File.extname(file_path)]).path
+               end
+
+        object = @s3_resource.bucket(bucket_name).object(file_path)
+        object.get(response_target: file)
+
+        [RecordMessage.new(
+          data: {
+            local_path: file,
+            file_name: File.basename(file_path),
+            file_path: file_path,
+            size: object.content_length,
+            file_type: File.extname(file_path).sub(".", ""),
+            modified_date: object.last_modified.to_s,
+            created_date: object.last_modified.to_s
+          },
+          emitted_at: Time.now.to_i
+        ).to_outhad_message]
+      rescue Aws::S3::Errors::NoSuchKey
+        raise "File not found: #{file_path}"
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/amazon_s3/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "AmazonS3",
+    "title": "Amazon S3",
+    "connector_type": "source",
+    "category": "Data Lake",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/amazon_s3",
+    "github_issue_label": "source-amazons3",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}