outhad-integrations 0.32.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 856eb97fb81feabc784eceb9abcbc826905dd7dc89f6bd6dfd3bf0cf5864835d
+  data.tar.gz: 6250c3405ae7422647f746879a4ed52371e85f802a4d7fe73454722d4526b1a1
+SHA512:
+  metadata.gz: 37f12681f0f97a1d7a5380e1558e1d938148f4f5f07387e13f1649d0dcd07406f6e8f8d9e5099f0ca3cafc30dba424847b39788837a84f108dd7e91bc3f3089e
+  data.tar.gz: 99863a1fcfa79beb365e1cb09a153ea1455e0b0cc443da5a732e3f0eee19197e3aa79443ccf27f4776b2f441bdd5443ba2c777d6cc4e01106b0a4803f39924b0

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,43 @@
+AllCops:
+  TargetRubyVersion: 2.6
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 300
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Max: 500
+
+Metrics/BlockLength:
+  Max: 120
+
+Metrics/AbcSize:
+  Max: 30
+
+Metrics/MethodLength:
+  Max: 50
+
+Metrics/CyclomaticComplexity:
+  Max: 50
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+3.2.0

data/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "cSpell.words": [
+    "multiwoven"
+  ]
+}

data/README.md

@@ -0,0 +1,76 @@
+
+## Outhad Integrations
+
+Outhad integrations is the collection of connectors built on top of [Outhad protocol](https://docs.outhad.com/guides/architecture/outhad-protocol).
+
+Outhad protocol is an open source standard for moving data between data sources to any third-part destinations.
+Anyone can build a connetor with basic ruby knowledge using the protocol.
+
+## Prerequisites
+
+Before you begin the installation, ensure you have the following dependencies installed:
+
+- **MySQL Client**
+  - Command: `brew install mysql-client`
+  - Description: Required for database interactions.
+
+- **Zstandard (zstd)**
+  - Command: `brew install zstd`
+  - Description: Needed for data compression and decompression.
+
+- **OpenSSL 3**
+  - Command: `brew install openssl@3`
+  - Description: Essential for secure communication.
+
+- **Oracle Instant Client**
+  - Download Link: https://www.oracle.com/database/technologies/instant-client/downloads.html
+  - Description: Required for database interactions.
+
+
+### Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add outhad-integrations
+
+
+### Usage
+
+#### Source
+```
+source = Outhad::Integrations::Source::[CONNECTOR_NAME]::Client.new
+source.read(sync_config)
+```
+#### Destination
+
+```
+destination = Outhad::Integrations::Destination::[CONNECTOR_NAME]::Client.new
+destination.write(sync_config, records)
+```
+
+#### Supported methods 
+Please refer [Outhad Protocol](https://docs.outhad.com/guides/architecture/outhad-protocol) to understand more about the supported methods on source and destination
+
+## Development
+
+- **Install Dependencies**
+  - Command: `bin/setup`
+  - Description: After checking out the repo, run this command to install dependencies.
+
+- **Run Tests**
+  - Command: `rake spec`
+  - Description: Run this command to execute the tests.
+
+- **Interactive Prompt**
+  - Command: `bin/console`
+  - Description: For an interactive prompt that allows you to experiment, run this command.
+
+- **Install Gem Locally**
+  - Command: `bundle exec rake install`
+  - Description: To install this gem onto your local machine, run this command.
+
+- **Release New Version**
+  - Steps:
+    1. Update the version number in `rollout.rb`.
+    2. Command: `bundle exec rake release`
+    3. Description: This command will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/lib/outhad/integrations/config.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations
+    class Config
+      attr_accessor :logger, :exception_reporter
+
+      def initialize(params = {})
+        @logger = params[:logger]
+        @exception_reporter = params[:exception_reporter]
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/base_connector.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    class BaseConnector
+      include Integrations::Protocol
+      include Utils
+      include Constants
+
+      def connector_spec
+        @connector_spec ||= begin
+          spec_json = keys_to_symbols(read_json(CONNECTOR_SPEC_PATH)).to_json
+          # returns Protocol::ConnectorSpecification
+          ConnectorSpecification.from_json(spec_json)
+        end
+      end
+
+      def meta_data
+        client_meta_data = read_json(META_DATA_PATH).deep_symbolize_keys
+        icon_name = client_meta_data[:data][:icon]
+        icon_url = "https://raw.githubusercontent.com/Outhad/outhad/main/integrations#{relative_path}/#{icon_name}"
+        client_meta_data[:data][:icon] = icon_url
+        # returns hash
+        @meta_data ||= client_meta_data
+      end
+
+      def relative_path
+        path = Object.const_source_location(self.class.to_s)[0]
+        connector_folder = File.dirname(path)
+        marker = "/lib/outhad/integrations/"
+        parts = connector_folder.split(marker)
+
+        marker + parts.last if parts.length > 1
+      end
+
+      # Connection config is a hash
+      def check_connection(_connection_config)
+        raise "Not implemented"
+        # returns Protocol.ConnectionStatus
+      end
+
+      # Connection config is a hash
+      def discover(_connection_config)
+        raise "Not implemented"
+        # returns Protocol::Catalog
+      end
+
+      private
+
+      def read_json(file_path)
+        path = Object.const_source_location(self.class.to_s)[0]
+        connector_folder = File.dirname(path)
+        file_path = File.join(
+          "#{connector_folder}/",
+          file_path
+        )
+        file_contents = File.read(file_path)
+        JSON.parse(file_contents)
+      end
+
+      def success_status
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_outhad_message
+      end
+
+      def failure_status(error)
+        message = error&.message || "failed"
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: message).to_outhad_message
+      end
+
+      def auth_headers(access_token)
+        {
+          "Accept" => "application/json",
+          "Authorization" => "Bearer #{access_token}",
+          "Content-Type" => "application/json"
+        }
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/constants.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    module Constants
+      # CONFIG
+      META_DATA_PATH = "config/meta.json"
+      CONNECTOR_SPEC_PATH = "config/spec.json"
+      CATALOG_SPEC_PATH   = "config/catalog.json"
+      SNOWFLAKE_MAC_DRIVER_PATH = "/opt/snowflake/snowflakeodbc/lib/universal/libSnowflake.dylib"
+      DATABRICKS_MAC_DRIVER_PATH = "/Library/simba/spark/lib/libsparkodbc_sb64-universal.dylib"
+
+      SNOWFLAKE_DRIVER_PATH = ENV["SNOWFLAKE_DRIVER_PATH"] || SNOWFLAKE_MAC_DRIVER_PATH
+      DATABRICKS_DRIVER_PATH = ENV["DATABRICKS_DRIVER_PATH"] || DATABRICKS_MAC_DRIVER_PATH
+
+      JSON_SCHEMA_URL = "https://json-schema.org/draft-07/schema#"
+
+      # CONNECTORS
+      INSTALL_HTTPFS_QUERY = ENV["INSTALL_HTTPFS_QUERY"] || "INSTALL HTTPFS; LOAD HTTPFS;"
+
+      KLAVIYO_AUTH_ENDPOINT = "https://a.klaviyo.com/api/lists/"
+      KLAVIYO_AUTH_PAYLOAD = {
+        data: {
+          type: "list",
+          attributes: {
+            name: "THIS IS REQUIRED"
+          }
+        }
+      }.freeze
+
+      ZENDESK_URL_SUFFIX = "zendesk.com/api/v2/"
+
+      FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS = "https://graph.facebook.com/v18.0/me/adaccounts?fields=id,name"
+
+      AIRTABLE_URL_BASE = "https://api.airtable.com/v0/"
+      AIRTABLE_BASES_ENDPOINT = "https://api.airtable.com/v0/meta/bases"
+      AIRTABLE_GET_BASE_SCHEMA_ENDPOINT = "https://api.airtable.com/v0/meta/bases/{baseId}/tables"
+
+      MS_EXCEL_AUTH_ENDPOINT = "https://graph.microsoft.com/v1.0/me"
+      MS_EXCEL_TABLE_ROW_WRITE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets/%<sheet_name>s/tables/%<table_name>s/rows"
+      MS_EXCEL_TABLE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/workbook/"\
+      "worksheets/%<sheet_name>s/tables?$select=name"
+      MS_EXCEL_FILES_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/root/children"
+      MS_EXCEL_WORKSHEETS_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets"
+      MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets/%<sheet_name>s/range(address='A1:Z1')/usedRange?$select=values"
+      MS_DYNAMICS_WHOAMI_API = "https://%<instance_url>s.crm.dynamics.com/api/data/v%<api_version>s/WhoAmI"
+      MS_DYNAMICS_REST_API = "https://%<instance_url>s.crm.dynamics.com/api/data/v%<api_version>s/%<entity>s"
+
+      DATABRICKS_HEALTH_URL  = "https://%<databricks_host>s/api/2.0/serving-endpoints/%<endpoint_name>s"
+      DATABRICKS_SERVING_URL = "https://%<databricks_host>s/serving-endpoints/%<endpoint_name>s/invocations"
+
+      GOOGLE_VERTEX_ENDPOINT_SERVICE_URL = "%<region>s-aiplatform.googleapis.com"
+      GOOGLE_VERTEX_MODEL_NAME = "projects/%<project_id>s/locations/%<region>s/endpoints/%<endpoint_id>s"
+
+      WATSONX_HEALTH_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v4/deployments?version=%<version>s"
+      WATSONX_PREDICTION_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v4/deployments/%<deployment_id>s/predictions?version=%<version>s"
+      WATSONX_GENERATION_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v1/deployments/%<deployment_id>s/text/generation?version=%<version>s"
+      WATSONX_STREAM_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v1/deployments/%<deployment_id>s/text/generation_stream?version=%<version>s"
+
+      WATSONX_DATA_QUERIES_URL = "https://%<region>s.lakehouse.cloud.ibm.com/lakehouse/api/v2/queries/execute/%<engine_id>s"
+
+      # HTTP
+      HTTP_GET = "GET"
+      HTTP_POST = "POST"
+      HTTP_PUT = "PUT"
+      HTTP_DELETE = "DELETE"
+      HTTP_PATCH = "PATCH"
+
+      # google sheets
+      GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive"
+      GOOGLE_SPREADSHEET_ID_REGEX = %r{/d/([-\w]{20,})/}.freeze
+
+      OPEN_AI_URL = "https://api.openai.com/v1/chat/completions"
+      ANTHROPIC_URL = "https://api.anthropic.com/v1/messages"
+
+      # Bedrock Models
+      MISTRAL_AI_MODEL = %w[
+        mistral.mistral-large-2402-v1:0
+        mistral.mistral-7b-instruct-v0:2
+        mistral.mixtral-8x7b-instruct-v0:1
+        mistral.mistral-small-2402-v1:0
+      ].freeze
+
+      # Intuit QuickBooks
+      QUICKBOOKS_SANDBOX_QUERY_URL = "https://sandbox-quickbooks.api.intuit.com/v3/company/%<realm_id>s/query?query=%<query>s"
+      QUICKBOOKS_PRODUCTION_QUERY_URL = "https://quickbooks.api.intuit.com/v3/company/%<realm_id>s/query?query=%<query>s"
+      QUICKBOOKS_REDIRECT_URL = "https://developer.intuit.com/v2/OAuth2Playground/RedirectUrl"
+
+      # Qdrant
+      QDRANT_SEARCH_URL = "%<host>s/collections/%<collection_name>s/points/search"
+
+      # Firecrawl
+      FIRECRAWL_CRAWL_URL = "https://api.firecrawl.dev/v1/crawl"
+      FIRECRAWL_SCRAPE_URL = "https://api.firecrawl.dev/v1/scrape"
+      FIRECRAWL_CRAWL_ACTIVE_URL = "https://api.firecrawl.dev/v1/crawl/active"
+      FIRECRAWL_GET_CRAWL_URL = "https://api.firecrawl.dev/v1/crawl/%<id>s"
+      FIRECRAWL_REQUEST_RATE_LIMIT = 5
+    end
+  end
+end

data/lib/outhad/integrations/core/destination_connector.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    class DestinationConnector < BaseConnector
+      # Records are transformed json payload send it to the destination
+      # SyncConfig is the Protocol::SyncConfig object
+      def write(_sync_config, _records, _action = "destination_insert")
+        raise "Not implemented"
+        # return Protocol::TrackingMessage
+      end
+
+      def tracking_message(success, failure, log_message_array)
+        Outhad::Integrations::Protocol::TrackingMessage.new(
+          success: success, failed: failure, logs: log_message_array
+        ).to_outhad_message
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/fullrefresher.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    module Fullrefresher
+      def write(sync_config, records, action = "destination_insert")
+        if sync_config && sync_config.sync_mode == "full_refresh" && !@full_refreshed
+          response = clear_all_records(sync_config)
+          return response unless response &&
+                                 response.control.status == Outhad::Integrations::Protocol::ConnectionStatusType["succeeded"]
+
+          @full_refreshed = true
+        end
+
+        super(sync_config, records, action)
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/http_client.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    class HttpClient
+      extend HttpHelper
+      class << self
+        def request(url, method, payload: nil, headers: {}, config: {})
+          uri = URI(url)
+          http = configure_http(uri, config)
+          request = build_request(method, uri, payload, headers)
+          http.request(request)
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/http_helper.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    module HttpHelper
+      def build_request(method, uri, payload, headers)
+        request_class = case method.upcase
+                        when Constants::HTTP_GET then Net::HTTP::Get
+                        when Constants::HTTP_POST then Net::HTTP::Post
+                        when Constants::HTTP_PUT then Net::HTTP::Put
+                        when Constants::HTTP_PATCH then Net::HTTP::Patch
+                        when Constants::HTTP_DELETE then Net::HTTP::Delete
+                        else raise ArgumentError, "Unsupported HTTP method: #{method}"
+                        end
+
+        request = request_class.new(uri)
+        headers.each { |key, value| request[key] = value }
+        request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase)
+        request
+      end
+
+      def configure_http(uri, config)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+
+        if config[:timeout]
+          timeout_value = config[:timeout].to_f
+          http.open_timeout = timeout_value
+          http.read_timeout = timeout_value
+        end
+
+        http
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/query_builder.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    class QueryBuilder
+      include Utils
+
+      def self.perform(action, table, record, primary_key = nil)
+        case action.downcase
+        when "destination_insert"
+          columns = record.keys.join(", ")
+          values = record.values.map { |value| "'#{value}'" }.join(", ")
+          # TODO: support bulk insert
+          "INSERT INTO #{table} (#{columns}) VALUES (#{values});"
+        when "destination_update"
+          # Ensure primary key is a string and exists within record for the WHERE clause
+          if record[primary_key].nil?
+            error_message = "Primary key '#{primary_key}' not found in record."
+            Integrations::Service.logger.error(error_message)
+            return error_message
+          end
+
+          primary_key_value = record.delete(primary_key) # Remove and return the primary key value
+          set_clause = record.map { |key, value| "#{key} = '#{value}'" }.join(", ")
+          where_clause = "#{primary_key} = '#{primary_key_value}'"
+          "UPDATE #{table} SET #{set_clause} WHERE #{where_clause};"
+        else
+          "Invalid action specified."
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/rate_limiter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    module RateLimiter
+      def write(sync_config, records, action = "destination_insert")
+        stream = sync_config.stream
+
+        @queue ||= Limiter::RateQueue.new(stream.request_rate_limit, interval: stream.rate_limit_unit_seconds) do
+          Integrations::Service.logger.info("Hit the limit for stream: #{stream.name}, waiting")
+        end
+
+        @queue.shift
+
+        super(sync_config, records, action)
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/source_connector.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    class SourceConnector < BaseConnector
+      # accepts Protocol::SyncConfig
+      def read(_sync_config)
+        raise "Not implemented"
+        # setup sync configs
+        # call query(connection, query)
+        # Returns list of RecordMessage
+      end
+
+      private
+
+      # This needs to be implemented as private method
+      # In every source connector. This will be used for model preview
+      def create_connection(connector_config)
+        # return a connection to the client's source
+      end
+
+      # This needs to be implemented as private method
+      # In every source connector. This will be used for model preview
+      def query(connection, query)
+        # return list of RecordMessage
+      end
+
+      def batched_query(sql_query, limit, offset)
+        offset = offset.to_i
+        limit = limit.to_i
+        raise ArgumentError, "Offset and limit must be non-negative" if offset.negative? || limit.negative?
+
+        # Removing any trailing semicolons
+        sql_query.chomp!(";")
+
+        # Checking if the query already has a LIMIT clause
+        raise ArgumentError, "Query already contains a LIMIT clause" if sql_query.match?(/LIMIT \d+/i)
+
+        # Appending the LIMIT and OFFSET clauses to the SQL query
+        "#{sql_query} LIMIT #{limit} OFFSET #{offset}"
+      end
+
+      def send_request(options = {})
+        Outhad::Integrations::Core::HttpClient.request(
+          options[:url],
+          options[:http_method],
+          payload: options[:payload],
+          headers: options[:headers],
+          config: options[:config]
+        )
+      end
+
+      def send_streaming_request(options = {})
+        Outhad::Integrations::Core::StreamingHttpClient.request(
+          options[:url],
+          options[:http_method],
+          payload: options[:payload],
+          headers: options[:headers],
+          config: options[:config]
+        ) do |chunk|
+          yield chunk if block_given? # Pass each chunk for processing (streaming response)
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/streaming_http_client.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    class StreamingHttpClient
+      extend HttpHelper
+      class << self
+        def request(url, method, payload: nil, headers: {}, config: {})
+          uri = URI(url)
+          http = configure_http(uri, config)
+          request = build_request(method, uri, payload, headers)
+          http.request(request) do |response|
+            response.read_body do |chunk|
+              yield chunk if block_given? # Pass each response chunk
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/core/unstructured_source_connector.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations::Core
+    class UnstructuredSourceConnector < SourceConnector
+      UNSTRUCTURED_SCHEMA = {
+        type: "object",
+        properties: {
+          element_id: { type: "string" },
+          text: { type: "string" },
+          created_date: { type: "string" },
+          modified_date: { type: "string" },
+          filename: { type: "string" },
+          filetype: { type: "string" }
+        },
+        required: %w[
+          element_id
+          text
+          created_date
+          modified_date
+          filename
+          filetype
+        ]
+      }.freeze
+
+      UNSTRUCTURED_STREAM_CONFIG = {
+        supported_sync_modes: ["incremental"],
+        source_defined_cursor: true,
+        default_cursor_field: ["modified_date"],
+        source_defined_primary_key: [["element_id"]]
+      }.freeze
+
+      # Commands for unstructured data operations
+      UNSTRUCTURED = "unstructured"
+      LIST_FILES_CMD = "list_files"
+      DOWNLOAD_FILE_CMD = "download_file"
+
+      def unstructured_data?(connection_config)
+        connection_config["data_type"] == UNSTRUCTURED
+      end
+
+      def create_unstructured_stream
+        Outhad::Integrations::Protocol::Stream.new(
+          name: UNSTRUCTURED,
+          action: StreamAction["fetch"],
+          json_schema: UNSTRUCTURED_SCHEMA,
+          **UNSTRUCTURED_STREAM_CONFIG
+        )
+      end
+    end
+  end
+end