outhad-integrations 0.32.0

data/lib/outhad/integrations/source/watsonx_data/client.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module WatsonxData
+    include Outhad::Integrations::Core
+    API_VERSION = "2021-05-01"
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        create_connection(connection_config)
+        response = execute_query(connection_config, "show catalogs")
+        success?(response) ? success_status : failure_status(nil)
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX DATA:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name,
+                  data_type,
+                  is_nullable
+                  FROM information_schema.columns
+                  WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
+                  ORDER BY table_name, ordinal_position"
+        response = execute_query(connection_config, query)
+        records = JSON.parse(response.body)["response"]["result"]
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX DATA:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        if connection_config[:engine] == "presto"
+          query = batched_query_for_presto(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        else
+          query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        end
+        query(connection_config, query)
+      rescue StandardError => e
+        handle_exception(e, { context: "WATSONX DATA:READ:EXCEPTION", type: "error" })
+      end
+
+      private
+
+      def batched_query_for_presto(query, limit, offset)
+        <<~SQL
+          SELECT * FROM (
+            SELECT *, ROW_NUMBER() OVER () as rownum FROM ( #{query} ) subquery
+          ) t
+          WHERE rownum > #{offset}
+          LIMIT #{limit}
+        SQL
+      end
+
+      def execute_query(connection_config, query)
+        connection_config.with_indifferent_access
+        get_access_token(connection_config[:api_key])
+        url = format(
+          WATSONX_DATA_QUERIES_URL,
+          region: connection_config[:region],
+          engine_id: connection_config[:engine_id]
+        )
+        headers = auth_headers(@access_token)
+        headers["AuthInstanceId"] = connection_config[:auth_instance_id]
+        send_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: {
+            sql_string: query,
+            catalog_name: connection_config[:database],
+            schema_name: connection_config[:schema]
+          },
+          headers: headers,
+          config: connection_config[:config]
+        )
+      end
+
+      def query(connection, query)
+        response = execute_query(connection, query)
+        response = JSON.parse(response.body).with_indifferent_access
+        records = response[:response][:result]
+        records.map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+        end
+      end
+
+      def create_connection(connection_config)
+        connection_config
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+
+      def get_access_token(api_key)
+        cache = defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : ActiveSupport::Cache::MemoryStore.new
+        cache_key = "watsonx_data_#{api_key}"
+        cached_token = cache.read(cache_key)
+        if cached_token
+          @access_token = cached_token
+        else
+          new_token = get_iam_token(api_key)
+          # puts new_token
+          # max expiration is 3 minutes. No way to make it higher
+          cache.write(cache_key, new_token, expires_in: 180)
+          @access_token = new_token
+        end
+      end
+
+      def get_iam_token(api_key)
+        uri = URI("https://iam.cloud.ibm.com/identity/token")
+        request = Net::HTTP::Post.new(uri)
+        request["Content-Type"] = "application/x-www-form-urlencoded"
+        request.body = "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=#{api_key}"
+        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+
+        raise "Failed to get IAM token: #{response.body}" unless response.is_a?(Net::HTTPSuccess)
+
+        JSON.parse(response.body)["access_token"]
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/watsonx_data/config/meta.json

@@ -0,0 +1,17 @@
+{
+    "data": {
+      "name": "WatsonxData",
+      "title": "WatsonX Data Endpoint",
+      "connector_type": "source",
+      "category": "Data Warehouse",
+      "sub_category": "Relational Database",
+      "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/watsonx_data",
+      "github_issue_label": "source-watsonx-data-endpoint",
+      "icon": "icon.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "outhad"]
+    }
+  }
+  

data/lib/outhad/integrations/source/watsonx_data/config/spec.json

@@ -0,0 +1,72 @@
+{
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/watsonx_data",
+    "stream_type": "dynamic",
+    "connector_query_type": "raw_sql",
+    "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "WatsonX Data Endpoint",
+      "type": "object",
+      "required": ["api_key","region","engine","engine_id","auth_instance_id","database","schema"],
+      "properties": {
+        "api_key": {
+          "type": "string",
+          "outhad_secret": true,
+          "title": "API Key",
+          "order": 0
+        },
+        "region": {
+          "description": "WatsonX Data region",
+          "type": "string",
+          "title": "Region",
+          "order": 1
+        },
+        "engine": {
+          "description": "Which engine is being used? (Presto/Spark)",
+          "type": "string",
+          "title": "Engine",
+          "enum": ["presto"],
+          "default": "presto",
+          "order": 2
+        },
+        "engine_id": {
+          "description": "Engine id",
+          "type": "string",
+          "title": "Engine Id",
+          "order": 3
+        },
+        "auth_instance_id": {
+          "description": "WatsonX Data Instance CRN",
+          "type": "string",
+          "title": "Instance CRN",
+          "order": 4
+        },
+        "database": {
+          "description": "The specific database to connect to.",
+          "type": "string",
+          "title": "Database",
+          "order": 5
+        },
+        "schema": {
+          "description": "The schema within the database.",
+          "type": "string",
+          "title": "Schema",
+          "order": 6
+        },
+        "config": {
+          "title": "",
+          "type": "object",
+          "properties": {
+            "timeout": {
+              "type": "string",
+              "default": "30",
+              "title": "HTTP Timeout",
+              "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
+              "order": 0
+            }
+          },
+          "order": 7
+        }
+      }
+    }
+  }
+  

data/lib/outhad/integrations/source/watsonx_data/icon.svg

@@ -0,0 +1 @@
+<svg id="Watsonx-Data--Streamline-Carbon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" height="16" width="16"><desc>Watsonx Data Streamline Icon: https://streamlinehq.com</desc><defs></defs><path d="M26 24c-1.1046 0-2 .8954-2 2 0 .0764.0142.1488.0225.2229C21.7417 28.0192 18.9433 29 16 29c-2.7746 0-5.3432-.881-7.4566-2.3676.2576.0261.517.0444.7798.0444C13.5561 26.6768 17 23.233 17 19h-2c0 3.1304-2.5464 5.6768-5.6768 5.6768-2.2111 0-4.1977-1.2816-5.1318-3.2725-.1365-.2972-.2595-.6007-.3738-.9094C4.4778 20.8169 5.2174 21 6 21c2.7568 0 5-2.2432 5-5v-2H9v2c0 1.6543-1.3457 3-3 3s-3-1.3457-3-3c0-2.1152.4917-4.1328 1.4619-5.9956l-1.7744-.9238C1.5835 11.2017 1 13.5943 1 16c0 8.271 6.729 15 15 15 3.3744 0 6.5818-1.1193 9.2048-3.1662.244.106.5123.1662.7952.1662 1.1046 0 2-.8954 2-2s-.8954-2-2-2Z" fill="#000000"></path><path transform="rotate(90 22 22)" d="M21 21h2v2h-2Z" fill="#000000"></path><path transform="rotate(90 16 16)" d="M15 15h2v2h-2Z" fill="#000000"></path><path transform="rotate(-90 10 10)" d="M9 9h2v2H9Z" fill="#000000"></path><path d="M16 1c-3.3744 0-6.5818 1.1193-9.2048 3.1662C6.5512 4.0602 6.2829 4 6 4c-1.1046 0-2 .8954-2 2s.8954 2 2 2 2-.8954 2-2c0-.0764-.0142-.1488-.0225-.2229C10.2583 3.9808 13.0567 3 16 3c2.7708 0 5.3363.8784 7.4481 2.3613-.249-.0242-.5005-.038-.7547-.038-4.2329 0-7.6768 3.4438-7.6768 7.6768h2c0-3.1304 2.5464-5.6768 5.6768-5.6768 2.0554 0 3.9068 1.0953 4.9186 2.8651.2153.4283.4053.8701.5729 1.3237C27.5234 11.1887 26.7844 11 26 11c-2.7568 0-5 2.2432-5 5v2h2v-2c0-1.6543 1.3457-3 3-3s3 1.3457 3 3c0 2.1152-.4917 4.1328-1.4619 5.9956l1.7744.9238C30.4165 20.7983 31 18.4057 31 16c0-8.271-6.729-15-15-15Z" fill="#000000"></path><path id="_Transparent_Rectangle_" d="M0 0h32v32H0Z" fill="none"></path></svg>

data/lib/outhad/integrations.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "json"
+require "dry-struct"
+require "dry-schema"
+require "dry-types"
+require "odbc"
+require "sequel"
+require "byebug"
+require "net/http"
+require "uri"
+require "active_support/core_ext/hash/indifferent_access"
+require "restforce"
+require "logger"
+require "slack-ruby-client"
+require "git"
+require "ruby-limiter"
+require "hubspot-api-client"
+require "google/apis/sheets_v4"
+require "stringio"
+require "stripe"
+require "net/sftp"
+require "csv"
+require "securerandom"
+require "aws-sdk-athena"
+require "zip"
+require "zendesk_api"
+require "faraday"
+require "base64"
+require "aws-sdk-s3"
+require "duckdb"
+require "iterable-api-client"
+require "aws-sdk-sts"
+require "ruby-oci8"
+require "aws-sdk-sagemaker"
+require "aws-sdk-sagemakerruntime"
+require "google/cloud/ai_platform/v1"
+require "grpc"
+require "MailchimpMarketing"
+require "aws-sdk-bedrockruntime"
+require "pinecone"
+require "intuit-oauth"
+require "nokogiri"
+require "xmlrpc/client"
+
+# Service
+require_relative "integrations/config"
+require_relative "integrations/rollout"
+require_relative "integrations/service"
+
+# Core
+require_relative "integrations/core/constants"
+require_relative "integrations/core/utils"
+require_relative "integrations/core/rate_limiter"
+require_relative "integrations/core/fullrefresher"
+require_relative "integrations/protocol/protocol"
+require_relative "integrations/core/base_connector"
+require_relative "integrations/core/source_connector"
+require_relative "integrations/core/destination_connector"
+require_relative "integrations/core/http_helper"
+require_relative "integrations/core/http_client"
+require_relative "integrations/core/streaming_http_client"
+require_relative "integrations/core/query_builder"
+require_relative "integrations/core/unstructured_source_connector"
+require_relative "integrations/core/vector_source_connector"
+
+# Source
+require_relative "integrations/source/snowflake/client"
+require_relative "integrations/source/redshift/client"
+require_relative "integrations/source/bigquery/client"
+require_relative "integrations/source/postgresql/client"
+require_relative "integrations/source/databricks/client"
+require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
+require_relative "integrations/source/aws_athena/client"
+require_relative "integrations/source/clickhouse/client"
+require_relative "integrations/source/amazon_s3/client"
+require_relative "integrations/source/maria_db/client"
+require_relative "integrations/source/oracle_db/client"
+require_relative "integrations/source/databrics_model/client"
+require_relative "integrations/source/aws_sagemaker_model/client"
+require_relative "integrations/source/google_vertex_model/client"
+require_relative "integrations/source/http_model/client"
+require_relative "integrations/source/open_ai/client"
+require_relative "integrations/source/sftp/client"
+require_relative "integrations/source/watsonx_ai/client"
+require_relative "integrations/source/watsonx_data/client"
+require_relative "integrations/source/anthropic/client"
+require_relative "integrations/source/aws_bedrock_model/client"
+require_relative "integrations/source/generic_open_ai/client"
+require_relative "integrations/source/intuit_quick_books/client"
+require_relative "integrations/source/pinecone_db/client"
+require_relative "integrations/source/qdrant/client"
+require_relative "integrations/source/firecrawl/client"
+require_relative "integrations/source/odoo/client"
+
+# Destination
+require_relative "integrations/destination/klaviyo/client"
+require_relative "integrations/destination/salesforce_crm/client"
+require_relative "integrations/destination/facebook_custom_audience/client"
+require_relative "integrations/destination/slack/client"
+require_relative "integrations/destination/hubspot/client"
+require_relative "integrations/destination/google_sheets/client"
+require_relative "integrations/destination/airtable/client"
+require_relative "integrations/destination/stripe/client"
+require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
+require_relative "integrations/destination/sftp/client"
+require_relative "integrations/destination/postgresql/client"
+require_relative "integrations/destination/zendesk/client"
+require_relative "integrations/destination/http/client"
+require_relative "integrations/destination/iterable/client"
+require_relative "integrations/destination/maria_db/client"
+require_relative "integrations/destination/databricks_lakehouse/client"
+require_relative "integrations/destination/oracle_db/client"
+require_relative "integrations/destination/microsoft_excel/client"
+require_relative "integrations/destination/microsoft_sql/client"
+require_relative "integrations/destination/mailchimp/client"
+require_relative "integrations/destination/ais_data_store/client"
+require_relative "integrations/destination/amazon_s3/client"
+require_relative "integrations/destination/microsoft_dynamics/client"
+require_relative "integrations/destination/qdrant/client"
+require_relative "integrations/destination/pinecone_db/client"
+require_relative "integrations/destination/odoo/client"
+
+module Outhad
+  module Integrations
+    class Error < StandardError; end
+    # Your code goes here...
+  end
+end

data/outhad-integrations.gemspec

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative "lib/outhad/integrations/rollout"
+
+Gem::Specification.new do |spec|
+  spec.name = "outhad-integrations"
+  spec.version = Outhad::Integrations::VERSION
+  spec.authors = ["Tanzi"]
+  spec.email = ["tanzi@outhad.com"]
+
+  spec.summary = "Integration suite for open source reverse ETL platform"
+  spec.description = "Outhad Integrations is a comprehensive Ruby gem designed to facilitate seamless connectivity between various data sources and SaaS platforms."
+
+  spec.homepage = "https://www.outhad.com/"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  # spec.metadata["allowed_push_host"] = nil
+  spec.metadata["github_repo"] = "https://github.com/tanzil7890/outhad"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/tanzil7890/outhad/tree/main/integrations"
+  #spec.metadata["changelog_uri"] = "https://github.com/Outhad/outhad/blob/main/integrations/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile vendor/])
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "activesupport"
+  spec.add_runtime_dependency "async-websocket"
+  spec.add_runtime_dependency "aws-sdk-athena"
+  spec.add_runtime_dependency "aws-sdk-cloudwatchlogs"
+  spec.add_runtime_dependency "aws-sdk-s3"
+  spec.add_runtime_dependency "aws-sdk-sts"
+  spec.add_runtime_dependency "aws-sigv4"
+  spec.add_runtime_dependency "csv"
+  spec.add_runtime_dependency "dry-schema"
+  spec.add_runtime_dependency "dry-struct"
+  spec.add_runtime_dependency "dry-types"
+  spec.add_runtime_dependency "duckdb"
+  spec.add_runtime_dependency "git"
+  spec.add_runtime_dependency "google-apis-sheets_v4"
+  spec.add_runtime_dependency "google-cloud-ai_platform-v1"
+  spec.add_runtime_dependency "google-cloud-bigquery"
+  spec.add_runtime_dependency "grpc"
+  spec.add_runtime_dependency "hubspot-api-client"
+  spec.add_runtime_dependency "iterable-api-client"
+  spec.add_runtime_dependency "MailchimpMarketing"
+  spec.add_runtime_dependency "net-sftp"
+  spec.add_runtime_dependency "pg"
+  spec.add_runtime_dependency "rake"
+  spec.add_runtime_dependency "restforce"
+  spec.add_runtime_dependency "ruby-limiter"
+  spec.add_runtime_dependency "ruby-oci8"
+  spec.add_runtime_dependency "ruby-odbc"
+  spec.add_runtime_dependency "rubyzip"
+  spec.add_runtime_dependency "sequel"
+  spec.add_runtime_dependency "slack-ruby-client"
+  spec.add_runtime_dependency "stripe"
+  spec.add_runtime_dependency "tiny_tds"
+  spec.add_runtime_dependency "zendesk_api"
+
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "simplecov_json_formatter"
+  spec.add_development_dependency "webmock"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/sig/outhad/integrations.rbs

@@ -0,0 +1,6 @@
+module Outhad
+  module Integrations
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end