outhad-integrations 0.32.0

data/lib/outhad/integrations/source/http_model/client.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module HttpModel
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = prepare_config(connection_config)
+        response = send_request(
+          url: connection_config[:url_host],
+          http_method: connection_config[:http_method],
+          payload: JSON.parse(connection_config[:request_format]),
+          headers: connection_config[:headers],
+          config: connection_config[:config]
+        )
+        success?(response) ? success_status : failure_status(nil)
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP MODEL:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP MODEL:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = prepare_config(sync_config.source.connection_specification)
+        stream = connection_config[:is_stream] ||= false
+        # The server checks the ConnectorQueryType.
+        # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
+        # This query is then sent to the AI/ML model.
+        payload = parse_json(sync_config.model.query)
+
+        if stream
+          run_model_stream(connection_config, payload) { |message| yield message if block_given? }
+        else
+          run_model(connection_config, payload)
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP MODEL:READ:EXCEPTION", type: "error" })
+      end
+
+      private
+
+      def prepare_config(config)
+        config.with_indifferent_access.tap do |conf|
+          conf[:config][:timeout] ||= 30
+        end
+      end
+
+      def parse_json(json_string)
+        JSON.parse(json_string)
+      rescue JSON::ParserError => e
+        handle_exception(e, { context: "HTTP MODEL:PARSE_JSON:EXCEPTION", type: "error" })
+        {}
+      end
+
+      def run_model(connection_config, payload)
+        response = send_request(
+          url: connection_config[:url_host],
+          http_method: connection_config[:http_method],
+          payload: payload,
+          headers: connection_config[:headers],
+          config: connection_config[:config]
+        )
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP MODEL:RUN_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def run_model_stream(connection_config, payload)
+        send_streaming_request(
+          url: connection_config[:url_host],
+          http_method: connection_config[:http_method],
+          payload: payload,
+          headers: connection_config[:headers],
+          config: connection_config[:config]
+        ) do |chunk|
+          process_streaming_response(chunk) { |message| yield message if block_given? }
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP MODEL:RUN_STREAM_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def process_response(response)
+        if success?(response)
+          data = JSON.parse(response.body)
+          [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_outhad_message]
+        else
+          create_log_message("HTTP MODEL:RUN_MODEL", "error", "request failed: #{response.body}")
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP MODEL:PROCESS_RESPONSE:EXCEPTION", type: "error" })
+      end
+
+      def process_streaming_response(chunk)
+        data = JSON.parse(chunk)
+        yield [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_outhad_message] if block_given?
+      rescue StandardError => e
+        handle_exception(e, { context: "HTTP MODEL:PROCESS_STREAMING_RESPONSE:EXCEPTION", type: "error" })
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/http_model/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/outhad/integrations/source/http_model/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "HttpModel",
+    "title": "HTTP Model Endpoint",
+    "connector_type": "source",
+    "category": "AI Model",
+    "sub_category": "AI_ML Service",
+    "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/http-model-endpoint",
+    "github_issue_label": "source-http-model",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/http_model/config/spec.json

@@ -0,0 +1,70 @@
+{
+  "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/http-model-endpoint",
+  "stream_type": "user_defined",
+  "connector_query_type": "ai_ml",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "HTTP Model Endpoint",
+    "type": "object",
+    "required": ["url_host", "http_method"],
+    "properties": {
+      "http_method": {
+        "type": "string",
+        "title": "HTTP Method",
+        "enum": ["POST", "GET"],
+        "order": 0
+      },
+      "url_host": {
+        "type": "string",
+        "title": "URL",
+        "order": 1
+      },
+      "is_stream": {
+        "type": "boolean",
+        "title": "Streaming Enabled",
+        "description": "Enables data streaming for such as chat, when supported by the model. When true, messages and model data are processed in chunks for immediate delivery, enhancing responsiveness. Default is false, processing only after the entire response is received.",
+        "default": false,
+        "order": 2
+      },
+      "headers": {
+        "title": "HTTP Headers",
+        "description": "Custom headers to include in the HTTP request. Useful for authentication, content type specifications, and other request metadata.",
+        "order": 3,
+        "additionalProperties": {
+          "type": "string"
+        },
+        "default": {
+          "Content-Type": "application/json"
+        }
+      },
+      "config": {
+        "title": "",
+        "type": "object",
+        "properties": {
+          "timeout": {
+            "type": "string",
+            "default": "30",
+            "title": "HTTP Timeout",
+            "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
+            "order": 0
+          }
+        },
+        "order": 4
+      },
+      "request_format": {
+        "title": "Request Format",
+        "description": "Sample Request Format",
+        "type": "string",
+        "x-request-format": true,
+        "order": 5
+      },
+      "response_format": {
+        "title": "Response Format",
+        "description": "Sample Response Format",
+        "type": "string",
+        "x-response-format": true,
+        "order": 6
+      }
+    }
+  }
+}

data/lib/outhad/integrations/source/http_model/icon.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<svg width="800px" height="800px" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none">
+
+
+
+
+
+

data/lib/outhad/integrations/source/intuit_quick_books/client.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module IntuitQuickBooks
+    include Outhad::Integrations::Core
+
+    QUICKBOOKS_OBJECTS = %w[Account Customer Employee Invoice TimeActivity].freeze
+    MAX_PER_PAGE = 1000
+
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        access_token = create_connection(connection_config)
+        query = "SELECT * FROM Customer STARTPOSITION 1 MAXRESULTS 1"
+        response = query_quickbooks(access_token, query)
+        if success?(response)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "INTUIT_QUICKBOOKS:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        access_token = create_connection(connection_config)
+        catalog = build_catalog(load_catalog.with_indifferent_access)
+        streams = catalog[:streams]
+        QUICKBOOKS_OBJECTS.each do |object|
+          query = "SELECT * FROM #{object}"
+          response = query_quickbooks(access_token, query)
+          streams << create_streams(JSON.parse(response.body)["QueryResponse"])[0]
+        rescue StandardError => e
+          handle_exception(e, { context: "INTUIT_QUICKBOOKS:DISCOVER:LOOP_EXCEPTION", type: "error" })
+          next
+        end
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, { context: "INTUIT_QUICKBOOKS:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        @connector_instance = sync_config&.source&.connector_instance
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        access_token = create_connection(connection_config)
+        query(access_token, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "INTUIT_QUICKBOOKS:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def query(access_token, query)
+        parsed = batched_query_for_quickbooks(query)
+        base_query = parsed[:base_query]
+        limit = parsed[:limit]
+        offset = parsed[:offset]
+        execute_query(access_token, base_query, limit, offset).map do |r|
+          flat_data = flatten_hash(r)
+          RecordMessage.new(data: flat_data, emitted_at: Time.now.to_i).to_outhad_message
+        end
+      end
+
+      def execute_query(access_token, base_query, limit, offset)
+        total_fetched = 0
+        current_offset = offset
+        result = []
+
+        while total_fetched < limit
+          batch_limit = [MAX_PER_PAGE, limit - total_fetched].min
+          paginated_query = "#{base_query} STARTPOSITION #{current_offset + 1} MAXRESULTS #{batch_limit}"
+
+          response = query_quickbooks(access_token, paginated_query)
+          records = JSON.parse(response.body)["QueryResponse"] || {}
+
+          break if records.empty?
+
+          records.each_value do |rows|
+            next unless rows.is_a?(Array)
+
+            rows.each do |row|
+              result << row
+            end
+          end
+
+          fetched_count = result.size - total_fetched
+          break if fetched_count < batch_limit
+
+          total_fetched += fetched_count
+          current_offset += fetched_count
+        end
+        result
+      end
+
+      def query_quickbooks(access_token, query)
+        encoded_query = URI.encode_www_form_component(query)
+        query_url = @environment == "sandbox" ? QUICKBOOKS_SANDBOX_QUERY_URL : QUICKBOOKS_PRODUCTION_QUERY_URL
+        send_request(
+          url: build_url(query_url, encoded_query),
+          http_method: HTTP_GET,
+          payload: {},
+          headers: auth_headers(access_token),
+          config: {}
+        )
+      end
+
+      def create_connection(connection_config)
+        load_connection_config(connection_config)
+        refresh_access_token
+      end
+
+      def load_connection_config(connection_config)
+        @client_id = connection_config[:client_id]
+        @client_secret = connection_config[:client_secret]
+        @realm_id = connection_config[:realm_id]
+        @environment = connection_config[:environment]
+        @refresh_token = if @connector_instance&.configuration
+                           @connector_instance.configuration["refresh_token"]
+                         else
+                           connection_config[:refresh_token]
+                         end
+      end
+
+      def refresh_access_token
+        oauth2_client = IntuitOAuth::Client.new(@client_id, @client_secret, QUICKBOOKS_REDIRECT_URL, @environment)
+        response = oauth2_client.token.refresh_tokens(@refresh_token)
+        if @connector_instance&.configuration
+          config = @connector_instance.configuration
+          config = {} unless config.is_a?(Hash)
+          new_config = config.merge("refresh_token" => response.refresh_token)
+          @connector_instance.update!(configuration: new_config)
+        end
+        response.access_token
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:table_name], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.filter_map do |table_name, rows|
+          if rows.is_a?(Array) && rows.all? { |row| row.is_a?(Hash) }
+            row_sample = rows.first || {}
+            columns = row_sample.map do |key, value|
+              {
+                column_name: key,
+                data_type: normalize_type(value),
+                is_nullable: rows.any? { |row| row[key].nil? }
+              }
+            end
+            { table_name: table_name, columns: columns }
+          end
+        end
+      end
+
+      def batched_query_for_quickbooks(query)
+        query = query.strip.chomp(";")
+        limit = query[/LIMIT\s+(\d+)/i, 1] || 1000
+        offset = query[/OFFSET\s+(\d+)/i, 1]
+
+        base_query = query.gsub(/LIMIT\s+\d+/i, "").gsub(/OFFSET\s+\d+/i, "").strip
+        {
+          base_query: base_query,
+          limit: limit.to_i,
+          offset: offset.to_i
+        }
+      end
+
+      def flatten_hash(hash, parent_key = "", result = {})
+        hash.each do |key, value|
+          full_key = parent_key.empty? ? key.to_s : "#{parent_key}.#{key}"
+
+          case value
+          when Hash
+            flatten_hash(value, full_key, result)
+          when Array
+            next
+          else
+            result[full_key] = value.is_a?(Integer) || value.is_a?(Float) ? value : value.to_s
+          end
+        end
+        result
+      end
+
+      def normalize_type(value)
+        case value
+        when Integer, Float then "NUMBER"
+        else "string"
+        end
+      end
+
+      def load_catalog
+        read_json(CATALOG_SPEC_PATH)
+      end
+
+      def build_url(url, query)
+        format(url, realm_id: @realm_id, query: query)
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/intuit_quick_books/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 500,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/outhad/integrations/source/intuit_quick_books/config/meta.json

@@ -0,0 +1,17 @@
+{
+    "data": {
+      "name": "IntuitQuickBooks",
+      "title": "Intuit QuickBooks",
+      "connector_type": "source",
+      "category": "Data Warehouse",
+      "sub_category": "Relational Database",
+      "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/intuit_quickbooks",
+      "github_issue_label": "source-intuit-quickbooks",
+      "icon": "icon.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "outhad"]
+    }
+  }
+  

data/lib/outhad/integrations/source/intuit_quick_books/config/spec.json

@@ -0,0 +1,44 @@
+{
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/intuit_quickbooks",
+    "stream_type": "dynamic",
+    "connector_query_type": "raw_sql",
+    "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "Intuit QuickBooks",
+      "type": "object",
+      "required": ["environment","client_id","client_secret","realm_id","refresh_token"],
+      "properties": {
+        "environment": {
+          "type": "string",
+          "title": "Environment",
+          "enum": ["sandbox", "production"],
+          "order": 0
+        },
+        "client_id": {
+          "type": "string",
+          "outhad_secret": true,
+          "title": "Client Id",
+          "order": 1
+        },
+        "client_secret": {
+          "type": "string",
+          "outhad_secret": true,
+          "title": "Client Secret",
+          "order": 2
+        },
+        "realm_id": {
+          "type": "string",
+          "outhad_secret": true,
+          "title": "Realm Id",
+          "order": 3
+        },
+        "refresh_token": {
+          "type": "string",
+          "outhad_secret": true,
+          "title": "Refresh Token",
+          "order": 4
+        }
+      }
+    }
+  }
+  

data/lib/outhad/integrations/source/intuit_quick_books/icon.svg

@@ -0,0 +1 @@
+<svg enable-background="new 0 0 2500 2500" viewBox="0 0 2500 2500" xmlns="http://www.w3.org/2000/svg"><circle cx="1250" cy="1250" fill="#2ca01c" r="1250"/><path d="m301.3 1249.6c.1 282.6 228 512.4 510.6 514.9h72.3v-188.9h-72.3c-175.2 47.8-355.9-55.5-403.6-230.7-.4-1.4-.7-2.8-1.1-4.2-49.1-177.5 53.7-361.4 230.6-412.5h36.1c45.3-9.9 92.2-9.9 137.5 0h175.6v1002.9c-.9 106.1 84.4 192.9 190.5 193.9v-1395.4h-364.5c-284.6 1.5-514 233.4-512.5 518v.1zm1387.5-519.8h-72.3v198.9h72.3c174.8-47.7 355.1 55.3 402.8 230 .4 1.3.7 2.7 1.1 4 48.8 176.9-53.7 360.1-229.9 411.1h-36.1c-45.3 9.9-92.2 9.9-137.5 0h-175.6v-1002.8c.9-106.1-84.4-192.9-190.5-193.9v1397.4h364.5c287.1-4.5 516.2-240.8 511.8-527.9-4.4-280.8-230.9-507.4-511.8-511.8z" fill="#fff"/></svg>

data/lib/outhad/integrations/source/maria_db/client.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Source
+  module MariaDB
+    include Outhad::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_outhad_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:database]}' ORDER BY table_name, ordinal_position;"
+        db = create_connection(connection_config)
+        results = query_execution(db, query)
+        catalog = Catalog.new(streams: create_streams(results))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MARIA:DB:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        db = create_connection(connection_config)
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MARIA:DB:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        Sequel.connect(
+          adapter: "mysql2",
+          host: connection_config[:host],
+          port: connection_config[:port],
+          user: connection_config[:username],
+          password: connection_config[:password],
+          database: connection_config[:database]
+        )
+      end
+
+      def query_execution(db, query)
+        db.fetch(query).all
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def query(db, query)
+        records = []
+        query_execution(db, query).map do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+        end
+        records
+      end
+
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry[:table_name]
+          column_data = {
+            column_name: entry[:column_name],
+            data_type: entry[:data_type],
+            is_nullable: entry[:is_nullable] == "YES"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result
+      end
+    end
+  end
+end

data/lib/outhad/integrations/source/maria_db/config/meta.json

@@ -0,0 +1,16 @@
+{
+  "data": {
+    "name": "MariaDB",
+    "title": "Maria DB",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "sub_category": "Relational Database",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/maria_db",
+    "github_issue_label": "source-maria-db",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/source/maria_db/config/spec.json

@@ -0,0 +1,48 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/maria_db",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Maria DB",
+    "type": "object",
+    "required": ["host", "port", "username", "password", "database"],
+    "properties": {
+      "host": {
+        "description": "The hostname or IP address of the server where the MariaDB database is hosted.",
+        "examples": ["localhost"],
+        "type": "string",
+        "title": "Host",
+        "order": 0
+      },
+      "port": {
+        "description": "The port number on which the MariaDB server is listening for connections.",
+        "examples": ["3306"],
+        "type": "string",
+        "title": "Port",
+        "order": 1
+      },
+      "username": {
+        "description": "The username used to authenticate and connect to the MariaDB database.",
+        "examples": ["root"],
+        "type": "string",
+        "title": "Username",
+        "order": 2
+      },
+      "password": {
+        "description": "The password corresponding to the username used for authentication.",
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Password",
+        "order": 3
+      },
+      "database": {
+        "description": "The name of the specific database within the MariaDB server to connect to.",
+        "examples": ["mydatabase"],
+        "type": "string",
+        "title": "Database",
+        "order": 4
+      }
+    }
+  }
+}