outhad-integrations 0.32.0

data/lib/outhad/integrations/destination/microsoft_sql/client.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "tiny_tds"
+
+module Outhad::Integrations::Destination
+  module MicrosoftSql
+    include Outhad::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        db_client = create_connection(connection_config)
+
+        if db_client.active?
+          ConnectionStatus.new(
+            status: ConnectionStatusType["succeeded"]
+          ).to_outhad_message
+        else
+          ConnectionStatus.new(
+            status: ConnectionStatusType["failed"], message: "MS SQL connection not active"
+          ).to_outhad_message
+        end
+      rescue TinyTds::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_outhad_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db_client = create_connection(connection_config)
+        records = db_client.execute(query) do |result|
+          result.map do |row|
+            row
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MSSQL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      ensure
+        db_client&.close
+      end
+
+      def write(sync_config, records, action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        table_name = sync_config.stream.name
+        primary_key = sync_config.model.primary_key
+        log_message_array = []
+        db = create_connection(connection_config)
+
+        write_success = 0
+        write_failure = 0
+
+        records.each do |record|
+          query = Outhad::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
+          logger.debug("MSSQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+          begin
+            response = db.execute(query)
+            response.do
+            write_success += 1
+            log_message_array << log_request_response("info", query, response)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "MSSQL:RECORD:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+            write_failure += 1
+            log_message_array << log_request_response("error", query, e.message)
+          end
+        end
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "MSSQL:RECORD:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def query(connection, query)
+        connection.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_outhad_message
+          end
+        end
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        TinyTds::Client.new(
+          username: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          host: connection_config[:host],
+          port: connection_config[:port],
+          database: connection_config[:database],
+          azure: true,
+          timeout: 3000
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/microsoft_sql/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "MicrosoftSql",
+    "title": "Microsoft SQL",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/ms_sql",
+    "github_issue_label": "destination-mssql",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/destination/microsoft_sql/config/spec.json

@@ -0,0 +1,68 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/ms_sql",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "MicrosoftSql",
+    "type": "object",
+    "required": ["host", "port", "database", "schema"],
+    "properties": {
+      "credentials": {
+        "title": "",
+        "type": "object",
+        "required": ["auth_type", "username", "password"],
+        "properties": {
+          "auth_type": {
+            "type": "string",
+            "default": "username/password",
+            "order": 0,
+            "readOnly": true
+          },
+          "username": {
+            "description": "Username refers to your individual MS SQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
+            "examples": ["MSSQL_USER"],
+            "type": "string",
+            "title": "Username",
+            "order": 1
+          },
+          "password": {
+            "description": "This field requires the password associated with the user account specified in the preceding section.",
+            "type": "string",
+            "outhad_secret": true,
+            "title": "Password",
+            "order": 2
+          }
+        },
+        "order": 0
+      },
+      "host": {
+        "description": "The hostname or IP address of your MS SQL server.",
+        "examples": ["127.0.0.1"],
+        "type": "string",
+        "title": "Host",
+        "order": 1
+      },
+      "port": {
+        "description": "The port number for your MS SQL server, which defaults to 1433, may vary based on your configuration. ",
+        "examples": ["1433"],
+        "type": "string",
+        "title": "Port",
+        "order": 2
+      },
+      "database": {
+        "description": "The specific MS SQL database to connect to.",
+        "examples": ["MSSQL_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 3
+      },
+      "schema": {
+        "description": "The schema within the MS SQL database.",
+        "examples": ["dbo"],
+        "type": "string",
+        "title": "Schema",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/microsoft_sql/icon.svg

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<svg width="800px" height="800px" viewBox="0 -141.54 1478.201 1478.201" xmlns="http://www.w3.org/2000/svg">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

data/lib/outhad/integrations/destination/odoo/client.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Destination
+  module Odoo
+    include Outhad::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        success_status
+      rescue StandardError => e
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+
+        models = @client.execute_kw(connection_config[:database], @uid, connection_config[:password],
+                                    "ir.model", "search_read", [[["transient", "=", false], ["abstract", "=", false]]], { 'fields': %w[name model] })
+
+        catalog = Catalog.new(streams: create_streams(connection_config, models))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "ODOO:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def write(sync_config, records, _action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        create_connection(connection_config)
+        model = sync_config.stream.name
+
+        write_success = 0
+        write_failure = 0
+        log_message_array = []
+
+        records.each do |record|
+          logger.debug("ODOO:WRITE:#{model} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+          begin
+            response = @client.execute_kw(connection_config[:database], @uid, connection_config[:password],
+                                          model, "create", [record])
+            write_success += 1
+            log_message_array << log_request_response("info", model, response)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "ODOO:WRITE:#{model}",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+            write_failure += 1
+            log_message_array << log_request_response("error", model, e.message)
+          end
+        end
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "ODOO:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_streams(connection_config, models)
+        models.map do |model|
+          fields = @client.execute_kw(connection_config[:database], @uid, connection_config[:password],
+                                      model["model"], "fields_get", [], { 'attributes': %w[name type] })
+          Outhad::Integrations::Protocol::Stream.new(name: model["model"], action: StreamAction["fetch"],
+                                                         supported_sync_modes: %w[incremental], json_schema: convert_to_json_schema(fields))
+        end
+      end
+
+      def convert_to_json_schema(fields)
+        json_schema = {
+          "type" => "object",
+          "properties" => {}
+        }
+        required_fields = []
+        fields.each do |field|
+          next if field[1]["readonly"]
+
+          column_name = field[1]["name"]
+          required = field[1]["required"]
+          type = field[1]["type"]
+          json_schema["properties"][column_name] = {
+            "type" => type
+          }
+          required_fields.push(column_name) if required
+        end
+        json_schema["required"] = required_fields
+        json_schema
+      end
+
+      def create_connection(connection_config)
+        common = XMLRPC::Client.new2("#{connection_config[:url]}/xmlrpc/2/common")
+        common.call("version")
+        @uid = common.call("authenticate", connection_config[:database], connection_config[:username],
+                           connection_config[:password], { 'raise_exception': true })
+        @client = XMLRPC::Client.new2("#{connection_config[:url]}/xmlrpc/2/object").proxy
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/odoo/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Odoo",
+    "title": "Odoo",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/odoo",
+    "github_issue_label": "destination-odoo",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/destination/odoo/config/spec.json

@@ -0,0 +1,39 @@
+
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/odoo",
+  "stream_type": "static",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Odoo",
+    "type": "object",
+    "required": ["url","database","username","password"],
+    "properties": {
+      "url": {
+        "description": "The Odoo host url.",
+        "type": "string",
+        "title": "URL",
+        "order": 0
+      },
+      "database": {
+        "description": "The Odoo database.",
+        "type": "string",
+        "title": "Database",
+        "order": 1
+      },
+      "username": {
+        "description": "The Odoo username.",
+        "type": "string",
+        "title": "Username",
+        "order": 2
+      },
+      "password": {
+        "description": "The Odoo password.",
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Password",
+        "order": 3
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/odoo/icon.svg

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 24.3.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 126.2 40" style="enable-background:new 0 0 126.2 40;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#8F8F8F;}
+	.st1{fill:#714B67;}
+</style>
+<g id="Group_982" transform="translate(-13.729 -4.35)">
+	<path id="Path_172" class="st0" d="M60.9,38c4.9,0,8.9-4,8.9-8.9c0-4.9-4-8.9-8.9-8.9c-4.9,0-8.9,4-8.9,8.9c0,0,0,0,0,0
+		C51.9,34,55.9,38,60.9,38 M76.1,28.8c0.1,8.4-6.6,15.4-15,15.5c-8.4,0.1-15.4-6.6-15.5-15c0-0.2,0-0.3,0-0.5
+		c0.3-8.6,7.6-15.4,16.2-15.1c2.9,0.1,5.6,1,8,2.6V7.4c0.1-1.7,1.5-3.1,3.3-3.1c1.7,0,3,1.4,3.1,3.1L76.1,28.8z M92.7,38
+		c4.9,0,8.9-4,8.9-8.9c0-4.9-4-8.9-8.9-8.9c-4.9,0-8.9,4-8.9,8.9c0,0,0,0,0,0C83.8,34,87.8,38,92.7,38L92.7,38 M92.7,44.3
+		c-8.4,0-15.2-6.8-15.2-15.2c0-8.4,6.8-15.2,15.2-15.2c8.4,0,15.2,6.8,15.2,15.2c0,0,0,0,0,0C108,37.4,101.2,44.3,92.7,44.3
+		 M124.6,38c4.9,0,8.9-4,8.9-8.9s-4-8.9-8.9-8.9c-4.9,0-8.9,4-8.9,8.9c0,0,0,0,0,0C115.7,34,119.7,38,124.6,38 M124.6,44.3
+		c-8.4,0-15.2-6.8-15.2-15.2c0-8.4,6.8-15.2,15.2-15.2c8.4,0,15.2,6.8,15.2,15.2c0,0,0,0,0,0C139.9,37.4,133,44.3,124.6,44.3"/>
+	<path id="Path_173" class="st1" d="M29,38c4.9,0,8.9-4,8.9-8.9c0-4.9-4-8.9-8.9-8.9c-4.9,0-8.9,4-8.9,8.9c0,0,0,0,0,0
+		C20,34,24,38,29,38 M29,44.3c-8.4,0-15.2-6.8-15.2-15.2S20.5,13.8,29,13.8S44.2,20.6,44.2,29c0,0,0,0,0,0
+		C44.2,37.4,37.4,44.3,29,44.3C29,44.3,29,44.3,29,44.3"/>
+</g>
+</svg>

data/lib/outhad/integrations/destination/oracle_db/client.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Destination
+  module Oracle
+    include Outhad::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_outhad_message
+      rescue StandardError => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_outhad_message
+      end
+
+      def discover(connection_config)
+        records = []
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, nullable
+                 FROM all_tab_columns
+                 WHERE owner = '#{connection_config[:username].upcase}'
+                 ORDER BY table_name, column_id"
+        conn = create_connection(connection_config)
+        cursor = conn.exec(query)
+        while (row = cursor.fetch)
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(
+          "ORACLE:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def write(sync_config, records, action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        table_name = sync_config.stream.name
+        primary_key = sync_config.model.primary_key
+        conn = create_connection(connection_config)
+
+        write_success = 0
+        write_failure = 0
+        log_message_array = []
+
+        records.each do |record|
+          query = Outhad::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
+          query = query.gsub(";", "")
+          logger.debug("ORACLE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+          begin
+            response = conn.exec(query)
+            conn.exec("COMMIT")
+            write_success += 1
+            log_message_array << log_request_response("info", query, response)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "ORACLE:RECORD:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+            write_failure += 1
+            log_message_array << log_request_response("error", query, e.message)
+          end
+        end
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "ORACLE:RECORD:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry[0]
+          column_data = {
+            column_name: entry[1],
+            data_type: entry[2],
+            is_nullable: entry[3] == "Y"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
+          { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/oracle_db/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Oracle",
+    "title": "Oracle DB",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/oracle",
+    "github_issue_label": "destination-oracle",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/destination/oracle_db/config/spec.json

@@ -0,0 +1,47 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/oracle",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Oracle",
+    "type": "object",
+    "required": ["host", "port", "sid", "username", "password"],
+    "properties": {
+      "host": {
+        "description": "The Oracle host.",
+        "examples": ["localhost"],
+        "type": "string",
+        "title": "Host",
+        "order": 0
+      },
+      "port": {
+        "description": "The Oracle port number.",
+        "examples": ["1521"],
+        "type": "string",
+        "title": "Port",
+        "order": 1
+      },
+      "sid": {
+        "description": "The name of your service in Oracle.",
+        "examples": ["ORCLPDB1"],
+        "type": "string",
+        "title": "SID",
+        "order": 2
+      },
+      "username": {
+        "description": "The username used to authenticate and connect.",
+        "type": "string",
+        "title": "Username",
+        "order": 3
+      },
+      "password": {
+        "description": "The password corresponding to the username used for authentication.",
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Password",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/oracle_db/icon.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
+  <path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
+</svg>