outhad-integrations 0.32.0

data/lib/outhad/integrations/destination/ais_data_store/config/spec.json

@@ -0,0 +1,68 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/ais_data_store",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "AIS Data Store",
+    "type": "object",
+    "required": ["host", "port", "database", "schema"],
+    "properties": {
+      "credentials": {
+        "title": "",
+        "type": "object",
+        "required": ["auth_type", "username", "password"],
+        "properties": {
+          "auth_type": {
+            "type": "string",
+            "default": "username/password",
+            "order": 0,
+            "readOnly": true
+          },
+          "username": {
+            "description": "Username refers to your individual AIS Data Store login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
+            "examples": ["AIS_DATA_STORE_USER"],
+            "type": "string",
+            "title": "Username",
+            "order": 1
+          },
+          "password": {
+            "description": "This field requires the password associated with the user account specified in the preceding section.",
+            "type": "string",
+            "outhad_secret": true,
+            "title": "Password",
+            "order": 2
+          }
+        },
+        "order": 0
+      },
+      "host": {
+        "description": "The hostname or IP address of your AIS Data Store server.",
+        "examples": ["127.0.0.1"],
+        "type": "string",
+        "title": "Host",
+        "order": 1
+      },
+      "port": {
+        "description": "The port number for your AIS Data Store server, which defaults to 5432, may vary based on your configuration. ",
+        "examples": ["5432"],
+        "type": "string",
+        "title": "Port",
+        "order": 2
+      },
+      "database": {
+        "description": "The specific AIS Data Store database to connect to.",
+        "examples": ["AIS_DATA_STORE_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 3
+      },
+      "schema": {
+        "description": "The schema within the AIS Data Store database.",
+        "examples": ["AIS_DATA_STORE_SCHEMA"],
+        "type": "string",
+        "title": "Schema",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/ais_data_store/icon.svg

@@ -0,0 +1,4 @@
+<svg width="66" height="64" viewBox="0 0 66 64" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M24.5072 18.9904C26.4821 18.9904 28.2101 20.0795 29.1976 21.6527H51.2916V18.2643C51.2916 15.3599 48.9464 13.0607 45.9841 13.0607H18.2123C15.4968 13.0607 13.3985 15.2389 13.3985 17.7802V33.1491C13.3985 34.9643 14.8796 36.4165 16.7311 36.4165H34.9988C35.8628 34.4802 37.8377 33.1491 40.0595 33.1491C43.1452 33.1491 45.7373 35.6904 45.7373 38.8368C45.7373 41.9831 43.1452 44.5244 40.0595 44.5244C38.4549 44.5244 36.9737 43.7984 35.8628 42.5882H13.1516V46.5817C13.1516 49.002 15.1265 50.9382 17.5951 50.9382H47.8356C49.687 50.9382 51.1682 49.486 51.0448 47.6708V32.181C51.0448 30.0027 49.1933 28.1875 46.9716 28.1875H28.9507C27.9633 29.5186 26.3587 30.3657 24.5072 30.3657C21.4215 30.3657 18.8294 27.8244 18.8294 24.6781C18.8294 21.5317 21.4215 18.9904 24.5072 18.9904Z" fill="#00249C"/>
+<path d="M57.0292 0H8.02359C3.57976 0 0 3.54198 0 7.93893V56.0611C0 60.458 3.57976 64 8.02359 64H57.0292C61.4731 64 65.0528 60.458 65.0528 56.0611V7.93893C65.1763 3.54198 61.4731 0 57.0292 0ZM59.1277 51.6641C59.1277 55.2061 56.1652 58.1374 52.5854 58.1374H12.4674C8.88767 58.1374 5.92512 55.2061 5.92512 51.6641V12.3359C5.92512 8.79389 8.88767 5.8626 12.4674 5.8626H52.7088C56.2886 5.8626 59.2512 8.79389 59.2512 12.3359V51.6641H59.1277Z" fill="#00249C"/>
+</svg>

data/lib/outhad/integrations/destination/amazon_s3/client.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Outhad::Integrations::Destination
+  module AmazonS3
+    include Outhad::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        conn = create_connection(connection_config)
+        conn.head_bucket(bucket: connection_config[:bucket_name])
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_outhad_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_outhad_message
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_outhad_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AMAZONS3:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def write(sync_config, records, _action = "destination_insert")
+        records_size = records.size
+        log_message_array = []
+        write_success = upload_csv_content(sync_config, records)
+        write_failure = records_size - write_success
+        log_message_array << log_request_response("info", @args, @response)
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "AMAZONS3:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        Aws::S3::Client.new(
+          region: connection_config[:region],
+          access_key_id: connection_config[:access_key_id],
+          secret_access_key: connection_config[:secret_access_key]
+        )
+      end
+
+      def upload_csv_content(sync_config, records)
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        conn = create_connection(connection_config)
+        file_name = generate_local_file_name(connection_config)
+        csv_content = generate_csv_content(records)
+        begin
+          @args = ["create", connection_config[:bucket_name], "#{connection_config[:file_path]}#{file_name}", csv_content]
+          @response = conn.put_object(
+            bucket: connection_config[:bucket_name],
+            key: "#{connection_config[:file_path]}#{file_name}",
+            body: csv_content
+          )
+          write_success = records.size
+        rescue StandardError => e
+          handle_exception(e, {
+                             context: "AMAZONS3:RECORD:WRITE:EXCEPTION",
+                             type: "error",
+                             sync_id: sync_config.sync_id,
+                             sync_run_id: sync_config.sync_run_id
+                           })
+          write_success = 0
+        end
+        write_success
+      end
+
+      def generate_csv_content(records)
+        CSV.generate do |csv|
+          headers = records.first.keys
+          csv << headers
+          records.each { |record| csv << record.values_at(*headers) }
+        end
+      end
+
+      def generate_local_file_name(connection_config)
+        timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
+        "#{connection_config[:file_name]}_#{timestamp}.#{connection_config[:format_type]}"
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/amazon_s3/config/catalog.json

@@ -0,0 +1,16 @@
+{
+  "request_rate_limit": 600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "schema_mode": "schemaless",
+  "streams": [
+    {
+      "name": "create",
+      "batch_support": true,
+      "batch_size": 100000,
+      "action": "create",
+      "json_schema": {},
+      "supported_sync_modes": ["full_refresh","incremental"]
+    }
+  ]
+}

data/lib/outhad/integrations/destination/amazon_s3/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "AmazonS3",
+    "title": "Amazon S3",
+    "connector_type": "source",
+    "category": "File Storage",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/file-storage/amazon_s3",
+    "github_issue_label": "source-amazons3",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/destination/amazon_s3/config/spec.json

@@ -0,0 +1,56 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/file-storage/amazon_s3",
+  "stream_type": "static",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "AmazonS3",
+    "required": ["access_key_id", "secret_access_key", "region", "bucket_name", "file_path", "file_name", "format_type" ],
+    "properties": {
+      "access_key_id": {
+        "description": "The AWS Access Key ID to use for authentication.",
+        "type": "string",
+        "title": "Personal Access Key",
+        "order": 0
+      },
+      "secret_access_key": {
+        "description": "The AWS Secret Access Key to use for authentication.",
+        "type": "string",
+        "outhad_secret": true,
+        "title": "Secret Access Key",
+        "order": 1
+      },
+      "region": {
+        "description": "AWS region.",
+        "type": "string",
+        "title": "Region",
+        "order": 2
+      },
+      "bucket_name": {
+        "title": "Bucket Name",
+        "description": "Amazon S3 bucket name.",
+        "type": "string",
+        "order": 3
+      },
+      "file_path": {
+        "title": "File Path",
+        "type": "string",
+        "description": "Path to the directory where files will be written.",
+        "order": 4
+      },
+      "file_name": {
+        "title": "File Name",
+        "type": "string",
+        "description": "Name of the file to be written.",
+        "order": 5
+      },
+      "format_type": {
+        "title": "File Format Type",
+        "type": "string",
+        "description": "Format of the data output.",
+        "order": 6,
+        "enum": ["csv"],
+        "default": "csv"
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/amazon_s3/icon.svg

@@ -0,0 +1,34 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="428" height="512" viewBox="0 0 428 512">
+  <defs>
+    <style>
+      .cls-1 {
+        fill: #e25444;
+      }
+
+      .cls-1, .cls-2, .cls-3 {
+        fill-rule: evenodd;
+      }
+
+      .cls-2 {
+        fill: #7b1d13;
+      }
+
+      .cls-3 {
+        fill: #58150d;
+      }
+    </style>
+  </defs>
+  <path class="cls-1" d="M378,99L295,257l83,158,34-19V118Z"/>
+  <path class="cls-2" d="M378,99L212,118,127.5,257,212,396l166,19V99Z"/>
+  <path class="cls-3" d="M43,99L16,111V403l27,12L212,257Z"/>
+  <path class="cls-1" d="M42.637,98.667l169.587,47.111V372.444L42.637,415.111V98.667Z"/>
+  <path class="cls-3" d="M212.313,170.667l-72.008-11.556,72.008-81.778,71.83,81.778Z"/>
+  <path class="cls-3" d="M284.143,159.111l-71.919,11.733-71.919-11.733V77.333"/>
+  <path class="cls-3" d="M212.313,342.222l-72.008,13.334,72.008,70.222,71.83-70.222Z"/>
+  <path class="cls-2" d="M212,16L140,54V159l72.224-20.333Z"/>
+  <path class="cls-2" d="M212.224,196.444l-71.919,7.823V309.105l71.919,8.228V196.444Z"/>
+  <path class="cls-2" d="M212.224,373.333L140.305,355.3V458.363L212.224,496V373.333Z"/>
+  <path class="cls-1" d="M284.143,355.3l-71.919,18.038V496l71.919-37.637V355.3Z"/>
+  <path class="cls-1" d="M212.224,196.444l71.919,7.823V309.105l-71.919,8.228V196.444Z"/>
+  <path class="cls-1" d="M212,16l72,38V159l-72-20V16Z"/>
+</svg>

data/lib/outhad/integrations/destination/databricks_lakehouse/client.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Outhad
+  module Integrations
+    module Destination
+      module DatabricksLakehouse
+        include Outhad::Integrations::Core
+        class Client < DestinationConnector
+          MAX_CHUNK_SIZE = 10
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            db = create_connection(connection_config)
+            response = db.get("/api/2.0/clusters/list")
+            if response.status == 200
+              success_status
+            else
+              failure_status(nil)
+            end
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "DATABRICKS:LAKEHOUSE:CHECK_CONNECTION:EXCEPTION",
+                               type: "error"
+                             })
+            failure_status(e)
+          end
+
+          def discover(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            table_query = "SHOW TABLES IN #{connection_config[:catalog]}.#{connection_config[:schema]};"
+            db = create_connection(connection_config)
+            records = []
+            table_response = db.post("/api/2.0/sql/statements", generate_body(connection_config[:warehouse_id], table_query).to_json)
+            table_response_body = JSON.parse(table_response.body)
+            table_response_body["result"]["data_array"].each do |table|
+              table_name = table[1]
+              query = "DESCRIBE TABLE #{connection_config[:catalog]}.#{connection_config[:schema]}.#{table_name};"
+              column_response = db.post("/api/2.0/sql/statements", generate_body(connection_config[:warehouse_id], query).to_json)
+              column_response_body = JSON.parse(column_response.body)
+              records << [table_name, column_response_body["result"]["data_array"]]
+            end
+            catalog = Catalog.new(streams: create_streams(records))
+            catalog.to_outhad_message
+          rescue StandardError => e
+            handle_exception(
+              "DATABRICKS:LAKEHOUSE:DISCOVER:EXCEPTION",
+              "error",
+              e
+            )
+          end
+
+          def write(sync_config, records, action = "destination_insert")
+            connection_config = sync_config.destination.connection_specification.with_indifferent_access
+            table_name = "#{connection_config[:catalog]}.#{connection_config[:schema]}.#{sync_config.stream.name}"
+            primary_key = sync_config.model.primary_key
+            db = create_connection(connection_config)
+            write_success = 0
+            write_failure = 0
+            log_message_array = []
+
+            records.each do |record|
+              query = Outhad::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
+              logger.debug("DATABRICKS:LAKEHOUSE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+              begin
+                arg = ["/api/2.0/sql/statements", generate_body(connection_config[:warehouse_id], query)]
+                response = db.post("/api/2.0/sql/statements", generate_body(connection_config[:warehouse_id], query).to_json)
+                if response.status == 200
+                  write_success += 1
+                else
+                  write_failure += 1
+                end
+                log_message_array << log_request_response("info", arg, response)
+              rescue StandardError => e
+                handle_exception(e, {
+                                   context: "DATABRICKS:LAKEHOUSE:RECORD:WRITE:EXCEPTION",
+                                   type: "error",
+                                   sync_id: sync_config.sync_id,
+                                   sync_run_id: sync_config.sync_run_id
+                                 })
+                write_failure += 1
+              end
+            end
+            tracking_message(write_success, write_failure)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "DATABRICKS:LAKEHOUSE:RECORD:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+          end
+
+          private
+
+          def create_connection(connection_config)
+            Faraday.new(url: connection_config[:host]) do |conn|
+              conn.headers["Authorization"] = "Bearer #{connection_config[:api_token]}"
+              conn.headers["Content-Type"] = "application/json"
+              conn.adapter Faraday.default_adapter
+            end
+          end
+
+          def generate_body(warehouse_id, query)
+            {
+              warehouse_id: warehouse_id,
+              statement: query,
+              wait_timeout: "15s"
+            }
+          end
+
+          def create_streams(records)
+            message = []
+            group_by_table(records).each_value do |r|
+              message << Outhad::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+            end
+            message
+          end
+
+          def group_by_table(records)
+            result = {}
+            records.each_with_index do |entries, index|
+              table_name = records[index][0]
+              column = []
+              entry_data = entries[1]
+              entry_data.each do |entry|
+                column << {
+                  column_name: entry[0],
+                  data_type: entry[1],
+                  is_nullable: true
+                }
+              end
+              result[index] ||= {}
+              result[index][:tablename] = table_name
+              result[index][:columns] = column
+            end
+            result
+          end
+
+          def tracking_message(success, failure)
+            Outhad::Integrations::Protocol::TrackingMessage.new(
+              success: success, failed: failure
+            ).to_outhad_message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/outhad/integrations/destination/databricks_lakehouse/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "DatabricksLakehouse",
+    "title": "Databricks Data Warehouse",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/analytics/databricks_lakehouse",
+    "github_issue_label": "destination-databricks-lakehouse",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "outhad"]
+  }
+}

data/lib/outhad/integrations/destination/databricks_lakehouse/config/spec.json

@@ -0,0 +1,44 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/analytics/databricks_lakehouse",
+  "stream_type": "static",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Databricks Datawarehouse",
+    "type": "object",
+    "required": ["host", "api_token", "warehouse_id", "catalog", "schema"],
+    "properties": {
+      "host": {
+        "description": "The databrick lakehouse host domain.",
+        "type": "string",
+        "title": "Host",
+        "order": 0
+      },
+      "api_token": {
+        "description": "The databrick lakehouse api token.",
+        "type": "string",
+        "outhad_secret": true,
+        "title": "API Token",
+        "order": 1
+      },"warehouse_id": {
+        "description": "The databrick lakehouse warehouse ID.",
+        "type": "string",
+        "title": "Warehouse ID",
+        "order": 2
+      },
+      "catalog": {
+        "description": "The name of the catalog",
+        "default": "hive_metastore",
+        "type": "string",
+        "title": "Databricks catalog",
+        "order": 3
+      },
+      "schema": {
+        "description": "The default schema tables are written.",
+        "default": "default",
+        "type": "string",
+        "title": "Database schema",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/outhad/integrations/destination/databricks_lakehouse/icon.svg

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Generator: Adobe Illustrator 18.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   id="Layer_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 64 64"
+   enable-background="new 0 0 640 96"
+   xml:space="preserve"
+   sodipodi:docname="databricks-icon.svg"
+   width="64"
+   height="64"
+   inkscape:version="0.92.4 (5da689c313, 2019-01-14)"><metadata
+   id="metadata45"><rdf:RDF><cc:Work
+       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+   id="defs43" /><sodipodi:namedview
+   pagecolor="#ffffff"
+   bordercolor="#666666"
+   borderopacity="1"
+   objecttolerance="10"
+   gridtolerance="10"
+   guidetolerance="10"
+   inkscape:pageopacity="0"
+   inkscape:pageshadow="2"
+   inkscape:window-width="1920"
+   inkscape:window-height="1001"
+   id="namedview41"
+   showgrid="false"
+   inkscape:zoom="2.996365"
+   inkscape:cx="53.009829"
+   inkscape:cy="48"
+   inkscape:window-x="-9"
+   inkscape:window-y="-9"
+   inkscape:window-maximized="1"
+   inkscape:current-layer="Layer_1" />
+
+
+
+<g
+   id="g50"
+   transform="matrix(0.75294118,0,0,0.75294118,1.0541175,18.823529)"><path
+     style="fill:#db1905"
+     inkscape:connector-curvature="0"
+     id="path2"
+     d="M 0,24.8 V 38.5 L 41.1,60 82.2,38.5 v -13.7 0 L 67.7,17.4 82.2,9.8 v -13.6 0 0 -0.1 L 82.1,-3.8 41.1,-25 0.1,-3.9 H 0 V 9.8 L 14.5,17.4 0,24.8" /><polygon
+     transform="translate(0,-36)"
+     style="fill:#ff5224"
+     id="polygon4"
+     points="0,60.8 41.1,82.3 82.2,60.8 67.7,53.4 41.1,67.3 14.5,53.4 " /><polygon
+     transform="translate(0,-36)"
+     style="fill:#ff5224"
+     id="polygon6"
+     points="82.2,32.2 41.1,53.7 0,32.2 41.1,11 " /></g>
+
+</svg>