multiwoven-integrations 0.5.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fdecb7677ac11971794080ac898200969ee1b243e917ccba7f1c7ce254ebdcb5
-  data.tar.gz: f609d9a9cfcef3644b07352d6704c713070046c5b8e405017e3e7f8c11d112e6
+  metadata.gz: 697c4972edd603a75aa7fd96653ce86c5946f90d90acf859d2577e8a1314608f
+  data.tar.gz: 6295ce58e8cdbe4c6f562202362a8e33c3dc0dd9db1dc368ec4be5cfce6ee3a9
 SHA512:
-  metadata.gz: 98d9e5c0028a6e6d3b45584ba8cbd01778040ec0a2a7cf77a1dc2e1a326735df427021300e3b7cf412ee270e8bbe67101c9148cf203518d6504f1af6fd32b2cd
-  data.tar.gz: f4aece39d2bdcb2901b2c9981e1ad4ca6ddb24333b8ab057b2fdd275c3496404569ac830abcd8d690f48ac39d493c661b305e119eee5a286d4dd211e8ba23d65
+  metadata.gz: 4868f70099d7fd8f8454e50371ab29ae622749175df9bcfa29f992223ef6b88946723442653f6ec82837326e506b86b583382478bd5fc5c4af94fa93eeccb594
+  data.tar.gz: 7a02dab7bf70873cf95455cf1906a7357c156803c4037c9adeb1e0dfb47ebe55cc0b24f219a0ba71152eb996b6e11ab244139d2267b2db5b8998dcd3dbe99dd0

data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json

@@ -23,7 +23,8 @@
             "type": "string"
           },
           "private_key_id": {
-            "type": "string"
+            "type": "string",
+            "multiwoven_secret": true
           },
           "private_key": {
             "type": "string",
@@ -34,19 +35,17 @@
             "format": "email"
           },
           "client_id": {
-            "type": "string"
+            "type": "string",
+            "multiwoven_secret": true
           },
           "auth_uri": {
-            "type": "string",
-            "format": "uri"
+            "type": "string"
           },
           "token_uri": {
-            "type": "string",
-            "format": "uri"
+            "type": "string"
           },
           "auth_provider_x509_cert_url": {
-            "type": "string",
-            "format": "uri"
+            "type": "string"
           },
           "client_x509_cert_url": {
             "type": "string",

data/lib/multiwoven/integrations/destination/oracle_db/client.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Destination
+  module Oracle
+    include Multiwoven::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        records = []
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, nullable
+                 FROM all_tab_columns
+                 WHERE owner = '#{connection_config[:username].upcase}'
+                 ORDER BY table_name, column_id"
+        conn = create_connection(connection_config)
+        cursor = conn.exec(query)
+        while (row = cursor.fetch)
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "ORACLE:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def write(sync_config, records, action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        table_name = sync_config.stream.name
+        primary_key = sync_config.model.primary_key
+        conn = create_connection(connection_config)
+
+        write_success = 0
+        write_failure = 0
+        log_message_array = []
+
+        records.each do |record|
+          query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
+          query = query.gsub(";", "")
+          logger.debug("ORACLE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+          begin
+            response = conn.exec(query)
+            conn.exec("COMMIT")
+            write_success += 1
+            log_message_array << log_request_response("info", query, response)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "ORACLE:RECORD:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+            write_failure += 1
+            log_message_array << log_request_response("error", query, e.message)
+          end
+        end
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "ORACLE:RECORD:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry[0]
+          column_data = {
+            column_name: entry[1],
+            data_type: entry[2],
+            is_nullable: entry[3] == "Y"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
+          { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/oracle_db/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Oracle",
+    "title": "Oracle",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle",
+    "github_issue_label": "destination-oracle",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/destination/oracle_db/config/spec.json

@@ -0,0 +1,47 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Oracle",
+    "type": "object",
+    "required": ["host", "port", "sid", "username", "password"],
+    "properties": {
+      "host": {
+        "description": "The Oracle host.",
+        "examples": ["localhost"],
+        "type": "string",
+        "title": "Host",
+        "order": 0
+      },
+      "port": {
+        "description": "The Oracle port number.",
+        "examples": ["1521"],
+        "type": "string",
+        "title": "Port",
+        "order": 1
+      },
+      "sid": {
+        "description": "The name of your service in Oracle.",
+        "examples": ["ORCLPDB1"],
+        "type": "string",
+        "title": "SID",
+        "order": 2
+      },
+      "username": {
+        "description": "The username used to authenticate and connect.",
+        "type": "string",
+        "title": "Username",
+        "order": 3
+      },
+      "password": {
+        "description": "The password corresponding to the username used for authentication.",
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Password",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/destination/oracle_db/icon.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
+  <path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
+</svg>

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.5.2"
+    VERSION = "0.7.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -15,6 +15,7 @@ module Multiwoven
       Clickhouse
       AmazonS3
       MariaDB
+      Oracle
     ].freeze
 
     ENABLED_DESTINATIONS = %w[
@@ -34,6 +35,7 @@ module Multiwoven
       Iterable
       MariaDB
       DatabricksLakehouse
+      Oracle
     ].freeze
   end
 end

data/lib/multiwoven/integrations/source/oracle_db/client.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module Oracle
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        records = []
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, nullable
+                 FROM all_tab_columns
+                 WHERE owner = '#{connection_config[:username].upcase}'
+                 ORDER BY table_name, column_id"
+        conn = create_connection(connection_config)
+        cursor = conn.exec(query)
+        while (row = cursor.fetch)
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "ORACLE:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        query = sync_config.model.query
+        db = create_connection(connection_config)
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "ORACLE:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def query(connection, query)
+        records = []
+        query = reformat_query(query)
+        cursor = connection.exec(query)
+        columns = cursor.get_col_names
+        while (row = cursor.fetch)
+          data_hash = columns.zip(row).to_h
+          records << RecordMessage.new(data: data_hash, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry[0]
+          column_data = {
+            column_name: entry[1],
+            data_type: entry[2],
+            is_nullable: entry[3] == "Y"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
+          { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
+        end
+      end
+
+      def reformat_query(sql_query)
+        offset = nil
+        limit = nil
+
+        sql_query = sql_query.gsub(";", "")
+
+        if sql_query.match?(/LIMIT (\d+)/i)
+          limit = sql_query.match(/LIMIT (\d+)/i)[1].to_i
+          sql_query.sub!(/LIMIT \d+/i, "")
+        end
+
+        if sql_query.match?(/OFFSET (\d+)/i)
+          offset = sql_query.match(/OFFSET (\d+)/i)[1].to_i
+          sql_query.sub!(/OFFSET \d+/i, "")
+        end
+
+        sql_query.strip!
+
+        if offset && limit
+          "#{sql_query} OFFSET #{offset} ROWS FETCH NEXT #{limit} ROWS ONLY"
+        elsif offset
+          "#{sql_query} OFFSET #{offset} ROWS"
+        elsif limit
+          "#{sql_query} FETCH NEXT #{limit} ROWS ONLY"
+        else
+          sql_query
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/oracle_db/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Oracle",
+    "title": "Oracle",
+    "connector_type": "source",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
+    "github_issue_label": "source-oracle",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/oracle_db/config/spec.json

@@ -0,0 +1,47 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Oracle",
+    "type": "object",
+    "required": ["host", "port", "sid", "username", "password"],
+    "properties": {
+      "host": {
+        "description": "The Oracle host.",
+        "examples": ["localhost"],
+        "type": "string",
+        "title": "Host",
+        "order": 0
+      },
+      "port": {
+        "description": "The Oracle port number.",
+        "examples": ["1521"],
+        "type": "string",
+        "title": "Port",
+        "order": 1
+      },
+      "sid": {
+        "description": "The name of your service in Oracle.",
+        "examples": ["ORCLPDB1"],
+        "type": "string",
+        "title": "SID",
+        "order": 2
+      },
+      "username": {
+        "description": "The username used to authenticate and connect.",
+        "type": "string",
+        "title": "Username",
+        "order": 3
+      },
+      "password": {
+        "description": "The password corresponding to the username used for authentication.",
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Password",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/oracle_db/icon.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
+  <path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
+</svg>

data/lib/multiwoven/integrations.rb

@@ -31,6 +31,7 @@ require "aws-sdk-s3"
 require "duckdb"
 require "iterable-api-client"
 require "aws-sdk-sts"
+require "ruby-oci8"
 
 # Service
 require_relative "integrations/config"
@@ -60,6 +61,7 @@ require_relative "integrations/source/aws_athena/client"
 require_relative "integrations/source/clickhouse/client"
 require_relative "integrations/source/amazon_s3/client"
 require_relative "integrations/source/maria_db/client"
+require_relative "integrations/source/oracle_db/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"
@@ -78,6 +80,7 @@ require_relative "integrations/destination/zendesk/client"
 require_relative "integrations/destination/iterable/client"
 require_relative "integrations/destination/maria_db/client"
 require_relative "integrations/destination/databricks_lakehouse/client"
+require_relative "integrations/destination/oracle_db/client"
 
 module Multiwoven
   module Integrations

data/multiwoven-integrations.gemspec

@@ -53,6 +53,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "rake"
   spec.add_runtime_dependency "restforce"
   spec.add_runtime_dependency "ruby-limiter"
+  spec.add_runtime_dependency "ruby-oci8"
   spec.add_runtime_dependency "ruby-odbc"
   spec.add_runtime_dependency "rubyzip"
   spec.add_runtime_dependency "sequel"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-07-16 00:00:00.000000000 Z
+date: 2024-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -290,6 +290,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-oci8
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-odbc
   requirement: !ruby/object:Gem::Requirement
@@ -530,6 +544,10 @@ files:
 - lib/multiwoven/integrations/destination/maria_db/config/meta.json
 - lib/multiwoven/integrations/destination/maria_db/config/spec.json
 - lib/multiwoven/integrations/destination/maria_db/icon.svg
+- lib/multiwoven/integrations/destination/oracle_db/client.rb
+- lib/multiwoven/integrations/destination/oracle_db/config/meta.json
+- lib/multiwoven/integrations/destination/oracle_db/config/spec.json
+- lib/multiwoven/integrations/destination/oracle_db/icon.svg
 - lib/multiwoven/integrations/destination/postgresql/client.rb
 - lib/multiwoven/integrations/destination/postgresql/config/meta.json
 - lib/multiwoven/integrations/destination/postgresql/config/spec.json
@@ -593,6 +611,10 @@ files:
 - lib/multiwoven/integrations/source/maria_db/config/meta.json
 - lib/multiwoven/integrations/source/maria_db/config/spec.json
 - lib/multiwoven/integrations/source/maria_db/icon.svg
+- lib/multiwoven/integrations/source/oracle_db/client.rb
+- lib/multiwoven/integrations/source/oracle_db/config/meta.json
+- lib/multiwoven/integrations/source/oracle_db/config/spec.json
+- lib/multiwoven/integrations/source/oracle_db/icon.svg
 - lib/multiwoven/integrations/source/postgresql/client.rb
 - lib/multiwoven/integrations/source/postgresql/config/meta.json
 - lib/multiwoven/integrations/source/postgresql/config/spec.json