RubyGems - multiwoven-integrations - Versions diffs - 0.5.2 → 0.7.0 - Mend

multiwoven-integrations 0.5.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (14) hide show

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fdecb7677ac11971794080ac898200969ee1b243e917ccba7f1c7ce254ebdcb5
-  data.tar.gz: f609d9a9cfcef3644b07352d6704c713070046c5b8e405017e3e7f8c11d112e6
+  metadata.gz: 697c4972edd603a75aa7fd96653ce86c5946f90d90acf859d2577e8a1314608f
+  data.tar.gz: 6295ce58e8cdbe4c6f562202362a8e33c3dc0dd9db1dc368ec4be5cfce6ee3a9
 SHA512:
-  metadata.gz: 98d9e5c0028a6e6d3b45584ba8cbd01778040ec0a2a7cf77a1dc2e1a326735df427021300e3b7cf412ee270e8bbe67101c9148cf203518d6504f1af6fd32b2cd
-  data.tar.gz: f4aece39d2bdcb2901b2c9981e1ad4ca6ddb24333b8ab057b2fdd275c3496404569ac830abcd8d690f48ac39d493c661b305e119eee5a286d4dd211e8ba23d65
+  metadata.gz: 4868f70099d7fd8f8454e50371ab29ae622749175df9bcfa29f992223ef6b88946723442653f6ec82837326e506b86b583382478bd5fc5c4af94fa93eeccb594
+  data.tar.gz: 7a02dab7bf70873cf95455cf1906a7357c156803c4037c9adeb1e0dfb47ebe55cc0b24f219a0ba71152eb996b6e11ab244139d2267b2db5b8998dcd3dbe99dd0

data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json CHANGED Viewed

@@ -23,7 +23,8 @@
             "type": "string"
           },
           "private_key_id": {
-            "type": "string"
+            "type": "string",
+            "multiwoven_secret": true
           },
           "private_key": {
             "type": "string",
@@ -34,19 +35,17 @@
             "format": "email"
           },
           "client_id": {
-            "type": "string"
+            "type": "string",
+            "multiwoven_secret": true
           },
           "auth_uri": {
-            "type": "string",
-            "format": "uri"
+            "type": "string"
           },
           "token_uri": {
-            "type": "string",
-            "format": "uri"
+            "type": "string"
           },
           "auth_provider_x509_cert_url": {
-            "type": "string",
-            "format": "uri"
+            "type": "string"
           },
           "client_x509_cert_url": {
             "type": "string",

data/lib/multiwoven/integrations/destination/oracle_db/client.rb ADDED Viewed

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+module Multiwoven::Integrations::Destination
+  module Oracle
+    include Multiwoven::Integrations::Core
+    class Client < DestinationConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_multiwoven_message
+      end
+      def discover(connection_config)
+        records = []
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, nullable
+                 FROM all_tab_columns
+                 WHERE owner = '#{connection_config[:username].upcase}'
+                 ORDER BY table_name, column_id"
+        conn = create_connection(connection_config)
+        cursor = conn.exec(query)
+        while (row = cursor.fetch)
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "ORACLE:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+      def write(sync_config, records, action = "destination_insert")
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        table_name = sync_config.stream.name
+        primary_key = sync_config.model.primary_key
+        conn = create_connection(connection_config)
+        write_success = 0
+        write_failure = 0
+        log_message_array = []
+        records.each do |record|
+          query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
+          query = query.gsub(";", "")
+          logger.debug("ORACLE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
+          begin
+            response = conn.exec(query)
+            conn.exec("COMMIT")
+            write_success += 1
+            log_message_array << log_request_response("info", query, response)
+          rescue StandardError => e
+            handle_exception(e, {
+                               context: "ORACLE:RECORD:WRITE:EXCEPTION",
+                               type: "error",
+                               sync_id: sync_config.sync_id,
+                               sync_run_id: sync_config.sync_run_id
+                             })
+            write_failure += 1
+            log_message_array << log_request_response("error", query, e.message)
+          end
+        end
+        tracking_message(write_success, write_failure, log_message_array)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "ORACLE:RECORD:WRITE:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+      private
+      def create_connection(connection_config)
+        OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
+      end
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry[0]
+          column_data = {
+            column_name: entry[1],
+            data_type: entry[2],
+            is_nullable: entry[3] == "Y"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
+          { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/oracle_db/config/meta.json ADDED Viewed

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Oracle",
+    "title": "Oracle",
+    "connector_type": "destination",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle",
+    "github_issue_label": "destination-oracle",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/destination/oracle_db/config/spec.json ADDED Viewed

@@ -0,0 +1,47 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Oracle",
+    "type": "object",
+    "required": ["host", "port", "sid", "username", "password"],
+    "properties": {
+      "host": {
+        "description": "The Oracle host.",
+        "examples": ["localhost"],
+        "type": "string",
+        "title": "Host",
+        "order": 0
+      },
+      "port": {
+        "description": "The Oracle port number.",
+        "examples": ["1521"],
+        "type": "string",
+        "title": "Port",
+        "order": 1
+      },
+      "sid": {
+        "description": "The name of your service in Oracle.",
+        "examples": ["ORCLPDB1"],
+        "type": "string",
+        "title": "SID",
+        "order": 2
+      },
+      "username": {
+        "description": "The username used to authenticate and connect.",
+        "type": "string",
+        "title": "Username",
+        "order": 3
+      },
+      "password": {
+        "description": "The password corresponding to the username used for authentication.",
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Password",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/destination/oracle_db/icon.svg ADDED Viewed

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
+  <path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
+</svg>

data/lib/multiwoven/integrations/rollout.rb CHANGED Viewed

@@ -2,7 +2,7 @@
 module Multiwoven
   module Integrations
-    VERSION = "0.5.2"
+    VERSION = "0.7.0"
     ENABLED_SOURCES = %w[
       Snowflake
@@ -15,6 +15,7 @@ module Multiwoven
       Clickhouse
       AmazonS3
       MariaDB
+      Oracle
     ].freeze
     ENABLED_DESTINATIONS = %w[
@@ -34,6 +35,7 @@ module Multiwoven
       Iterable
       MariaDB
       DatabricksLakehouse
+      Oracle
     ].freeze
   end
 end

data/lib/multiwoven/integrations/source/oracle_db/client.rb ADDED Viewed

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+module Multiwoven::Integrations::Source
+  module Oracle
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_multiwoven_message
+      end
+      def discover(connection_config)
+        records = []
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, nullable
+                 FROM all_tab_columns
+                 WHERE owner = '#{connection_config[:username].upcase}'
+                 ORDER BY table_name, column_id"
+        conn = create_connection(connection_config)
+        cursor = conn.exec(query)
+        while (row = cursor.fetch)
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "ORACLE:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        query = sync_config.model.query
+        db = create_connection(connection_config)
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "ORACLE:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+      private
+      def create_connection(connection_config)
+        OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
+      end
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+      def query(connection, query)
+        records = []
+        query = reformat_query(query)
+        cursor = connection.exec(query)
+        columns = cursor.get_col_names
+        while (row = cursor.fetch)
+          data_hash = columns.zip(row).to_h
+          records << RecordMessage.new(data: data_hash, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry[0]
+          column_data = {
+            column_name: entry[1],
+            data_type: entry[2],
+            is_nullable: entry[3] == "Y"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
+          { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
+        end
+      end
+      def reformat_query(sql_query)
+        offset = nil
+        limit = nil
+        sql_query = sql_query.gsub(";", "")
+        if sql_query.match?(/LIMIT (\d+)/i)
+          limit = sql_query.match(/LIMIT (\d+)/i)[1].to_i
+          sql_query.sub!(/LIMIT \d+/i, "")
+        end
+        if sql_query.match?(/OFFSET (\d+)/i)
+          offset = sql_query.match(/OFFSET (\d+)/i)[1].to_i
+          sql_query.sub!(/OFFSET \d+/i, "")
+        end
+        sql_query.strip!
+        if offset && limit
+          "#{sql_query} OFFSET #{offset} ROWS FETCH NEXT #{limit} ROWS ONLY"
+        elsif offset
+          "#{sql_query} OFFSET #{offset} ROWS"
+        elsif limit
+          "#{sql_query} FETCH NEXT #{limit} ROWS ONLY"
+        else
+          sql_query
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/oracle_db/config/meta.json ADDED Viewed

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Oracle",
+    "title": "Oracle",
+    "connector_type": "source",
+    "category": "Database",
+    "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
+    "github_issue_label": "source-oracle",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/oracle_db/config/spec.json ADDED Viewed

@@ -0,0 +1,47 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Oracle",
+    "type": "object",
+    "required": ["host", "port", "sid", "username", "password"],
+    "properties": {
+      "host": {
+        "description": "The Oracle host.",
+        "examples": ["localhost"],
+        "type": "string",
+        "title": "Host",
+        "order": 0
+      },
+      "port": {
+        "description": "The Oracle port number.",
+        "examples": ["1521"],
+        "type": "string",
+        "title": "Port",
+        "order": 1
+      },
+      "sid": {
+        "description": "The name of your service in Oracle.",
+        "examples": ["ORCLPDB1"],
+        "type": "string",
+        "title": "SID",
+        "order": 2
+      },
+      "username": {
+        "description": "The username used to authenticate and connect.",
+        "type": "string",
+        "title": "Username",
+        "order": 3
+      },
+      "password": {
+        "description": "The password corresponding to the username used for authentication.",
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Password",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/oracle_db/icon.svg ADDED Viewed

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
+  <path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
+</svg>

data/lib/multiwoven/integrations.rb CHANGED Viewed

@@ -31,6 +31,7 @@ require "aws-sdk-s3"
 require "duckdb"
 require "iterable-api-client"
 require "aws-sdk-sts"
+require "ruby-oci8"
 # Service
 require_relative "integrations/config"
@@ -60,6 +61,7 @@ require_relative "integrations/source/aws_athena/client"
 require_relative "integrations/source/clickhouse/client"
 require_relative "integrations/source/amazon_s3/client"
 require_relative "integrations/source/maria_db/client"
+require_relative "integrations/source/oracle_db/client"
 # Destination
 require_relative "integrations/destination/klaviyo/client"
@@ -78,6 +80,7 @@ require_relative "integrations/destination/zendesk/client"
 require_relative "integrations/destination/iterable/client"
 require_relative "integrations/destination/maria_db/client"
 require_relative "integrations/destination/databricks_lakehouse/client"
+require_relative "integrations/destination/oracle_db/client"
 module Multiwoven
   module Integrations

data/multiwoven-integrations.gemspec CHANGED Viewed

@@ -53,6 +53,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "rake"
   spec.add_runtime_dependency "restforce"
   spec.add_runtime_dependency "ruby-limiter"
+  spec.add_runtime_dependency "ruby-oci8"
   spec.add_runtime_dependency "ruby-odbc"
   spec.add_runtime_dependency "rubyzip"
   spec.add_runtime_dependency "sequel"

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-07-16 00:00:00.000000000 Z
+date: 2024-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -290,6 +290,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-oci8
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-odbc
   requirement: !ruby/object:Gem::Requirement
@@ -530,6 +544,10 @@ files:
 - lib/multiwoven/integrations/destination/maria_db/config/meta.json
 - lib/multiwoven/integrations/destination/maria_db/config/spec.json
 - lib/multiwoven/integrations/destination/maria_db/icon.svg
+- lib/multiwoven/integrations/destination/oracle_db/client.rb
+- lib/multiwoven/integrations/destination/oracle_db/config/meta.json
+- lib/multiwoven/integrations/destination/oracle_db/config/spec.json
+- lib/multiwoven/integrations/destination/oracle_db/icon.svg
 - lib/multiwoven/integrations/destination/postgresql/client.rb
 - lib/multiwoven/integrations/destination/postgresql/config/meta.json
 - lib/multiwoven/integrations/destination/postgresql/config/spec.json
@@ -593,6 +611,10 @@ files:
 - lib/multiwoven/integrations/source/maria_db/config/meta.json
 - lib/multiwoven/integrations/source/maria_db/config/spec.json
 - lib/multiwoven/integrations/source/maria_db/icon.svg
+- lib/multiwoven/integrations/source/oracle_db/client.rb
+- lib/multiwoven/integrations/source/oracle_db/config/meta.json
+- lib/multiwoven/integrations/source/oracle_db/config/spec.json
+- lib/multiwoven/integrations/source/oracle_db/icon.svg
 - lib/multiwoven/integrations/source/postgresql/client.rb
 - lib/multiwoven/integrations/source/postgresql/config/meta.json
 - lib/multiwoven/integrations/source/postgresql/config/spec.json