multiwoven-integrations 0.1.68 → 0.1.70

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77816f54f13495e303382ef4c1e2e932472475d31e0f486d3dcc0535e561f36d
-  data.tar.gz: 854a6871e65a3ad4764548ea14793a04c02d42cde0ed31712c252aff960134ab
+  metadata.gz: 86c32dc17e06c1d25daf3444873c1a368fae14b9be61970a08413e7122606859
+  data.tar.gz: a9fde38fbfa372e81d61f1b007a390ef5fde4ae25aed3b08a0f008bf8aea3ba3
 SHA512:
-  metadata.gz: 0de8475fa0b27194c1a61b58d56f11fe3a5df8058c6065664a2c67c7af5c5d4c709aab352c0be4c7f53b9f62fda1031a47f1590844755b54897b38a652173306
-  data.tar.gz: dce5ca510e72bccf871075a1a5dd523db56961454b8d4ba987e6037b219f07683a79b1dabc03e51e9b533b18af52ce048ad60684fb2f9ee6e619c9fac272316b
+  metadata.gz: 3deaadcb0c86572b5d8f4030ca19859a51142c8d7bf2725e990dd748013f91b28941fd724cdb3e56f4d2b0bc54bd9c20fcefda8e55f9b198a62866d770f2bb44
+  data.tar.gz: 3728413ea4d86eca6901ea1ec6b5d851edf2a4e321778884596bc0699e210a3235f79b5871e44418bc3e28b989990815672cf5d1541177c2b51a1fc6e4ccad02

data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb

@@ -14,6 +14,16 @@ module Multiwoven
 
         class Client < DestinationConnector
           prepend Multiwoven::Integrations::Core::RateLimiter
+
+          def initialize
+            super
+            @logger = Integrations::Service.logger
+            Restforce.configure do |config|
+              config.logger = @logger
+            end
+            Restforce.log = true
+          end
+
           def check_connection(connection_config)
             connection_config = connection_config.with_indifferent_access
             initialize_client(connection_config)
@@ -39,6 +49,8 @@ module Multiwoven
 
           def write(sync_config, records, action = "create")
             @action = sync_config.stream.action || action
+            @sync_id = sync_config.sync_id
+            @sync_run_id = sync_config.sync_run_id
             initialize_client(sync_config.destination.connection_specification)
             process_records(records, sync_config.stream)
           rescue StandardError => e
@@ -79,6 +91,7 @@ module Multiwoven
           def send_data_to_salesforce(stream_name, record = {})
             method_name = "upsert!"
             args = [stream_name, "Id", record]
+            @logger.debug("sync_id: #{@sync_id}, sync_run_id: #{@sync_run_id}, record: #{record}")
             @client.send(method_name, *args)
           end
 

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -175,6 +175,8 @@ module Multiwoven
       attribute? :cursor_field, Types::String.optional
       attribute? :current_cursor_field, Types::String.optional
       attribute :destination_sync_mode, DestinationSyncMode
+      attribute? :sync_id, Types::String.optional
+      attribute? :sync_run_id, Types::String.optional
     end
 
     class ControlMessage < ProtocolModel

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.68"
+    VERSION = "0.1.70"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -12,6 +12,7 @@ module Multiwoven
       Databricks
       SalesforceConsumerGoodsCloud
       AwsAthena
+      Clickhouse
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/clickhouse/client.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module Clickhouse
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:database]}' ORDER BY table_name, ordinal_position;"
+        db = create_connection(connection_config)
+        records = query_execution(db, query)
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "CLICKHOUSE:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        db = create_connection(connection_config)
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(
+          "CLICKHOUSE:READ:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      private
+
+      def query(connection, query)
+        query_execution(connection, query).map do |row|
+          RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+
+      def create_connection(connection_config)
+        @auth_token = Base64.strict_encode64("#{connection_config[:username]}:#{connection_config[:password]}")
+        Faraday.new(connection_config[:host]) do |faraday|
+          faraday.request :url_encoded
+          faraday.adapter Faraday.default_adapter
+        end
+      end
+
+      def query_execution(connection, query)
+        response = connection.post do |req|
+          req.url "/"
+          req.headers["Authorization"] = "Basic #{@auth_token}"
+          req.headers["Content-Type"] = "text/plain"
+          req.body = query
+        end
+        column_names = query[/SELECT (.*?) FROM/i, 1].split(",").map(&:strip)
+        response.body.strip.split("\n").map do |row|
+          columns = row.split("\t")
+          column_names.zip(columns).to_h
+        end
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |_, r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        result = {}
+        records.each_with_index do |entry, index|
+          table_name = entry["table_name"]
+          column_data = {
+            column_name: entry["column_name"],
+            data_type: entry["data_type"].gsub(/Nullable\((\w+)\)/, '\1').downcase.gsub!(/\d+/, ""),
+            is_nullable: entry["is_nullable"] == "1"
+          }
+          result[index] ||= {}
+          result[index][:tablename] = table_name
+          result[index][:columns] = [column_data]
+        end
+        result
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/clickhouse/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Clickhouse",
+    "title": "ClickHouse",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "documentation_url": "https://docs.multiwoven.com/integrations/sources/clickhouse",
+    "github_issue_label": "source-clickhouse",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/clickhouse/config/spec.json

@@ -0,0 +1,42 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/clickhouse",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "ClickHouse",
+    "type": "object",
+    "required": ["url", "username", "password", "database"],
+    "properties": {
+      "url": {
+        "description": "The ClickHouse host url to connect.",
+        "examples": ["tu61szglca.us-west-2.aws.clickhouse.cloud"],
+        "type": "string",
+        "title": "Personal URL",
+        "order": 0
+      },
+      "username": {
+        "description": "The username for ClickHouse.",
+        "examples": ["Default"],
+        "type": "string",
+        "title": "Username",
+        "order": 1
+      },
+      "password": {
+        "description": "The password for ClickHouse.",
+        "examples": ["Default"],
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "Password",
+        "order": 2
+      },
+      "database": {
+        "description": "The ClickHouse database.",
+        "examples": ["default"],
+        "type": "string",
+        "title": "Database",
+        "order": 3
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/clickhouse/icon.svg

@@ -0,0 +1,4 @@
+<svg height="2222" viewBox="0 0 9 8" width="2500" xmlns="http://www.w3.org/2000/svg">
+    <path d="m0 7h1v1h-1z" fill="#f00"/>
+    <path d="m0 0h1v7h-1zm2 0h1v8h-1zm2 0h1v8h-1zm2 0h1v8h-1zm2 3.25h1v1.5h-1z" fill="#fc0"/>
+</svg>

data/lib/multiwoven/integrations.rb

@@ -25,6 +25,8 @@ require "securerandom"
 require "aws-sdk-athena"
 require "zip"
 require "zendesk_api"
+require "faraday"
+require "base64"
 
 # Service
 require_relative "integrations/config"
@@ -51,6 +53,7 @@ require_relative "integrations/source/postgresql/client"
 require_relative "integrations/source/databricks/client"
 require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
 require_relative "integrations/source/aws_athena/client"
+require_relative "integrations/source/clickhouse/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.68
+  version: 0.1.70
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-05-21 00:00:00.000000000 Z
+date: 2024-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -508,6 +508,10 @@ files:
 - lib/multiwoven/integrations/source/bigquery/config/meta.json
 - lib/multiwoven/integrations/source/bigquery/config/spec.json
 - lib/multiwoven/integrations/source/bigquery/icon.svg
+- lib/multiwoven/integrations/source/clickhouse/client.rb
+- lib/multiwoven/integrations/source/clickhouse/config/meta.json
+- lib/multiwoven/integrations/source/clickhouse/config/spec.json
+- lib/multiwoven/integrations/source/clickhouse/icon.svg
 - lib/multiwoven/integrations/source/databricks/client.rb
 - lib/multiwoven/integrations/source/databricks/config/meta.json
 - lib/multiwoven/integrations/source/databricks/config/spec.json