etl-integrations 0.1.81

data/lib/multiwoven/integrations/rollout.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    VERSION = "0.1.81"
+
+    ENABLED_SOURCES = %w[
+      Snowflake
+      Redshift
+      Bigquery
+      Postgresql
+      Databricks
+      SalesforceConsumerGoodsCloud
+      ZohoBooks
+    ].freeze
+
+    ENABLED_DESTINATIONS = %w[
+      Klaviyo
+      SalesforceCrm
+      FacebookCustomAudience
+      Slack
+      Hubspot
+      GoogleSheets
+      Airtable
+      Stripe
+      SalesforceConsumerGoodsCloud
+      Sftp
+      Postgresql
+      Tally
+    ].freeze
+  end
+end

data/lib/multiwoven/integrations/service.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    class Service
+      def initialize
+        yield(self.class.config) if block_given?
+      end
+      class << self
+        def connectors
+          {
+            source: build_connectors(
+              ENABLED_SOURCES, "Source"
+            ),
+            destination: build_connectors(
+              ENABLED_DESTINATIONS, "Destination"
+            )
+          }
+        end
+
+        def connector_class(connector_type, connector_name) 
+          # *args
+          Object.const_get(
+            "Multiwoven::Integrations::#{connector_type}::#{connector_name}::Client"
+          )
+
+          # # Retrieve the class dynamically
+          # klass = Object.const_get(
+          #   "Multiwoven::Integrations::#{connector_type}::#{connector_name}::Client"
+          # )
+
+          # # Instantiate the class (assuming it's not a singleton)
+          # client_instance = klass.new
+
+          # # Only apply base_url if connector_name starts with "Zoho" or "zoho"
+          # if connector_name.match?(/^Zoho/i)
+          #   # Check for base_url in the args
+          #   base_url = args.find { |arg| arg.is_a?(String) && arg.start_with?("http") }
+
+          #   # If base_url is found, call the method to set it
+          #   if base_url
+          #     if client_instance.respond_to?(:setting_base_url)
+          #       client_instance.setting_base_url(base_url)
+          #     else
+          #       raise NoMethodError, "Undefined method `setting_base_url` for #{client_instance.class}"
+          #     end
+          #   end
+          # end
+
+          # # Return the instance for further use
+          # klass
+
+        end
+
+        def logger
+          config.logger || default_logger
+        end
+
+        def config
+          @config ||= Config.new
+        end
+
+        private
+
+        def build_connectors(enabled_connectors, type)
+          enabled_connectors.map do |connector|
+            client = connector_class(type, connector).new
+            client.meta_data[:data][:connector_spec] = client.connector_spec.to_h
+            client.meta_data[:data]
+          end
+        end
+
+        def default_logger
+          @default_logger ||= Logger.new($stdout)
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/bigquery/client.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "google/cloud/bigquery"
+
+module Multiwoven::Integrations::Source
+  module Bigquery
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        bigquery = create_connection(connection_config)
+        bigquery.datasets
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        bigquery = create_connection(connection_config)
+        target_dataset_id = connection_config["dataset_id"]
+        records = bigquery.datasets.flat_map do |dataset|
+          next unless dataset.dataset_id == target_dataset_id
+
+          dataset.tables.flat_map do |table|
+            table.schema.fields.map do |field|
+              {
+                table_name: table.table_id,
+                column_name: field.name,
+                data_type: field.type,
+                is_nullable: field.mode == "NULLABLE"
+              }
+            end
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "BIGQUERY:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        bigquery = create_connection(connection_config)
+
+        query(bigquery, query)
+      rescue StandardError => e
+        handle_exception(
+          "BIGQUERY:READ:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      private
+
+      def query(connection, query)
+        records = []
+        results = connection.query(query) || []
+        results.each do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+
+      def create_connection(connection_config)
+        Google::Cloud::Bigquery.new(
+          project: connection_config["project_id"],
+          credentials: connection_config["credentials_json"]
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/bigquery/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Bigquery",
+    "title": "Google BigQuery",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-bigquery",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/bigquery/config/spec.json

@@ -0,0 +1,82 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/bigquery",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Google BigQuery",
+    "type": "object",
+    "required": ["project_id", "credentials_json"],
+    "properties": {
+      "project_id": {
+        "type": "string",
+        "description": "The project ID, serving as the unique identifier for the Google Cloud project associated with your BigQuery data warehouse, can be located within the Google Cloud web console. This is achieved by either selecting your project name from the top navigation bar or by navigating to the project settings page.",
+        "title": "Project ID"
+      },
+      "dataset_id": {
+        "type": "string",
+        "description": "The dataset ID to search for tables and views. If you are only loading data from one dataset, setting this option could result in much faster schema discovery. You can get ",
+        "title": "Default Dataset ID"
+      },
+      "credentials_json": {
+        "type": "object",
+        "description": "You can get the keys from the Google Cloud web console. First, go to the IAM page and select Service Accounts from the left menu. Next, locate your service account in the list, click on its Keys tab, and then click Add Key. Lastly, click Create new key and select JSON.",
+        "title": "",
+        "properties": {
+          "type": {
+            "type": "string",
+            "enum": ["service_account"]
+          },
+          "project_id": {
+            "type": "string"
+          },
+          "private_key_id": {
+            "type": "string"
+          },
+          "private_key": {
+            "type": "string"
+          },
+          "client_email": {
+            "type": "string",
+            "format": "email"
+          },
+          "client_id": {
+            "type": "string"
+          },
+          "auth_uri": {
+            "type": "string",
+            "format": "uri"
+          },
+          "token_uri": {
+            "type": "string",
+            "format": "uri"
+          },
+          "auth_provider_x509_cert_url": {
+            "type": "string",
+            "format": "uri"
+          },
+          "client_x509_cert_url": {
+            "type": "string",
+            "format": "uri"
+          },
+          "universe_domain": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "type",
+          "project_id",
+          "private_key_id",
+          "private_key",
+          "client_email",
+          "client_id",
+          "auth_uri",
+          "token_uri",
+          "auth_provider_x509_cert_url",
+          "client_x509_cert_url",
+          "universe_domain"
+        ]
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/bigquery/icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="-1.633235433328256 7.0326093303156565 131.26574682416876 114.63439066968435"><linearGradient id="a" gradientUnits="userSpaceOnUse" x1="64" x2="64" y1="7.034" y2="120.789"><stop offset="0" stop-color="#4387fd"/><stop offset="1" stop-color="#4683ea"/></linearGradient><path d="M27.79 115.217L1.54 69.749a11.499 11.499 0 0 1 0-11.499l26.25-45.467a11.5 11.5 0 0 1 9.96-5.75h52.5a11.5 11.5 0 0 1 9.959 5.75l26.25 45.467a11.499 11.499 0 0 1 0 11.5l-26.25 45.467a11.5 11.5 0 0 1-9.959 5.749h-52.5a11.499 11.499 0 0 1-9.96-5.75z" fill="url(#a)"/><path clip-path="url(#b)" d="M119.229 86.48L80.625 47.874 64 43.425l-14.933 5.55L43.3 64l4.637 16.729 40.938 40.938 8.687-.386z" opacity=".07"/><g fill="#fff"><path d="M64 40.804c-12.81 0-23.195 10.385-23.195 23.196 0 12.81 10.385 23.195 23.195 23.195S87.194 76.81 87.194 64c0-12.811-10.385-23.196-23.194-23.196m0 40.795c-9.72 0-17.6-7.88-17.6-17.6S54.28 46.4 64 46.4 81.6 54.28 81.6 64 73.72 81.6 64 81.6"/><path d="M52.99 63.104v7.21a12.794 12.794 0 0 0 4.38 4.475V63.104zM61.675 57.026v19.411c.745.137 1.507.22 2.29.22.714 0 1.41-.075 2.093-.189V57.026zM70.766 66.1v8.562a12.786 12.786 0 0 0 4.382-4.7v-3.861zM80.691 78.287l-2.403 2.405a1.088 1.088 0 0 0 0 1.537l9.115 9.112a1.088 1.088 0 0 0 1.537 0l2.403-2.402a1.092 1.092 0 0 0 0-1.536l-9.116-9.116a1.09 1.09 0 0 0-1.536 0"/></g></svg>

data/lib/multiwoven/integrations/source/databricks/client.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module Databricks
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+      rescue Sequel::DatabaseConnectionError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                FROM system.information_schema.columns
+                WHERE table_schema = \'#{connection_config[:schema]}\' AND table_catalog = \'#{connection_config[:catalog]}\'
+                ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+
+        records = []
+        db.fetch(query.gsub("\n", "")) do |row|
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "DATABRICKS:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        db = create_connection(connection_config)
+
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(
+          "DATABRICKS:READ:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      private
+
+      def query(connection, query)
+        records = []
+        connection.fetch(query) do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+
+      def create_connection(connection_config)
+        Sequel.odbc(drvconnect: generate_drvconnect(connection_config))
+      end
+
+      def generate_drvconnect(connection_config)
+        "Driver=#{DATABRICKS_DRIVER_PATH};
+        Host=#{connection_config[:host]};
+        PORT=#{connection_config[:port]};
+        SSL=1;
+        HTTPPath=#{connection_config[:http_path]};
+        PWD=#{connection_config[:access_token]};
+        UID=token;
+        ThriftTransport=2;AuthMech=3;
+        AllowSelfSignedServerCert=1;
+        CAIssuedCertNamesMismatch=1"
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/databricks/config/meta.json

@@ -0,0 +1,16 @@
+{
+    "data": {
+      "name": "Databricks",
+      "title": "Databricks",
+      "connector_type": "source",
+      "category": "Data Warehouse",
+      "documentation_url": "https://docs.mutliwoven.com",
+      "github_issue_label": "source-databricks",
+      "icon": "icon.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "multiwoven"]
+    }
+  }
+  

data/lib/multiwoven/integrations/source/databricks/config/spec.json

@@ -0,0 +1,56 @@
+{
+    "documentation_url": "https://docs.multiwoven.com/integrations/sources/databricks",
+    "stream_type": "dynamic",
+    "connector_query_type": "raw_sql",
+    "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "Databricks",
+      "type": "object",
+      "required": ["host", "port", "http_path", "catalog", "schema"],
+      "properties": {
+        "host": {
+          "title": "Server Hostname",
+          "description": "Server host name for the Databricks Cluster. It is different from the SQL Endpoint Cluster.",
+          "type": "string",
+          "examples": ["abc-12345678-wxyz.cloud.databricks.com"],
+          "order": 0
+        },
+        "port": {
+          "title": "Port",
+          "description": "",
+          "type": "string",
+          "default": "443",
+          "order": 1
+        },
+        "access_token": {
+          "title": "Personal Access Token",
+          "description": "",
+          "type": "string",
+          "multiwoven_secret": true,
+          "order": 2
+        },
+        "http_path": {
+          "title": "HTTP Path",
+          "description": "",
+          "examples": ["sql/protocolvx/o/1234567489/0000-1111111-abcd90"],
+          "type": "string",
+          "order": 3
+        },
+        "catalog": {
+          "description": "The name of the catalog",
+          "default": "hive_metastore",
+          "type": "string",
+          "title": "Databricks catalog",
+          "order": 4
+        },
+        "schema": {
+          "description": "The default schema tables are written.",
+          "default": "default",
+          "type": "string",
+          "title": "Database schema",
+          "order": 5
+        }
+      }
+    }
+  }
+  

data/lib/multiwoven/integrations/source/databricks/icon.svg

@@ -0,0 +1,19 @@
+<svg version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 40.1 42" style="enable-background:new 0 0 40.1 42;" xml:space="preserve">
+ <style type="text/css">
+  .st0{fill:#FF3621;}
+ </style>
+ <metadata>
+  <sfw xmlns="ns_sfw;">
+   <slices>
+   </slices>
+   <sliceSourceBounds bottomLeftOrigin="true" height="42" width="40.1" x="-69.1" y="-10.5">
+   </sliceSourceBounds>
+  </sfw>
+ </metadata>
+ <g>
+  <path class="st0" d="M40.1,31.1v-7.4l-0.8-0.5L20.1,33.7l-18.2-10l0-4.3l18.2,9.9l20.1-10.9v-7.3l-0.8-0.5L20.1,21.2L2.6,11.6
+		L20.1,2l14.1,7.7l1.1-0.6V8.3L20.1,0L0,10.9V12L20.1,23l18.2-10v4.4l-18.2,10L0.8,16.8L0,17.3v7.4l20.1,10.9l18.2-9.9v4.3l-18.2,10
+		L0.8,29.5L0,30v1.1L20.1,42L40.1,31.1z">
+  </path>
+ </g>
+</svg>

data/lib/multiwoven/integrations/source/postgresql/client.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Multiwoven::Integrations::Source
+  module Postgresql
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue PG::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+        records = db.exec(query) do |result|
+          result.map do |row|
+            row
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "POSTGRESQL:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      ensure
+        db&.close
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
+        db = create_connection(connection_config)
+
+        query(db, query)
+      rescue StandardError => e
+        handle_exception(
+          "POSTGRESQL:READ:EXCEPTION",
+          "error",
+          e
+        )
+      ensure
+        db&.close
+      end
+
+      private
+
+      def query(connection, query)
+        connection.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+          end
+        end
+      end
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        PG.connect(
+          host: connection_config[:host],
+          dbname: connection_config[:database],
+          user: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          port: connection_config[:port]
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/postgresql/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Postgresql",
+    "title": "PostgreSQL",
+    "connector_type": "source",
+    "category": "Data Warehouse",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-postgresql",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}