multiwoven-integrations 0.1.0

data/lib/multiwoven/integrations/source/bigquery/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data":
+    {
+      "name": "BigQuery",
+      "connector_type": "source",
+      "connector_subtype": "database",
+      "documentation_url": "https://docs.mutliwoven.com",
+      "github_issue_label": "source-bigquery",
+      "icon": "bigquery.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "multiwoven"]
+    }
+}

data/lib/multiwoven/integrations/source/bigquery/config/spec.json

@@ -0,0 +1,28 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/bigquery",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "BigQuery Source Spec",
+    "type": "object",
+    "required": ["project_id", "credentials_json"],
+    "properties": {
+      "project_id": {
+        "type": "string",
+        "description": "The GCP project ID for the project containing the target BigQuery dataset.",
+        "title": "Project ID"
+      },
+      "dataset_id": {
+        "type": "string",
+        "description": "The dataset ID to search for tables and views. If you are only loading data from one dataset, setting this option could result in much faster schema discovery.",
+        "title": "Default Dataset ID"
+      },
+      "credentials_json": {
+        "type": "string",
+        "description": "The contents of your Service Account Key JSON file. See the <a href=\"https://docs.multiwoven.com/integrations/sources/bigquery#setup-the-bigquery-source-in-multiwoven\">docs</a> for more information on how to obtain this key.",
+        "title": "Credentials JSON",
+        "multiwoven_secret": true
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/redshift/client.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Multiwoven::Integrations::Source
+  module Redshift
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        create_connection(connection_config)
+        ConnectionStatus.new(
+          status: ConnectionStatusType["succeeded"]
+        ).to_multiwoven_message
+      rescue PG::Error => e
+        ConnectionStatus.new(
+          status: ConnectionStatusType["failed"], message: e.message
+        ).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                 FROM information_schema.columns
+                 WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
+                 ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+        records = db.exec(query) do |result|
+          result.map do |row|
+            row
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "REDSHIFT:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      ensure
+        db&.close
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        query = sync_config.model.query
+        db = create_connection(connection_config)
+        records = db.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+          end
+        end
+        records
+      rescue StandardError => e
+        handle_exception(
+          "REDSHIFT:READ:EXCEPTION",
+          "error",
+          e
+        )
+      ensure
+        db&.close
+      end
+
+      private
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
+
+        PG.connect(
+          host: connection_config[:host],
+          dbname: connection_config[:database],
+          user: connection_config[:credentials][:username],
+          password: connection_config[:credentials][:password],
+          port: connection_config[:port]
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map do |column|
+              {
+                column_name: column["column_name"],
+                type: column["data_type"],
+                optional: column["is_nullable"] == "YES"
+              }
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/redshift/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data":
+    {
+      "name": "Redshift",
+      "connector_type": "source",
+      "connector_subtype": "database",
+      "documentation_url": "https://docs.mutliwoven.com",
+      "github_issue_label": "source-redshift",
+      "icon": "redshift.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "multiwoven"]
+    }
+}

data/lib/multiwoven/integrations/source/redshift/config/spec.json

@@ -0,0 +1,74 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/redshift",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Redshift Source Spec",
+    "type": "object",
+    "required": ["host", "port", "database", "schema"],
+    "properties": {
+      "credentials": {
+        "title": "Authorization Method",
+        "type": "object",
+        "oneOf": [
+          {
+            "title": "Username and Password",
+            "type": "object",
+            "required": ["username", "password", "auth_type"],
+            "order": 1,
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "username/password",
+                "order": 0
+              },
+              "username": {
+                "description": "The username for Redshift database access.",
+                "examples": ["REDSHIFT_USER"],
+                "type": "string",
+                "title": "Username",
+                "order": 1
+              },
+              "password": {
+                "description": "The password for the Redshift user.",
+                "type": "string",
+                "multiwoven_secret": true,
+                "title": "Password",
+                "order": 2
+              }
+            }
+          }
+        ],
+        "order": 0
+      },
+      "host": {
+        "description": "The host endpoint of the Redshift cluster.",
+        "examples": ["example-redshift-cluster.abcdefg.us-west-2.redshift.amazonaws.com"],
+        "type": "string",
+        "title": "Host",
+        "order": 1
+      },
+      "port": {
+        "description": "The port on which Redshift is running (default is 5439).",
+        "examples": ["5439"],
+        "type": "string",
+        "title": "Port",
+        "order": 2
+      },
+      "database": {
+        "description": "The specific Redshift database to connect to.",
+        "examples": ["REDSHIFT_DB"],
+        "type": "string",
+        "title": "Database",
+        "order": 3
+      },
+      "schema": {
+        "description": "The schema within the Redshift database.",
+        "examples": ["REDSHIFT_SCHEMA"],
+        "type": "string",
+        "title": "Schema",
+        "order": 4
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/snowflake/client.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module Snowflake
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        create_connection(connection_config)
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+      rescue Sequel::DatabaseConnectionError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        query = "SELECT table_name, column_name, data_type, is_nullable
+                FROM information_schema.columns
+                WHERE table_schema = \'#{connection_config[:schema]}\' AND table_catalog = \'#{connection_config[:database]}\'
+                ORDER BY table_name, ordinal_position;"
+
+        db = create_connection(connection_config)
+
+        records = []
+        db.fetch(query.gsub("\n", "")) do |row|
+          records << row
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "SNOWFLAKE:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        query = sync_config.model.query
+        db = create_connection(connection_config)
+
+        records = []
+        db.fetch(query) do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+
+        records
+      rescue StandardError => e
+        handle_exception(
+          "SNOWFLAKE:READ:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      private
+
+      def create_connection(connection_config)
+        raise "Unsupported Auth type" if connection_config[:credentials][:auth_type] != "username/password"
+
+        Sequel.odbc(drvconnect: generate_drvconnect(connection_config))
+      end
+
+      def generate_drvconnect(connection_config)
+        c = connection_config[:credentials]
+        "driver=#{SNOWFLAKE_DRIVER_PATH};server=#{connection_config[:host]};uid=#{c[:username]};pwd=#{c[:password]};schema=#{connection_config[:schema]};database=#{connection_config[:database]};warehouse=#{connection_config[:warehouse]};"
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/snowflake/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data":
+    {
+      "name": "Snowflake",
+      "connector_type": "source",
+      "connector_subtype": "database",
+      "documentation_url": "https://docs.mutliwoven.com",
+      "github_issue_label": "source-snowflake",
+      "icon": "snowflake.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "multiwoven"]
+    }
+}

data/lib/multiwoven/integrations/source/snowflake/config/spec.json

@@ -0,0 +1,87 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/sources/snowflake",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Snowflake Source Spec",
+    "type": "object",
+    "required": ["host", "role", "warehouse", "database"],
+    "properties": {
+      "credentials": {
+        "title": "Authorization Method",
+        "type": "object",
+        "oneOf": [  
+          {
+            "title": "Username and Password",
+            "type": "object",
+            "required": ["username", "password", "auth_type"],
+            "order": 1,
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "username/password",
+                "order": 0
+              },
+              "username": {
+                "description": "The username you created to allow multiwoven to access the database.",
+                "examples": ["MULTIWOVEN_USER"],
+                "type": "string",
+                "title": "Username",
+                "order": 1
+              },
+              "password": {
+                "description": "The password associated with the username.",
+                "type": "string",
+                "multiwoven_secret": true,
+                "title": "Password",
+                "order": 2
+              }
+            }
+          }
+        ],
+        "order": 0
+      },
+      "host": {
+        "description": "The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com).",
+        "examples": ["accountname.us-east-2.aws.snowflakecomputing.com"],
+        "type": "string",
+        "title": "Account Name",
+        "order": 1
+      },
+      "role": {
+        "description": "The role you created for multiwoven to access Snowflake.",
+        "examples": ["MULTIWOVEN_ROLE"],
+        "type": "string",
+        "title": "Role",
+        "order": 2
+      },
+      "warehouse": {
+        "description": "The warehouse you created for multiwoven to access data.",
+        "examples": ["MULTIWOVEN_WAREHOUSE"],
+        "type": "string",
+        "title": "Warehouse",
+        "order": 3
+      },
+      "database": {
+        "description": "The database you created for multiwoven to access data.",
+        "examples": ["MULTIWOVEN_DATABASE"],
+        "type": "string",
+        "title": "Database",
+        "order": 4
+      },
+      "schema": {
+        "description": "The source Snowflake schema tables. Leave empty to access tables from multiple schemas.",
+        "examples": ["MULTIWOVEN_SCHEMA"],
+        "type": "string",
+        "title": "Schema",
+        "order": 5
+      },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).",
+        "title": "JDBC URL Params",
+        "type": "string",
+        "order": 6
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "json"
+require "dry-struct"
+require "dry-schema"
+require "dry-types"
+require "odbc"
+require "sequel"
+require "byebug"
+require "net/http"
+require "uri"
+require "active_support/core_ext/hash/indifferent_access"
+
+# Service
+require_relative "integrations/config"
+require_relative "integrations/rollout"
+require_relative "integrations/service"
+
+# Core
+require_relative "integrations/core/constants"
+require_relative "integrations/core/utils"
+require_relative "integrations/protocol/protocol"
+require_relative "integrations/core/base_connector"
+require_relative "integrations/core/source_connector"
+require_relative "integrations/core/destination_connector"
+require_relative "integrations/core/http_client"
+
+# Source
+require_relative "integrations/source/snowflake/client"
+require_relative "integrations/source/redshift/client"
+require_relative "integrations/source/bigquery/client"
+
+# Destination
+require_relative "integrations/destination/klaviyo/client"
+
+module Multiwoven
+  module Integrations
+    class Error < StandardError; end
+    # Your code goes here...
+  end
+end

data/multiwoven-integrations.gemspec

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative "lib/multiwoven/integrations/rollout"
+
+Gem::Specification.new do |spec|
+  spec.name = "multiwoven-integrations"
+  spec.version = Multiwoven::Integrations::VERSION
+  spec.authors = ["Subin T P"]
+  spec.email = ["subin@multiwoven.com"]
+
+  spec.summary = "Open source data activation"
+  spec.description = "Wrapper of connectors"
+  spec.homepage = "https://www.multiwoven.com/"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  # spec.metadata["allowed_push_host"] = nil
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://www.multiwoven.com/"
+  spec.metadata["changelog_uri"] = "https://www.multiwoven.com/"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "activesupport"
+  spec.add_runtime_dependency "dry-schema"
+  spec.add_runtime_dependency "dry-struct"
+  spec.add_runtime_dependency "dry-types"
+  spec.add_runtime_dependency "google-cloud-bigquery"
+  spec.add_runtime_dependency "pg"
+  spec.add_runtime_dependency "rake"
+  spec.add_runtime_dependency "ruby-odbc", "~> 0.999992"
+  spec.add_runtime_dependency "sequel"
+
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "simplecov_json_formatter"
+  spec.add_development_dependency "webmock"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/sig/multiwoven/integrations.rbs

@@ -0,0 +1,6 @@
+module Multiwoven
+  module Integrations
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end