multiwoven-integrations 0.1.0

data/lib/multiwoven/integrations/destination/klaviyo/config/catalog.json

@@ -0,0 +1,124 @@
+{
+    "streams": [
+      {
+        "name": "profile",
+        "action": "create",
+        "url": "https://a.klaviyo.com/api/profiles",
+        "method": "POST",
+        "json_schema": {
+          "type": "object",
+          "additionalProperties": true,
+          "properties": {
+            "type": {
+              "type": [
+                "null",
+                "string"
+              ]
+            },
+            "id": {
+              "type": "string"
+            },
+            "updated": {
+              "type": [
+                "null",
+                "string"
+              ],
+              "format": "date-time"
+            },
+            "attributes": {
+              "type": [
+                "null",
+                "object"
+              ],
+              "additionalProperties": true,
+              "properties": {
+                "email": {
+                  "type": [
+                    "null",
+                    "string"
+                  ]
+                },
+                "phone_number": {
+                  "type": [
+                    "null",
+                    "string"
+                  ]
+                },
+                "first_name": {
+                  "type": [
+                    "null",
+                    "string"
+                  ]
+                },
+                "last_name": {
+                  "type": [
+                    "null",
+                    "string"
+                  ]
+                },
+                "properties": {
+                  "type": [
+                    "null",
+                    "object"
+                  ],
+                  "additionalProperties": true
+                },
+                "organization": {
+                  "type": [
+                    "null",
+                    "string"
+                  ]
+                },
+                "title": {
+                  "type": [
+                    "null",
+                    "string"
+                  ]
+                },
+                "last_event_date": {
+                  "type": [
+                    "null",
+                    "string"
+                  ],
+                  "format": "date-time"
+                }
+              }
+            },
+            "links": {
+              "type": [
+                "null",
+                "object"
+              ]
+            },
+            "relationships": {
+              "type": [
+                "null",
+                "object"
+              ]
+            },
+            "segments": {
+              "type": [
+                "null",
+                "object"
+              ]
+            }
+          }
+        },
+        "supported_sync_modes": [
+          "full_refresh",
+          "incremental"
+        ],
+        "source_defined_cursor": true,
+        "default_cursor_field": [
+          "updated"
+        ],
+        "source_defined_primary_key": [
+          [
+            "id",
+            "email"
+          ]
+        ]
+        
+      }
+    ]
+  }

data/lib/multiwoven/integrations/destination/klaviyo/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data":
+    {
+      "name": "Klaviyo",
+      "connector_type": "destination",
+      "connector_subtype": "API",
+      "documentation_url": "https://docs.mutliwoven.com",
+      "github_issue_label": "destination-klaviyo",
+      "icon": "klaviyo.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "multiwoven"]
+    }
+}

data/lib/multiwoven/integrations/destination/klaviyo/config/spec.json

@@ -0,0 +1,22 @@
+{
+    "documentation_url": "https://docs.multiwoven.com/integrations/destination/klaviyo",
+    "stream_type": "static",
+    "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "Klaviyo Destination Spec",
+      "type": "object",
+      "required": ["public_api_key", "private_api_key"],
+      "properties": {
+        "public_api_keyhost": {
+          "type": "string",
+          "title": "Public API Key",
+          "order": 0
+        },
+        "private_api_key": {
+          "type": "string",
+          "title": "Private API Key",
+          "order": 1
+        }
+      }
+    }
+  }

data/lib/multiwoven/integrations/protocol/protocol.json

@@ -0,0 +1,189 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Mutliwoven Protocol",
+  "type": "object",
+  "description": "Mutliwoven protocol schema",
+  "version": "1.0.0",
+  "definitions": {
+    "SyncMode": {
+      "type": "string",
+      "enum": [
+        "full_refresh",
+        "incremental"
+      ]
+    },
+    "SyncStatus": {
+      "type": "string",
+      "enum": [
+        "STARTED",
+        "RUNNING",
+        "COMPLETE",
+        "INCOMPLETE"
+      ]
+    },
+    "DestinationSyncMode": {
+      "type": "string",
+      "enum": [
+        "append",
+        "overwrite",
+        "append_dedup"
+      ]
+    },
+    "ProtocolModel": {
+      "type": "object",
+      "additionalProperties": true
+    },
+    "ConnectionStatus": {
+      "type": "object",
+      "properties": {
+        "status": {
+          "$ref": "#/definitions/Types/String.enum(SUCCEEDED,FAILED)"
+        },
+        "message": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": true
+    },
+    "ConnectorSpecification": {
+      "type": "object",
+      "properties": {
+        "documentation_url": {
+          "type": "string"
+        },
+        "changelog_url": {
+          "type": "string"
+        },
+        "connection_specification": {
+          "type": "object"
+        },
+        "supports_normalization": {
+          "type": "boolean",
+          "default": false
+        },
+        "supports_dbt": {
+          "type": "boolean",
+          "default": false
+        },
+        "supported_destination_sync_modes": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/DestinationSyncMode"
+          }
+        }
+      },
+      "additionalProperties": true
+    },
+    "LogMessage": {
+      "type": "object",
+      "properties": {
+        "level": {
+          "$ref": "#/definitions/Types/String.enum(FATAL,ERROR,WARN,INFO,DEBUG,TRACE)"
+        },
+        "message": {
+          "type": "string"
+        },
+        "stack_trace": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": true
+    },
+    "RecordMessage": {
+      "type": "object",
+      "properties": {
+        "stream": {
+          "type": "string"
+        },
+        "data": {
+          "type": "object"
+        },
+        "emitted_at": {
+          "type": "integer"
+        }
+      },
+      "additionalProperties": true
+    },
+    "Stream": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "json_schema": {
+          "type": "object"
+        },
+        "supported_sync_modes": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/SyncMode"
+          }
+        },
+        "source_defined_cursor": {
+          "type": "boolean"
+        },
+        "default_cursor_field": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "source_defined_primary_key": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "namespace": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": true
+    },
+    "Catalog": {
+      "type": "object",
+      "properties": {
+        "streams": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Stream"
+          }
+        }
+      },
+      "additionalProperties": true
+    },
+    "SyncConfig": {
+      "type": "object",
+      "properties": {
+        "stream": {
+          "$ref": "#/definitions/Stream"
+        },
+        "sync_mode": {
+          "$ref": "#/definitions/SyncMode"
+        },
+        "cursor_field": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "destination_sync_mode": {
+          "$ref": "#/definitions/DestinationSyncMode"
+        },
+        "primary_key": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        }
+      },
+      "additionalProperties": true
+    }
+  }
+}

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations::Protocol
+    module Types
+      include Dry.Types()
+    end
+
+    SyncMode = Types::String.enum("full_refresh", "incremental")
+    SyncStatus = Types::String.enum("started", "running", "complete", "incomplete")
+    DestinationSyncMode = Types::String.enum("insert", "upsert")
+    ConnectorType = Types::String.enum("source", "destination")
+    ModelQueryType = Types::String.enum("raw_sql", "dbt")
+    ConnectionStatusType = Types::String.enum("succeeded", "failed")
+    StreamType = Types::String.enum("static", "dynamic")
+    StreamAction = Types::String.enum("fetch", "create", "update", "delete")
+    MultiwovenMessageType = Types::String.enum(
+      "record", "log", "connector_spec",
+      "connection_status", "catalog", "control",
+      "tracking"
+    )
+    ControlMessageType = Types::String.enum(
+      "rate_limit", "connection_config"
+    )
+    LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
+
+    class ProtocolModel < Dry::Struct
+      extend Multiwoven::Integrations::Core::Utils
+      class << self
+        def from_json(json_string)
+          data = JSON.parse(json_string)
+          new(keys_to_symbols(data))
+        end
+      end
+    end
+
+    class ConnectionStatus < ProtocolModel
+      attribute :status, ConnectionStatusType
+      attribute? :message, Types::String.optional
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["connection_status"],
+          connection_status: self
+        )
+      end
+    end
+
+    class ConnectorSpecification < ProtocolModel
+      attribute? :documentation_url, Types::String.optional
+      attribute? :changelog_url, Types::String.optional
+      attribute :connection_specification, Types::Hash
+      attribute :supports_normalization, Types::Bool.default(false)
+      attribute :supports_dbt, Types::Bool.default(false)
+      attribute :stream_type, StreamType
+      attribute? :supported_destination_sync_modes, Types::Array.of(DestinationSyncMode).optional
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["connector_spec"],
+          connector_spec: self
+        )
+      end
+    end
+
+    class Connector < ProtocolModel
+      attribute :name, Types::String
+      attribute :type, ConnectorType
+      attribute :connection_specification, Types::Hash
+    end
+
+    class LogMessage < ProtocolModel
+      attribute :level, LogLevel
+      attribute :message, Types::String
+      attribute? :name, Types::String.optional
+      attribute? :stack_trace, Types::String.optional
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["log"],
+          log: self
+        )
+      end
+    end
+
+    class Model < ProtocolModel
+      attribute? :name, Types::String.optional
+      attribute :query, Types::String
+      attribute :query_type, ModelQueryType
+      attribute :primary_key, Types::String
+    end
+
+    class RecordMessage < ProtocolModel
+      attribute :data, Types::Hash
+      attribute :emitted_at, Types::Integer
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["record"],
+          record: self
+        )
+      end
+    end
+
+    class Stream < ProtocolModel
+      # Common
+      attribute :name, Types::String
+      attribute? :action, StreamAction
+      attribute :json_schema, Types::Hash
+      attribute? :supported_sync_modes, Types::Array.of(SyncMode).optional
+
+      attribute? :source_defined_cursor, Types::Bool.optional
+      attribute? :default_cursor_field, Types::Array.of(Types::String).optional
+      attribute? :source_defined_primary_key, Types::Array.of(Types::Array.of(Types::String)).optional
+      attribute? :namespace, Types::String.optional
+      # Applicable for API streams
+      attribute? :url, Types::String.optional
+      attribute? :request_method, Types::String.optional
+    end
+
+    class Catalog < ProtocolModel
+      attribute :streams, Types::Array.of(Stream)
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["catalog"],
+          catalog: self
+        )
+      end
+    end
+
+    class SyncConfig < ProtocolModel
+      attribute :source, Connector
+      attribute :destination, Connector
+      attribute :model, Model
+      attribute :stream, Stream
+      attribute :sync_mode, SyncMode
+      attribute? :cursor_field, Types::String.optional
+      attribute :destination_sync_mode, DestinationSyncMode
+    end
+
+    class ControlMessage < ProtocolModel
+      attribute :type, ControlMessageType
+      attribute :emitted_at, Types::Integer
+      attribute? :meta, Types::Hash
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["control"],
+          control: self
+        )
+      end
+    end
+
+    class TrackingMessage < ProtocolModel
+      attribute :success, Types::Integer.default(0)
+      attribute :failed, Types::Integer.default(0)
+      attribute? :meta, Types::Hash
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["tracking"],
+          tracking: self
+        )
+      end
+    end
+
+    class MultiwovenMessage < ProtocolModel
+      attribute :type, MultiwovenMessageType
+      attribute? :log, LogMessage.optional
+      attribute? :connection_status, ConnectionStatus.optional
+      attribute? :connector_spec, ConnectorSpecification.optional
+      attribute? :catalog, Catalog.optional
+      attribute? :record, RecordMessage.optional
+      attribute? :control, ControlMessage.optional
+      attribute? :tracking, TrackingMessage.optional
+    end
+  end
+end

data/lib/multiwoven/integrations/rollout.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    VERSION = "0.1.0"
+
+    ENABLED_SOURCES = %w[
+      Snowflake
+      Redshift
+      Bigquery
+    ].freeze
+
+    ENABLED_DESTINATIONS = %w[
+      Klaviyo
+    ].freeze
+  end
+end

data/lib/multiwoven/integrations/service.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    class Service
+      class << self
+        def initialize
+          yield(config) if block_given?
+        end
+
+        def connectors
+          {
+            source: build_connectors(
+              ENABLED_SOURCES, "Source"
+            ),
+            destination: build_connectors(
+              ENABLED_DESTINATIONS, "Destination"
+            )
+          }
+        end
+
+        def connector_class(connector_type, connector_name)
+          Object.const_get(
+            "Multiwoven::Integrations::#{connector_type}::#{connector_name}::Client"
+          )
+        end
+
+        def logger
+          config.logger || default_logger
+        end
+
+        def config
+          @config ||= Config.new
+        end
+
+        private
+
+        def build_connectors(enabled_connectors, type)
+          enabled_connectors.map do |connector|
+            client = connector_class(
+              type, connector
+            ).new
+            client.meta_data["data"].to_h.merge(
+              {
+                connector_spec: client.connector_spec.to_h
+              }
+            )
+          end
+        end
+
+        def default_logger
+          @default_logger ||= Logger.new($stdout)
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/bigquery/client.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "google/cloud/bigquery"
+
+module Multiwoven::Integrations::Source
+  module Bigquery
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        bigquery = create_connection(connection_config)
+        bigquery.datasets
+        ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(connection_config)
+        bigquery = create_connection(connection_config)
+        target_dataset_id = connection_config["dataset_id"]
+        records = bigquery.datasets.flat_map do |dataset|
+          next unless dataset.dataset_id == target_dataset_id
+
+          dataset.tables.flat_map do |table|
+            table.schema.fields.map do |field|
+              {
+                table_name: table.table_id,
+                column_name: field.name,
+                data_type: field.type,
+                is_nullable: field.mode == "NULLABLE"
+              }
+            end
+          end
+        end
+        catalog = Catalog.new(streams: create_streams(records))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "BIGQUERY:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        query = sync_config.model.query
+        bigquery = create_connection(connection_config)
+        records = []
+        results = bigquery.query query
+        results.each do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i)
+        end
+
+        records
+      rescue StandardError => e
+        handle_exception(
+          "BIGQUERY:READ:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def create_connection(connection_config)
+        Google::Cloud::Bigquery.new(
+          project: connection_config["project_id"],
+          credentials: connection_config["credentials_json"]
+        )
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(records)
+        records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
+          {
+            tablename: table_name,
+            columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
+          }
+        end
+      end
+    end
+  end
+end