etl-integrations 0.1.81

data/lib/multiwoven/integrations/destination/tally/client.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "net/http"
+require "uri"
+require "json"
+require 'builder' # Add builder gem for XML generation
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module Tally
+        include Multiwoven::Integrations::Core
+
+        API_VERSION = "1.0"
+
+        class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
+
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            initialize_client(connection_config)
+            authenticate_client
+            success_status
+          rescue StandardError => e
+            failure_status(e)
+          end
+
+          def discover(_connection_config = nil)
+            catalog = build_catalog(load_catalog)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception("TALLY:DISCOVER:EXCEPTION", "error", e)
+          end
+
+          def write(sync_config, records, action = "create")
+            @action = sync_config.stream.action || action
+            initialize_client(sync_config.destination.connection_specification)
+            process_records(records, sync_config.stream)
+          rescue StandardError => e
+            handle_exception("TALLY:WRITE:EXCEPTION", "error", e)
+          end
+
+          private
+
+          def initialize_client(config)
+            config = config.with_indifferent_access
+            @tally_url = config[:tally_url]
+            @company_name = config[:company_name]
+          end
+
+          def process_records(records, stream)
+            write_success = 0
+            write_failure = 0
+            properties = stream.json_schema[:properties]
+            records.each do |record_object|
+              record = extract_data(record_object, properties)
+              xml_payload = convert_to_xml(record) # Convert record to XML
+              process_record(xml_payload)
+              write_success += 1
+            rescue StandardError => e
+              handle_exception("TALLY:WRITE:EXCEPTION", "error", e)
+              write_failure += 1
+            end
+            tracking_message(write_success, write_failure)
+          end
+
+          def process_record(xml_payload)
+            send_data_to_tally(xml_payload)
+          end
+
+          def send_data_to_tally(xml_payload = {})
+            uri = URI.parse("#{@tally_url}/#{@company_name}")
+            request = Net::HTTP::Post.new(uri)
+            request.content_type = 'application/xml' # Tally uses XML format
+            request.body = xml_payload
+            response = Net::HTTP.start(uri.hostname, uri.port) { |http| http.request(request) }
+
+            raise StandardError, response.body unless response.is_a?(Net::HTTPSuccess)
+          end
+
+          def convert_to_xml(record)
+            # Using Builder gem to create XML structure
+            xml = Builder::XmlMarkup.new
+            xml.instruct! :xml, version: '1.0', encoding: 'UTF-8'
+            xml.ENVELOPE do
+              xml.HEADER do
+                xml.TALLYREQUEST 'Import Data'
+              end
+              xml.BODY do
+                xml.IMPORTDATA do
+                  xml.REQUESTDESC do
+                    xml.REPORTNAME 'Vouchers'
+                    xml.STATICVARIABLES do
+                      xml.SVCURRENTCOMPANY @company_name
+                    end
+                  end
+                  xml.REQUESTDATA do
+                    xml.TALLYMESSAGE('xmlns:UDF' => 'TallyUDF') do
+                      xml.VOUCHER do
+                        xml.DATE '20240915' # Static or dynamic date
+                        xml.VOUCHERTYPENAME 'Sales'
+                        xml.NARRATION 'Test invoice from ETL Tool'
+                        xml.VOUCHERNUMBER record[:VOUCHERNUMBER] || 'INV-1002'
+                        xml.PARTYNAME record[:PARTYNAME] || 'XYZ Ltd'
+                        xml.AMOUNT record[:AMOUNT] || 10000
+                        xml.LEDGERENTRIES__LIST do
+                          xml.LEDGERNAME 'Sales Account'
+                          xml.AMOUNT record[:AMOUNT] || 10000
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            end
+            xml.target! # Returns the generated XML string
+          end
+
+          def authenticate_client
+            # Tally may not need specific authentication steps like OAuth;
+            # skipping authentication or implementing basic connection validation.
+            true
+          end
+
+          def success_status
+            ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+          end
+
+          def failure_status(error)
+            ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
+          end
+
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
+          end
+
+          def tracking_message(success, failure)
+            Multiwoven::Integrations::Protocol::TrackingMessage.new(
+              success: success, failed: failure
+            ).to_multiwoven_message
+          end
+
+          def log_debug(message)
+            Multiwoven::Integrations::Service.logger.debug(message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/tally/config/catalog.json

@@ -0,0 +1,62 @@
+{
+  "request_rate_limit": 100000,
+  "request_rate_limit_unit": "day",
+  "request_rate_concurrency": 10,
+  "streams": [
+    {
+      "name": "Voucher",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "properties": {
+          "VoucherNumber": {
+            "type": "string"
+          },
+          "VoucherType": {
+            "type": "string",
+            "enum": [
+              "Payment",
+              "Receipt",
+              "Journal",
+              "Contra",
+              "Sales",
+              "Purchase",
+              "Credit Note",
+              "Debit Note"
+            ]
+          },
+          "Date": {
+            "type": "string",
+            "format": "date"
+          },
+          "LedgerName": {
+            "type": "string"
+          },
+          "Amount": {
+            "type": "number"
+          },
+          "Narration": {
+            "type": ["string", "null"]
+          },
+          "PartyLedgerName": {
+            "type": ["string", "null"]
+          },
+          "InvoiceNumber": {
+            "type": ["string", "null"]
+          },
+          "CostCenter": {
+            "type": ["string", "null"]
+          },
+          "CostCategory": {
+            "type": ["string", "null"]
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"],
+      "source_defined_cursor": true,
+      "default_cursor_field": ["Date"],
+      "source_defined_primary_key": [["VoucherNumber"]]
+    }
+  ]
+}

data/lib/multiwoven/integrations/destination/tally/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Tally",
+    "title": "Tally",
+    "connector_type": "destination",
+    "category": "Accounting",
+    "documentation_url": "https://docs.multiwoven.com/tally",
+    "github_issue_label": "destination-tally",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven", "accounting", "tally"]
+  }
+}

data/lib/multiwoven/integrations/destination/tally/config/spec.json

@@ -0,0 +1,45 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/destination/tally",
+  "stream_type": "static",
+  "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "Tally",
+      "type": "object",
+      "required": [
+          "server_url",
+          "company_name"
+      ],
+      "properties": {
+          "server_url": {
+              "type": "string",
+              "title": "Server URL",
+              "order": 0,
+              "description": "The URL or IP address of the Tally server."
+          },
+          "company_name": {
+              "type": "string",
+              "title": "Company Name",
+              "order": 1,
+              "description": "The company name in Tally to connect with."
+          },
+          "license_key": {
+              "type": "string",
+              "title": "Tally License Key",
+              "order": 2,
+              "description": "Tally license key (if applicable)."
+          },
+          "username": {
+              "type": "string",
+              "title": "Tally Username",
+              "order": 3,
+              "description": "Username for authentication (if required)."
+          },
+          "password": {
+              "type": "string",
+              "title": "Tally Password",
+              "order": 4,
+              "description": "Password for authentication (if required)."
+          }
+      }
+  }
+}

data/lib/multiwoven/integrations/destination/tally/icon.svg

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M3.661 2.67105C3.71317 2.39988 3.97931 2.18005 4.25546 2.18005H6.25546C6.5316 2.18005 6.71317 2.39988 6.661 2.67105L5.33853 9.54499C5.28636 9.81616 5.02021 10.036 4.74406 10.036H2.74406C2.46792 10.036 2.28636 9.81616 2.33853 9.54499L3.661 2.67105Z" fill="#000000"/>
+<path d="M8.661 2.67105C8.71317 2.39988 8.97931 2.18005 9.25546 2.18005H11.2555C11.5316 2.18005 11.7132 2.39988 11.661 2.67105L8.07144 21.3289C8.01927 21.6001 7.75312 21.8199 7.47698 21.8199H5.47698C5.20083 21.8199 5.01927 21.6001 5.07144 21.3289L8.661 2.67105Z" fill="#000000"/>
+<path d="M13.661 2.67105C13.7132 2.39988 13.9793 2.18005 14.2555 2.18005H16.2555C16.5316 2.18005 16.7132 2.39988 16.661 2.67105L13.0714 21.3289C13.0193 21.6001 12.7531 21.8199 12.477 21.8199H10.477C10.2008 21.8199 10.0193 21.6001 10.0714 21.3289L13.661 2.67105Z" fill="#000000"/>
+<path d="M18.661 2.67105C18.7132 2.39988 18.9793 2.18005 19.2555 2.18005H21.2555C21.5316 2.18005 21.7132 2.39988 21.661 2.67105L20.3385 9.54499C20.2864 9.81616 20.0202 10.036 19.7441 10.036H17.7441C17.4679 10.036 17.2864 9.81616 17.3385 9.54499L18.661 2.67105Z" fill="#000000"/>
+</svg>

data/lib/multiwoven/integrations/protocol/protocol.json

@@ -0,0 +1,189 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Mutliwoven Protocol",
+  "type": "object",
+  "description": "Mutliwoven protocol schema",
+  "version": "1.0.0",
+  "definitions": {
+    "SyncMode": {
+      "type": "string",
+      "enum": [
+        "full_refresh",
+        "incremental"
+      ]
+    },
+    "SyncStatus": {
+      "type": "string",
+      "enum": [
+        "STARTED",
+        "RUNNING",
+        "COMPLETE",
+        "INCOMPLETE"
+      ]
+    },
+    "DestinationSyncMode": {
+      "type": "string",
+      "enum": [
+        "append",
+        "overwrite",
+        "append_dedup"
+      ]
+    },
+    "ProtocolModel": {
+      "type": "object",
+      "additionalProperties": true
+    },
+    "ConnectionStatus": {
+      "type": "object",
+      "properties": {
+        "status": {
+          "$ref": "#/definitions/Types/String.enum(SUCCEEDED,FAILED)"
+        },
+        "message": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": true
+    },
+    "ConnectorSpecification": {
+      "type": "object",
+      "properties": {
+        "documentation_url": {
+          "type": "string"
+        },
+        "changelog_url": {
+          "type": "string"
+        },
+        "connection_specification": {
+          "type": "object"
+        },
+        "supports_normalization": {
+          "type": "boolean",
+          "default": false
+        },
+        "supports_dbt": {
+          "type": "boolean",
+          "default": false
+        },
+        "supported_destination_sync_modes": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/DestinationSyncMode"
+          }
+        }
+      },
+      "additionalProperties": true
+    },
+    "LogMessage": {
+      "type": "object",
+      "properties": {
+        "level": {
+          "$ref": "#/definitions/Types/String.enum(FATAL,ERROR,WARN,INFO,DEBUG,TRACE)"
+        },
+        "message": {
+          "type": "string"
+        },
+        "stack_trace": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": true
+    },
+    "RecordMessage": {
+      "type": "object",
+      "properties": {
+        "stream": {
+          "type": "string"
+        },
+        "data": {
+          "type": "object"
+        },
+        "emitted_at": {
+          "type": "integer"
+        }
+      },
+      "additionalProperties": true
+    },
+    "Stream": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "json_schema": {
+          "type": "object"
+        },
+        "supported_sync_modes": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/SyncMode"
+          }
+        },
+        "source_defined_cursor": {
+          "type": "boolean"
+        },
+        "default_cursor_field": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "source_defined_primary_key": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "namespace": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": true
+    },
+    "Catalog": {
+      "type": "object",
+      "properties": {
+        "streams": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Stream"
+          }
+        }
+      },
+      "additionalProperties": true
+    },
+    "SyncConfig": {
+      "type": "object",
+      "properties": {
+        "stream": {
+          "$ref": "#/definitions/Stream"
+        },
+        "sync_mode": {
+          "$ref": "#/definitions/SyncMode"
+        },
+        "cursor_field": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "destination_sync_mode": {
+          "$ref": "#/definitions/DestinationSyncMode"
+        },
+        "primary_key": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        }
+      },
+      "additionalProperties": true
+    }
+  }
+}

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations::Protocol
+    module Types
+      include Dry.Types()
+    end
+
+    SyncMode = Types::String.enum("full_refresh", "incremental")
+    SyncStatus = Types::String.enum("started", "running", "complete", "incomplete")
+    DestinationSyncMode = Types::String.enum("insert", "upsert")
+    ConnectorType = Types::String.enum("source", "destination")
+    ConnectorQueryType = Types::String.enum("raw_sql", "soql")
+    ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql")
+    ConnectionStatusType = Types::String.enum("succeeded", "failed")
+    StreamType = Types::String.enum("static", "dynamic")
+    StreamAction = Types::String.enum("fetch", "create", "update", "delete")
+    MultiwovenMessageType = Types::String.enum(
+      "record", "log", "connector_spec",
+      "connection_status", "catalog", "control",
+      "tracking"
+    )
+    ControlMessageType = Types::String.enum(
+      "rate_limit", "connection_config", "full_refresh"
+    )
+    LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
+    RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
+    SchemaMode = Types::String.enum("schema", "schemaless")
+
+    class ProtocolModel < Dry::Struct
+      extend Multiwoven::Integrations::Core::Utils
+      class << self
+        def from_json(json_string)
+          data = JSON.parse(json_string)
+          new(keys_to_symbols(data))
+        end
+      end
+    end
+
+    class ConnectionStatus < ProtocolModel
+      attribute :status, ConnectionStatusType
+      attribute? :message, Types::String.optional
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["connection_status"],
+          connection_status: self
+        )
+      end
+    end
+
+    class ConnectorSpecification < ProtocolModel
+      attribute? :documentation_url, Types::String.optional
+      attribute? :changelog_url, Types::String.optional
+      attribute :connection_specification, Types::Hash
+      attribute :supports_normalization, Types::Bool.default(false)
+      attribute :supports_dbt, Types::Bool.default(false)
+      attribute :stream_type, StreamType
+      attribute? :supported_destination_sync_modes, Types::Array.of(DestinationSyncMode).optional
+      attribute? :connector_query_type, ConnectorQueryType
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["connector_spec"],
+          connector_spec: self
+        )
+      end
+    end
+
+    class Connector < ProtocolModel
+      attribute :name, Types::String
+      attribute :type, ConnectorType
+      attribute :connection_specification, Types::Hash
+      attribute :query_type, Types::String.default("raw_sql").enum(*ConnectorQueryType.values)
+    end
+
+    class LogMessage < ProtocolModel
+      attribute :level, LogLevel
+      attribute :message, Types::String
+      attribute? :name, Types::String.optional
+      attribute? :stack_trace, Types::String.optional
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["log"],
+          log: self
+        )
+      end
+    end
+
+    class Model < ProtocolModel
+      attribute? :name, Types::String.optional
+      attribute :query, Types::String
+      attribute :query_type, ModelQueryType
+      attribute :primary_key, Types::String
+    end
+
+    class RecordMessage < ProtocolModel
+      attribute :data, Types::Hash
+      attribute :emitted_at, Types::Integer
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["record"],
+          record: self
+        )
+      end
+    end
+
+    class Stream < ProtocolModel
+      # Common
+      attribute :name, Types::String
+      attribute? :action, StreamAction
+      attribute :json_schema, Types::Hash
+      attribute? :supported_sync_modes, Types::Array.of(SyncMode).optional.default(["incremental"])
+
+      # Applicable for database streams
+      attribute :source_defined_cursor, Types::Bool.default(false)
+      attribute? :default_cursor_field, Types::Array.of(Types::String).optional
+      attribute? :source_defined_primary_key, Types::Array.of(Types::Array.of(Types::String)).optional
+      attribute? :namespace, Types::String.optional
+      # Applicable for API streams
+      attribute? :url, Types::String.optional
+      attribute? :request_method, Types::String.optional
+      attribute :batch_support, Types::Bool.default(false)
+      attribute :batch_size, Types::Integer.default(1)
+      # Rate limits
+      attribute? :request_rate_limit, Types::Integer
+      attribute? :request_rate_limit_unit, RequestRateLimitingUnit
+      attribute? :request_rate_concurrency, Types::Integer
+
+      def rate_limit_unit_seconds
+        case request_rate_limit_unit
+        when "minute"
+          60 # Seconds in a minute
+        when "hour"
+          3600 # Seconds in an hour
+        when "day"
+          86_400 # Seconds in a day
+        else
+          1 # Default case, consider as seconds or handle as error
+        end
+      end
+    end
+
+    class Catalog < ProtocolModel
+      attribute :streams, Types::Array.of(Stream)
+
+      # Rate limits
+      attribute? :request_rate_limit, Types::Integer.default(60)
+      attribute? :request_rate_limit_unit, RequestRateLimitingUnit
+      attribute? :request_rate_concurrency, Types::Integer.default(10)
+      attribute? :schema_mode, Types::String.optional.default("schema")
+      attribute :source_defined_cursor, Types::Bool.default(false)
+      attribute? :default_cursor_field, Types::Array.of(Types::String).optional
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["catalog"],
+          catalog: self
+        )
+      end
+    end
+
+    class SyncConfig < ProtocolModel
+      attr_accessor :offset, :limit
+
+      attribute :source, Connector
+      attribute :destination, Connector
+      attribute :model, Model
+      attribute :stream, Stream
+      attribute :sync_mode, SyncMode
+      attribute? :cursor_field, Types::String.optional
+      attribute? :current_cursor_field, Types::String.optional
+      attribute :destination_sync_mode, DestinationSyncMode
+    end
+
+    class ControlMessage < ProtocolModel
+      attribute :type, ControlMessageType
+      attribute :emitted_at, Types::Integer
+      attribute? :status, ConnectionStatusType.optional
+      attribute? :meta, Types::Hash
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["control"],
+          control: self
+        )
+      end
+    end
+
+    class TrackingMessage < ProtocolModel
+      attribute :success, Types::Integer.default(0)
+      attribute :failed, Types::Integer.default(0)
+      attribute? :meta, Types::Hash
+
+      def to_multiwoven_message
+        MultiwovenMessage.new(
+          type: MultiwovenMessageType["tracking"],
+          tracking: self
+        )
+      end
+    end
+
+    class MultiwovenMessage < ProtocolModel
+      attribute :type, MultiwovenMessageType
+      attribute? :log, LogMessage.optional
+      attribute? :connection_status, ConnectionStatus.optional
+      attribute? :connector_spec, ConnectorSpecification.optional
+      attribute? :catalog, Catalog.optional
+      attribute? :record, RecordMessage.optional
+      attribute? :control, ControlMessage.optional
+      attribute? :tracking, TrackingMessage.optional
+    end
+  end
+end