multiwoven-integrations 0.15.11 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47ebb726bc626f15a399cb9f1b94f594efa9b4fb75df06482014f10e37e9f6aa
-  data.tar.gz: 429e1a5d6f20f8445c9ef1fd6ecc9073443d97b9d8572c4224d871d4826c9fb7
+  metadata.gz: 85dd988a522326237c492ac96db325a16225f7e0ab96438fdc3b1475139e07e0
+  data.tar.gz: b36e2addc56a428a574684f317f1b1b10f2f864ea915f31564290f607604bf24
 SHA512:
-  metadata.gz: 9a2757cadcb8754020f92c4a9503eb27418c5408eda08bc90baadf2d055c2d6cdb6a0877a8a0e744ecf0cafa578d1397efcc53f9c8ef13903dde6b77d22f1029
-  data.tar.gz: 054cb615fe6ccb54536648970ae365a7ec6326ed037d9f2fe2592ba0197d8c9dc87fc165bc89790df2d98fed8542cadfb9a98095ad84097fe963e584e8dec137
+  metadata.gz: 5b2ea53b9847be5f4c4d2618cfe1a89376f980aefff351cce6a7b63b8f76d56f3067180fab42d2d3d1977996e8b34308f085ffcb793aa6e62c894048f6116549
+  data.tar.gz: 9e320fedd61454ea428ff0935bf7fae584f21850f2a467b951dfc98c22b24a982f9fbf76007481416241c4b35c671bfd8a362d34236e2cb7ced1a2b5325b68ce

data/lib/multiwoven/integrations/core/constants.rb

@@ -63,6 +63,8 @@ module Multiwoven
       # google sheets
       GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive"
       GOOGLE_SPREADSHEET_ID_REGEX = %r{/d/([-\w]{20,})/}.freeze
+
+      OPEN_AI_URL = "https://api.openai.com/v1/chat/completions"
     end
   end
 end

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.15.11"
+    VERSION = "0.16.0"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -20,6 +20,7 @@ module Multiwoven
       AwsSagemakerModel
       VertexModel
       HttpModel
+      OpenAI
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/http_model/config/meta.json

@@ -4,7 +4,7 @@
     "title": "HTTP Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
-    "documentation_url": "https://docs.mutliwoven.com",
+    "documentation_url": "https://docs.mutltiwoven.com",
     "github_issue_label": "source-http-model",
     "icon": "icon.svg",
     "license": "MIT",

data/lib/multiwoven/integrations/source/open_ai/client.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module OpenAI
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = prepare_config(connection_config)
+        response = send_request(
+          url: OPEN_AI_URL,
+          http_method: HTTP_POST,
+          payload: JSON.parse(connection_config[:request_format]),
+          headers: auth_headers(connection_config[:api_key]),
+          config: connection_config[:config]
+        )
+        success?(response) ? success_status : failure_status(nil)
+      rescue StandardError => e
+        handle_exception(e, { context: "OPEN AI:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, { context: "OPEN AI:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = prepare_config(sync_config.source.connection_specification)
+        stream = connection_config[:is_stream] ||= false
+        # The server checks the ConnectorQueryType.
+        # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
+        # This query is then sent to the AI/ML model.
+        payload = parse_json(sync_config.model.query)
+
+        if stream
+          run_model_stream(connection_config, payload) { |message| yield message if block_given? }
+        else
+          run_model(connection_config, payload)
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "OPEN AI:READ:EXCEPTION", type: "error" })
+      end
+
+      private
+
+      def prepare_config(config)
+        config.with_indifferent_access.tap do |conf|
+          conf[:config][:timeout] ||= 30
+        end
+      end
+
+      def parse_json(json_string)
+        JSON.parse(json_string)
+      rescue JSON::ParserError => e
+        handle_exception(e, { context: "OPEN AI:PARSE_JSON:EXCEPTION", type: "error" })
+        {}
+      end
+
+      def run_model(connection_config, payload)
+        response = send_request(
+          url: OPEN_AI_URL,
+          http_method: HTTP_POST,
+          payload: payload,
+          headers: auth_headers(connection_config[:api_key]),
+          config: connection_config[:config]
+        )
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, { context: "OPEN AI:RUN_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def run_model_stream(connection_config, payload)
+        send_streaming_request(
+          url: OPEN_AI_URL,
+          http_method: HTTP_POST,
+          payload: payload,
+          headers: auth_headers(connection_config[:api_key]),
+          config: connection_config[:config]
+        ) do |chunk|
+          process_streaming_response(chunk) { |message| yield message if block_given? }
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "OPEN AI:RUN_STREAM_MODEL:EXCEPTION", type: "error" })
+      end
+
+      def process_response(response)
+        if success?(response)
+          data = JSON.parse(response.body)
+          [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
+        else
+          create_log_message("OPEN AI:RUN_MODEL", "error", "request failed: #{response.body}")
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "OPEN AI:PROCESS_RESPONSE:EXCEPTION", type: "error" })
+      end
+
+      def extract_data_entries(chunk)
+        chunk.split(/^data: /).map(&:strip).reject(&:empty?)
+      end
+
+      def process_streaming_response(chunk)
+        data_entries = extract_data_entries(chunk)
+        data_entries.each do |entry|
+          next if entry == "[DONE]"
+
+          data = parse_json(entry)
+          yield [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] if block_given?
+        rescue StandardError => e
+          handle_exception(e, { context: "OPEN AI:PROCESS_STREAMING_RESPONSE:EXCEPTION", type: "error", entry: entry })
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/open_ai/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/multiwoven/integrations/source/open_ai/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "OpenAI",
+    "title": "OpenAI Model Endpoint",
+    "connector_type": "source",
+    "category": "AI Model",
+    "documentation_url": "https://docs.mutltiwoven.com",
+    "github_issue_label": "source-open-ai-model",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/open_ai/config/spec.json

@@ -0,0 +1,54 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/source/open-ai-endpoint",
+  "stream_type": "user_defined",
+  "connector_query_type": "ai_ml",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Open AI Endpoint",
+    "type": "object",
+    "required": ["api_key", "request_format", "response_format"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "multiwoven_secret": true,
+        "title": "API Key",
+        "order": 0
+      },
+      "is_stream": {
+        "type": "boolean",
+        "title": "Streaming Enabled",
+        "description": "Enables data streaming for such as chat, when supported by the model. When true, messages and model data are processed in chunks for immediate delivery, enhancing responsiveness. Default is false, processing only after the entire response is received.",
+        "default": false,
+        "order": 1
+      },
+      "config": {
+        "title": "",
+        "type": "object",
+        "properties": {
+          "timeout": {
+            "type": "string",
+            "default": "30",
+            "title": "HTTP Timeout",
+            "description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
+            "order": 0
+          }
+        },
+        "order": 2
+      },
+      "request_format": {
+        "title": "Request Format",
+        "description": "Sample Request Format",
+        "type": "string",
+        "x-request-format": true,
+        "order": 3
+      },
+      "response_format": {
+        "title": "Response Format",
+        "description": "Sample Response Format",
+        "type": "string",
+        "x-response-format": true,
+        "order": 4
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/open_ai/icon.svg

@@ -0,0 +1 @@
+<svg viewBox="0 0 1180 320" xmlns="http://www.w3.org/2000/svg"><path d="m367.44 153.84c0 52.32 33.6 88.8 80.16 88.8s80.16-36.48 80.16-88.8-33.6-88.8-80.16-88.8-80.16 36.48-80.16 88.8zm129.6 0c0 37.44-20.4 61.68-49.44 61.68s-49.44-24.24-49.44-61.68 20.4-61.68 49.44-61.68 49.44 24.24 49.44 61.68z"/><path d="m614.27 242.64c35.28 0 55.44-29.76 55.44-65.52s-20.16-65.52-55.44-65.52c-16.32 0-28.32 6.48-36.24 15.84v-13.44h-28.8v169.2h28.8v-56.4c7.92 9.36 19.92 15.84 36.24 15.84zm-36.96-69.12c0-23.76 13.44-36.72 31.2-36.72 20.88 0 32.16 16.32 32.16 40.32s-11.28 40.32-32.16 40.32c-17.76 0-31.2-13.2-31.2-36.48z"/><path d="m747.65 242.64c25.2 0 45.12-13.2 54-35.28l-24.72-9.36c-3.84 12.96-15.12 20.16-29.28 20.16-18.48 0-31.44-13.2-33.6-34.8h88.32v-9.6c0-34.56-19.44-62.16-55.92-62.16s-60 28.56-60 65.52c0 38.88 25.2 65.52 61.2 65.52zm-1.44-106.8c18.24 0 26.88 12 27.12 25.92h-57.84c4.32-17.04 15.84-25.92 30.72-25.92z"/><path d="m823.98 240h28.8v-73.92c0-18 13.2-27.6 26.16-27.6 15.84 0 22.08 11.28 22.08 26.88v74.64h28.8v-83.04c0-27.12-15.84-45.36-42.24-45.36-16.32 0-27.6 7.44-34.8 15.84v-13.44h-28.8z"/><path d="m1014.17 67.68-65.28 172.32h30.48l14.64-39.36h74.4l14.88 39.36h30.96l-65.28-172.32zm16.8 34.08 27.36 72h-54.24z"/><path d="m1163.69 68.18h-30.72v172.32h30.72z"/><path d="m297.06 130.97c7.26-21.79 4.76-45.66-6.85-65.48-17.46-30.4-52.56-46.04-86.84-38.68-15.25-17.18-37.16-26.95-60.13-26.81-35.04-.08-66.13 22.48-76.91 55.82-22.51 4.61-41.94 18.7-53.31 38.67-17.59 30.32-13.58 68.54 9.92 94.54-7.26 21.79-4.76 45.66 6.85 65.48 17.46 30.4 52.56 46.04 86.84 38.68 15.24 17.18 37.16 26.95 60.13 26.8 35.06.09 66.16-22.49 76.94-55.86 22.51-4.61 41.94-18.7 53.31-38.67 17.57-30.32 13.55-68.51-9.94-94.51zm-120.28 168.11c-14.03.02-27.62-4.89-38.39-13.88.49-.26 1.34-.73 1.89-1.07l63.72-36.8c3.26-1.85 5.26-5.32 5.24-9.07v-89.83l26.93 15.55c.29.14.48.42.52.74v74.39c-.04 33.08-26.83 59.9-59.91 59.97zm-128.84-55.03c-7.03-12.14-9.56-26.37-7.15-40.18.47.28 1.3.79 1.89 1.13l63.72 36.8c3.23 1.89 7.23 1.89 10.47 0l77.79-44.92v31.1c.02.32-.13.63-.38.83l-64.41 37.19c-28.69 16.52-65.33 6.7-81.92-21.95zm-16.77-139.09c7-12.16 18.05-21.46 31.21-26.29 0 .55-.03 1.52-.03 2.2v73.61c-.02 3.74 1.98 7.21 5.23 9.06l77.79 44.91-26.93 15.55c-.27.18-.61.21-.91.08l-64.42-37.22c-28.63-16.58-38.45-53.21-21.95-81.89zm221.26 51.49-77.79-44.92 26.93-15.54c.27-.18.61-.21.91-.08l64.42 37.19c28.68 16.57 38.51 53.26 21.94 81.94-7.01 12.14-18.05 21.44-31.2 26.28v-75.81c.03-3.74-1.96-7.2-5.2-9.06zm26.8-40.34c-.47-.29-1.3-.79-1.89-1.13l-63.72-36.8c-3.23-1.89-7.23-1.89-10.47 0l-77.79 44.92v-31.1c-.02-.32.13-.63.38-.83l64.41-37.16c28.69-16.55 65.37-6.7 81.91 22 6.99 12.12 9.52 26.31 7.15 40.1zm-168.51 55.43-26.94-15.55c-.29-.14-.48-.42-.52-.74v-74.39c.02-33.12 26.89-59.96 60.01-59.94 14.01 0 27.57 4.92 38.34 13.88-.49.26-1.33.73-1.89 1.07l-63.72 36.8c-3.26 1.85-5.26 5.31-5.24 9.06l-.04 89.79zm14.63-31.54 34.65-20.01 34.65 20v40.01l-34.65 20-34.65-20z"/></svg>

data/lib/multiwoven/integrations.rb

@@ -73,6 +73,7 @@ require_relative "integrations/source/databrics_model/client"
 require_relative "integrations/source/aws_sagemaker_model/client"
 require_relative "integrations/source/google_vertex_model/client"
 require_relative "integrations/source/http_model/client"
+require_relative "integrations/source/open_ai/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.15.11
+  version: 0.16.0
 platform: ruby
 authors:
 - Subin T P
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-12-13 00:00:00.000000000 Z
+date: 2024-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -735,6 +735,11 @@ files:
 - lib/multiwoven/integrations/source/maria_db/config/meta.json
 - lib/multiwoven/integrations/source/maria_db/config/spec.json
 - lib/multiwoven/integrations/source/maria_db/icon.svg
+- lib/multiwoven/integrations/source/open_ai/client.rb
+- lib/multiwoven/integrations/source/open_ai/config/catalog.json
+- lib/multiwoven/integrations/source/open_ai/config/meta.json
+- lib/multiwoven/integrations/source/open_ai/config/spec.json
+- lib/multiwoven/integrations/source/open_ai/icon.svg
 - lib/multiwoven/integrations/source/oracle_db/client.rb
 - lib/multiwoven/integrations/source/oracle_db/config/meta.json
 - lib/multiwoven/integrations/source/oracle_db/config/spec.json
@@ -767,7 +772,7 @@ metadata:
   homepage_uri: https://www.multiwoven.com/
   source_code_uri: https://github.com/Multiwoven/multiwoven/tree/main/integrations
   changelog_uri: https://github.com/Multiwoven/multiwoven/blob/main/integrations/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -783,7 +788,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.4.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Integration suite for open source reverse ETL platform
 test_files: []