multiwoven-integrations 0.13.2 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f20d31c38b9e0f1311379e2d2b5f6637b2d5241abe6e0504ed0b448eb044909
-  data.tar.gz: 40f8da2f352ffbbbadee190efd5db47e89008e15d95165f7d7b9cba6d64574ce
+  metadata.gz: 214d9d0a9b6aed5037d74615ef55f30994cc6231e5be6b0ee1ffd95cdef4f96b
+  data.tar.gz: 84d1a191ec5681d0d1a25a16153614b9b8bdaad426263b6bc416109396545149
 SHA512:
-  metadata.gz: cd8d4130999ac7759f1bd8aefa1430ab9d8d2c3bd3a96c392896461ef46e64ac987546d523fd4d4e1851a337ed04e598e0f00ee480c2ce355fa8faaae8855707
-  data.tar.gz: 064f797d1d1bd3b5290a58325b8c75a8961b4341217a93f6f17ab91403de4f16143f90243f8519abda0b2c04304fe97aee0c89450b853ff1ea956a635e3efc0b
+  metadata.gz: 7f19bfda47c77a07744f473c1f1be751c2724453afcf66add6c780aeca06be0a380861afc666690f25eea2f180e4623b995f7ddd60492f9a0e20be940076f08e
+  data.tar.gz: 7f837557c947c38ff53c7434c88bd1a996cc7d441024c7e2c3392001abcdb32a6e72ec7515c626bba2a734c016f4e5179bedff3ce3241284bd9a0757d86cef34

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -11,7 +11,7 @@ module Multiwoven
     DestinationSyncMode = Types::String.enum("insert", "upsert")
     ConnectorType = Types::String.enum("source", "destination")
     ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml")
-    ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml")
+    ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml", "dynamic_sql")
     ConnectionStatusType = Types::String.enum("succeeded", "failed")
     StreamType = Types::String.enum("static", "dynamic", "user_defined")
     StreamAction = Types::String.enum("fetch", "create", "update", "delete")

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.13.2"
+    VERSION = "0.14.1"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -19,6 +19,7 @@ module Multiwoven
       DatabricksModel
       AwsSagemakerModel
       VertexModel
+      HttpModel
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/http_model/client.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module HttpModel
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        url_host = connection_config[:url_host]
+        headers = connection_config[:headers]
+        response = Multiwoven::Integrations::Core::HttpClient.request(
+          url_host,
+          HTTP_GET,
+          headers: headers
+        )
+        if success?(response)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "HTTP MODEL:CHECK_CONNECTION:EXCEPTION",
+                           type: "error"
+                         })
+        failure_status(e)
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "HTTP MODEL:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        # The server checks the ConnectorQueryType.
+        # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
+        # This query is then sent to the AI/ML model.
+        payload = JSON.parse(sync_config.model.query)
+        run_model(connection_config, payload)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "HTTP MODEL:READ:EXCEPTION",
+                           type: "error"
+                         })
+      end
+
+      private
+
+      def run_model(connection_config, payload)
+        connection_config = connection_config.with_indifferent_access
+        url_host = connection_config[:url_host]
+        headers = connection_config[:headers]
+        config = connection_config[:config]
+        config[:timeout] ||= 30
+        response = send_request(url_host, payload, headers, config)
+        process_response(response)
+      rescue StandardError => e
+        handle_exception(e, context: "HTTP MODEL:RUN_MODEL:EXCEPTION", type: "error")
+      end
+
+      def process_response(response)
+        if success?(response)
+          data = JSON.parse(response.body)
+          [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
+        else
+          create_log_message("HTTP MODEL:RUN_MODEL", "error", "request failed")
+        end
+      end
+
+      def send_request(url, payload, headers, config)
+        Multiwoven::Integrations::Core::HttpClient.request(
+          url,
+          HTTP_POST,
+          payload: payload,
+          headers: headers,
+          config: config
+        )
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/http_model/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/multiwoven/integrations/source/http_model/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "HttpModel",
+    "title": "HTTP Model",
+    "connector_type": "source",
+    "category": "AI Model",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "source-http-model",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/http_model/config/spec.json

@@ -0,0 +1,37 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/source/http-model",
+  "stream_type": "user_defined",
+  "connector_query_type": "ai_ml",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Http Model",
+    "type": "object",
+    "required": ["url_host"],
+    "properties": {
+      "url_host": {
+        "type": "string",
+        "title": "URL",
+        "order": 0
+      },
+      "headers": {
+        "type": "string",
+        "title": "Http Headers",
+        "order": 1
+      },
+      "config": {
+        "title": "",
+        "type": "object",
+        "properties": {
+          "timeout": {
+            "type": "integer",
+            "minimum": 0,
+            "default": 30,
+            "title": "Http Timeout",
+            "order": 0
+          }
+        },
+        "order": 2
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/http_model/icon.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<svg width="800px" height="800px" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none">
+
+
+
+
+
+

data/lib/multiwoven/integrations.rb

@@ -70,6 +70,7 @@ require_relative "integrations/source/oracle_db/client"
 require_relative "integrations/source/databrics_model/client"
 require_relative "integrations/source/aws_sagemaker_model/client"
 require_relative "integrations/source/google_vertex_model/client"
+require_relative "integrations/source/http_model/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.13.2
+  version: 0.14.1
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-11-11 00:00:00.000000000 Z
+date: 2024-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -720,6 +720,11 @@ files:
 - lib/multiwoven/integrations/source/google_vertex_model/config/meta.json
 - lib/multiwoven/integrations/source/google_vertex_model/config/spec.json
 - lib/multiwoven/integrations/source/google_vertex_model/icon.svg
+- lib/multiwoven/integrations/source/http_model/client.rb
+- lib/multiwoven/integrations/source/http_model/config/catalog.json
+- lib/multiwoven/integrations/source/http_model/config/meta.json
+- lib/multiwoven/integrations/source/http_model/config/spec.json
+- lib/multiwoven/integrations/source/http_model/icon.svg
 - lib/multiwoven/integrations/source/maria_db/client.rb
 - lib/multiwoven/integrations/source/maria_db/config/meta.json
 - lib/multiwoven/integrations/source/maria_db/config/spec.json