RubyGems - multiwoven-integrations - Versions diffs - 0.30.0 → 0.30.2 - Mend

multiwoven-integrations 0.30.0 → 0.30.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (31) hide show

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2d7bca699532e6c4911498f3bbd9548ee96f7dc1a36bb6c24629e85b61f2f57
-  data.tar.gz: f99005b99170a995b223d3e16a5d79625e021a1156ffca0c96ebd0ca227e1df5
+  metadata.gz: 51616b2e7b06336873aceb0c9e5731526dd7e57ba332eab190bc215d99c4ed8d
+  data.tar.gz: d376ae2826566eea31fd92dbd4e4ddf16034e1ef9085edf08f7b554744424583
 SHA512:
-  metadata.gz: 591361ae2cb41fe8ba21f9c910e9d9cc174a9de3c679612ed574745dd34d536fc8052272df89c773854012ddfcfc85a4360eac559b518c3476415bf0ee66cdf0
-  data.tar.gz: d44eb9694eb04719415c17a4d861c549d3c7e8e43094f2f4ee82bbb8b3e1ea751c053bff3a9dfc42753e946f4c96fb4f0ba24b134c0a274363b668e5e091783b
+  metadata.gz: fc9778e2da0499ed9bec602005b3b69aadd8e038e251d77acc8407bde99dbf213c1ff0d6f937a832c1d4aa660dbae578de8042caf9c0720fca3ddcff2e0d762a
+  data.tar.gz: a8438815522f82a1bc6db95c6bd29c94a55f182bf459f90c3e01209f54b17e303d939280a63549f6c1eff5da4e57ca1f80e85b98646f6f935995f3a371f7b274

data/lib/multiwoven/integrations/rollout.rb CHANGED Viewed

@@ -2,7 +2,7 @@
 module Multiwoven
   module Integrations
-    VERSION = "0.30.0"
+    VERSION = "0.30.2"
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/amazon_s3/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Amazon S3",
     "connector_type": "source",
     "category": "Data Lake",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/amazon_s3",
     "github_issue_label": "source-amazons3",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/anthropic/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Anthropic Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "LLM",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/anthropic-model",
     "github_issue_label": "source-anthropic-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/aws_athena/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "AWS Athena",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/aws_athena",
     "github_issue_label": "source-aws-athena",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/aws_bedrock_model/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "AWS Bedrock Model",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/aws_bedrock-model",
     "github_issue_label": "source-aws-bedrock-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "AWS Sagemaker Model",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/aws_sagemaker-model",
     "github_issue_label": "source-aws-sagemaker-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/bigquery/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Google BigQuery",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/bquery",
     "github_issue_label": "source-bigquery",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/clickhouse/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "ClickHouse",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/clickhouse",
     "github_issue_label": "source-clickhouse",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/databricks/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
       "title": "Databricks Data Warehouse",
       "connector_type": "source",
       "category": "Data Warehouse",
+      "sub_category": "Relational Database",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/databricks",
       "github_issue_label": "source-databricks",
       "icon": "icon.svg",

data/lib/multiwoven/integrations/source/databrics_model/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
       "title": "Databricks Model",
       "connector_type": "source",
       "category": "AI Model",
+      "sub_category": "AI_ML Service",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/databricks-model",
       "github_issue_label": "source-databricks-foundation",
       "icon": "icon.svg",

data/lib/multiwoven/integrations/source/firecrawl/client.rb CHANGED Viewed

@@ -18,12 +18,9 @@ module Multiwoven::Integrations::Source
         failure_status(e)
       end
-      def discover(connection_config)
-        connection_config = connection_config.with_indifferent_access
-        create_connection(connection_config)
-        response = execute_scrape(FIRECRAWL_SCRAPE_URL)
-        results = JSON.parse(response.body)
-        catalog = Catalog.new(streams: create_streams(results))
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
         catalog.to_multiwoven_message
       rescue StandardError => e
         handle_exception(e, { context: "FIRECRAWL:DISCOVER:EXCEPTION", type: "error" })
@@ -32,9 +29,8 @@ module Multiwoven::Integrations::Source
       def read(sync_config)
         connection_config = sync_config.source.connection_specification
         connection_config = connection_config.with_indifferent_access
-        query = sync_config.model.query
         url = create_connection(connection_config)
-        query(url, query)
+        query(url, nil)
       rescue StandardError => e
         handle_exception(e, {
                            context: "FIRECRAWL:READ:EXCEPTION",
@@ -62,7 +58,16 @@ module Multiwoven::Integrations::Source
         FIRECRAWL_CRAWL_URL
       end
-      def query(url, _query)
+      def query(url, query)
+        has_limit = query.match(/LIMIT\s+(\d+)\s*$/i) if query.present?
+        if has_limit.present?
+          if @config["includePaths"]&.any?
+            path = @config["includePaths"].first
+            @config["url"] = URI.join(@config["url"], path).to_s
+          end
+          @config.delete("includePaths")
+          @config[:limit] = has_limit[1].to_i
+        end
         request = execute_crawl(url)
         request = JSON.parse(request.body)
         crawl_url = get_request_url(request)
@@ -89,16 +94,6 @@ module Multiwoven::Integrations::Source
         )
       end
-      def execute_scrape(url)
-        send_request(
-          url: url,
-          http_method: HTTP_POST,
-          payload: JSON.parse({ "url": @base_url }.to_json),
-          headers: auth_headers(@api_key),
-          config: {}
-        )
-      end
       def crawl_activity
         send_request(
           url: FIRECRAWL_CRAWL_ACTIVE_URL,
@@ -148,37 +143,6 @@ module Multiwoven::Integrations::Source
         end
       end
-      def create_streams(records)
-        group_by_table(records).map do |r|
-          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
-        end
-      end
-      def group_by_table(response)
-        columns = response["data"].map do |key, value|
-          {
-            column_name: key,
-            data_type: "string",
-            is_nullable: value.nil?
-          }
-        end
-        if response["data"]["metadata"]["url"]
-          columns << {
-            column_name: "url",
-            data_type: "string",
-            is_nullable: response["data"]["metadata"]["url"].nil?
-          }
-        end
-        [
-          {
-            tablename: "scrape",
-            columns: columns
-          }
-        ]
-      end
       def build_url(url, id)
         format(url, id: id)
       end

data/lib/multiwoven/integrations/source/firecrawl/config/catalog.json ADDED Viewed

@@ -0,0 +1,26 @@
+{
+  "request_rate_limit": 15,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 5,
+  "streams": [
+    {
+      "name": "scrape",
+      "action": "fetch",
+      "json_schema": {
+        "type": "object",
+        "properties": {
+          "markdown": {
+            "type": "string"
+          },
+          "metadata": {
+            "type": "string"
+          },
+          "url": {
+            "type": "string"
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"]
+    }
+  ]
+}

data/lib/multiwoven/integrations/source/firecrawl/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
       "title": "Firecrawl",
       "connector_type": "source",
       "category": "Data Warehouse",
+      "sub_category": "Web Scraper",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/firecrawl",
       "github_issue_label": "source-firecrawl",
       "icon": "icon.svg",

data/lib/multiwoven/integrations/source/generic_open_ai/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Generic OpenAI Spec Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/generic_open_ai-endpoint",
     "github_issue_label": "source-generic-open-ai-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/google_vertex_model/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Google Vertex",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/google_vertex-model",
     "github_issue_label": "source-vertex",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/http_model/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "HTTP Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/http-model-endpoint",
     "github_issue_label": "source-http-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/intuit_quick_books/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
       "title": "Intuit QuickBooks",
       "connector_type": "source",
       "category": "Data Warehouse",
+      "sub_category": "Relational Database",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/intuit_quickbooks",
       "github_issue_label": "source-intuit-quickbooks",
       "icon": "icon.svg",

data/lib/multiwoven/integrations/source/maria_db/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Maria DB",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/maria_db",
     "github_issue_label": "source-maria-db",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/open_ai/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "OpenAI Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "LLM",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/open_ai-model",
     "github_issue_label": "source-open-ai-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/oracle_db/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Oracle DB",
     "connector_type": "source",
     "category": "Database",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/oracle",
     "github_issue_label": "source-oracle",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/pinecone_db/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Pinecone DB",
     "connector_type": "source",
     "category": "Database",
+    "sub_category": "Vector Database",
     "documentation_url": "https://docs.squared.ai/activation/vector_search/pinecone_db",
     "github_issue_label": "source-pinecone-db",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/postgresql/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "PostgreSQL",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/postgresql",
     "github_issue_label": "source-postgresql",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/qdrant/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Qdrant",
     "connector_type": "source",
     "category": "Database",
+    "sub_category": "Vector Database",
     "documentation_url": "https://docs.squared.ai/activation/vector_search/qdrant",
     "github_issue_label": "source-qdrant",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/redshift/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Amazon Redshift",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/redshift",
     "github_issue_label": "source-redshift",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/meta.json CHANGED Viewed

@@ -6,6 +6,7 @@
     "connector_sub_type": "api",
     "direct_query_support": true,
     "category": "Retail",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/salesforce-consumer-goods-cloud",
     "github_issue_label": "source-salesforce-consumer-goods",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/sftp/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "SFTP",
     "connector_type": "source",
     "category": "File Storage",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sftp",
     "github_issue_label": "source-sftp",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/snowflake/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "Snowflake",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/snowflake",
     "github_issue_label": "source-snowflake",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/watsonx_ai/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
     "title": "WatsonX AI Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/watsonx_ai-model",
     "github_issue_label": "source-watsonx-ai-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/watsonx_data/config/meta.json CHANGED Viewed

@@ -4,6 +4,7 @@
       "title": "WatsonX Data Endpoint",
       "connector_type": "source",
       "category": "Data Warehouse",
+      "sub_category": "Relational Database",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/watsonx_data",
       "github_issue_label": "source-watsonx-data-endpoint",
       "icon": "icon.svg",

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.30.0
+  version: 0.30.2
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-06-25 00:00:00.000000000 Z
+date: 2025-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -752,6 +752,7 @@ files:
 - lib/multiwoven/integrations/source/databrics_model/config/spec.json
 - lib/multiwoven/integrations/source/databrics_model/icon.svg
 - lib/multiwoven/integrations/source/firecrawl/client.rb
+- lib/multiwoven/integrations/source/firecrawl/config/catalog.json
 - lib/multiwoven/integrations/source/firecrawl/config/meta.json
 - lib/multiwoven/integrations/source/firecrawl/config/spec.json
 - lib/multiwoven/integrations/source/firecrawl/icon.svg