multiwoven-integrations 0.29.1 → 0.30.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61efa047be9009c6caba7c4e7cadde86454e61e193c2a641d0f68c05dc807a7a
-  data.tar.gz: fcab50ec033298e5d983a247a5f631649df789b82adacfd10b44ada2d0f6cef6
+  metadata.gz: 77b60f06178a034c3b84ac94d582f500a25f2e987c6a3b813b8220e52bf2958c
+  data.tar.gz: abd9af3678499e495b4f6b6650db793aa9a29a1a7781ff84caad07d1826e1b88
 SHA512:
-  metadata.gz: e91cf290679f5bd8ede4251562d8ec04c9b1a6f0a7b137587ddb48048a1a4406514c20f9f263f9fad94b281ab07f26cc5d4e44393e89617b666d3693366fc4e9
-  data.tar.gz: 696599dd7fe27266d719ffaaa20b004b7173a8af57c55211846ac41651cf370a7d940988ab2287d2aaea3183ccd6b754a93869e1482fd4508c30bafc25197d59
+  metadata.gz: f4535cf7f29ee0dec79736d292018fa18c5d0126e7603401f07dec97a10a8b63efb1f616afcf0b230cd2858f8a7979b2e191d6b169d6c2fb41ea06c9bf64eded
+  data.tar.gz: b4a3288ae74b92dfcf729d970db0a2cadd670aee1a5ee83f536bab6c42c07394a587b18a2e48fcf913257d0e6d6e97d6283aecf342c3b43b14ae0e53ac93dd95

data/lib/multiwoven/integrations/core/constants.rb

@@ -91,6 +91,13 @@ module Multiwoven
 
       # Qdrant
       QDRANT_SEARCH_URL = "%<host>s/collections/%<collection_name>s/points/search"
+
+      # Firecrawl
+      FIRECRAWL_CRAWL_URL = "https://api.firecrawl.dev/v1/crawl"
+      FIRECRAWL_SCRAPE_URL = "https://api.firecrawl.dev/v1/scrape"
+      FIRECRAWL_CRAWL_ACTIVE_URL = "https://api.firecrawl.dev/v1/crawl/active"
+      FIRECRAWL_GET_CRAWL_URL = "https://api.firecrawl.dev/v1/crawl/%<id>s"
+      FIRECRAWL_REQUEST_RATE_LIMIT = 5
     end
   end
 end

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.29.1"
+    VERSION = "0.30.1"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -30,6 +30,7 @@ module Multiwoven
       IntuitQuickBooks
       PineconeDB
       Qdrant
+      Firecrawl
     ].freeze
 
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/amazon_s3/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Amazon S3",
     "connector_type": "source",
     "category": "Data Lake",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/amazon_s3",
     "github_issue_label": "source-amazons3",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/anthropic/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Anthropic Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "LLM",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/anthropic-model",
     "github_issue_label": "source-anthropic-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/aws_athena/config/meta.json

@@ -4,6 +4,7 @@
     "title": "AWS Athena",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/aws_athena",
     "github_issue_label": "source-aws-athena",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/aws_bedrock_model/config/meta.json

@@ -4,6 +4,7 @@
     "title": "AWS Bedrock Model",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/aws_bedrock-model",
     "github_issue_label": "source-aws-bedrock-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json

@@ -4,6 +4,7 @@
     "title": "AWS Sagemaker Model",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/aws_sagemaker-model",
     "github_issue_label": "source-aws-sagemaker-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/bigquery/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Google BigQuery",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/bquery",
     "github_issue_label": "source-bigquery",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/clickhouse/config/meta.json

@@ -4,6 +4,7 @@
     "title": "ClickHouse",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/clickhouse",
     "github_issue_label": "source-clickhouse",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/databricks/config/meta.json

@@ -4,6 +4,7 @@
       "title": "Databricks Data Warehouse",
       "connector_type": "source",
       "category": "Data Warehouse",
+      "sub_category": "Relational Database",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/databricks",
       "github_issue_label": "source-databricks",
       "icon": "icon.svg",

data/lib/multiwoven/integrations/source/databrics_model/config/meta.json

@@ -4,6 +4,7 @@
       "title": "Databricks Model",
       "connector_type": "source",
       "category": "AI Model",
+      "sub_category": "AI_ML Service",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/databricks-model",
       "github_issue_label": "source-databricks-foundation",
       "icon": "icon.svg",

data/lib/multiwoven/integrations/source/firecrawl/client.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Source
+  module Firecrawl
+    include Multiwoven::Integrations::Core
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        request = crawl_activity
+        if success?(request)
+          success_status
+        else
+          failure_status(nil)
+        end
+      rescue StandardError => e
+        handle_exception(e, { context: "FIRECRAWL:CHECK_CONNECTION:EXCEPTION", type: "error" })
+        failure_status(e)
+      end
+
+      def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        create_connection(connection_config)
+        response = execute_scrape(FIRECRAWL_SCRAPE_URL)
+        results = JSON.parse(response.body)
+        catalog = Catalog.new(streams: create_streams(results))
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, { context: "FIRECRAWL:DISCOVER:EXCEPTION", type: "error" })
+      end
+
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
+        query = sync_config.model.query
+        url = create_connection(connection_config)
+        query(url, query)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "FIRECRAWL:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+
+      private
+
+      def create_connection(connection_config)
+        @base_url = connection_config[:base_url]
+        @api_key = connection_config[:api_key]
+        @config = if connection_config[:config].present?
+                    connection_config[:config].transform_values do |value|
+                      JSON.parse(value)
+                    rescue JSON::ParserError
+                      value
+                    end
+                  else
+                    {}
+                  end
+        @config[:url] ||= connection_config[:base_url]
+        FIRECRAWL_CRAWL_URL
+      end
+
+      def query(url, _query)
+        request = execute_crawl(url)
+        request = JSON.parse(request.body)
+        crawl_url = get_request_url(request)
+        response = get_crawl_result(crawl_url)
+        response["data"].map do |row|
+          metadata_json = row["metadata"].to_json if row["metadata"]
+          metadata_url = row["metadata"]["url"]
+          data = {
+            "metadata": metadata_json,
+            "markdown": row["markdown"],
+            "url": metadata_url
+          }
+          RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+
+      def execute_crawl(url)
+        send_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: JSON.parse(@config.to_json),
+          headers: auth_headers(@api_key),
+          config: {}
+        )
+      end
+
+      def execute_scrape(url)
+        send_request(
+          url: url,
+          http_method: HTTP_POST,
+          payload: JSON.parse({ "url": @base_url }.to_json),
+          headers: auth_headers(@api_key),
+          config: {}
+        )
+      end
+
+      def crawl_activity
+        send_request(
+          url: FIRECRAWL_CRAWL_ACTIVE_URL,
+          http_method: HTTP_GET,
+          payload: {},
+          headers: auth_headers(@api_key),
+          config: {}
+        )
+      end
+
+      # This is to make sure the /crawl/{id} was returned in request.
+      # If not use /crawl/active to retrieve it.
+      def get_request_url(request)
+        if request["url"].blank?
+          if request["error"].present?
+            time = request["error"][/retry after (\d+)s/, 1].to_i
+            sleep(time)
+            execute_crawl(FIRECRAWL_CRAWL_URL)
+          end
+          active = crawl_activity
+          crawl_active = JSON.parse(active.body)
+
+          raise "Missing crawl result URL and no active crawl ID available." unless crawl_active["crawls"][-1]["id"].present?
+
+          crawl_id = crawl_active["crawls"][-1]["id"]
+          build_url(FIRECRAWL_GET_CRAWL_URL, crawl_id.to_s)
+        else
+          request["url"]
+        end
+      end
+
+      # Crawl job needs time to finish task. This method will check if the job is complete.
+      # If not sleep for 5 seconds and check again.
+      def get_crawl_result(url)
+        loop do
+          response = send_request(
+            url: url,
+            http_method: HTTP_GET,
+            payload: {},
+            headers: auth_headers(@api_key),
+            config: {}
+          )
+          response = JSON.parse(response.body)
+          return response if response["status"] != "scraping"
+
+          sleep(FIRECRAWL_REQUEST_RATE_LIMIT)
+        end
+      end
+
+      def create_streams(records)
+        group_by_table(records).map do |r|
+          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+        end
+      end
+
+      def group_by_table(response)
+        columns = response["data"].map do |key, value|
+          {
+            column_name: key,
+            data_type: "string",
+            is_nullable: value.nil?
+          }
+        end
+
+        if response["data"]["metadata"]["url"]
+          columns << {
+            column_name: "url",
+            data_type: "string",
+            is_nullable: response["data"]["metadata"]["url"].nil?
+          }
+        end
+
+        [
+          {
+            tablename: "scrape",
+            columns: columns
+          }
+        ]
+      end
+
+      def build_url(url, id)
+        format(url, id: id)
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/firecrawl/config/meta.json

@@ -0,0 +1,17 @@
+{
+    "data": {
+      "name": "Firecrawl",
+      "title": "Firecrawl",
+      "connector_type": "source",
+      "category": "Data Warehouse",
+      "sub_category": "Web Scraper",
+      "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/firecrawl",
+      "github_issue_label": "source-firecrawl",
+      "icon": "icon.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "multiwoven"]
+    }
+  }
+  

data/lib/multiwoven/integrations/source/firecrawl/config/spec.json

@@ -0,0 +1,31 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/firecrawl",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Firecrawl",
+    "type": "object",
+    "required": ["api_key","base_url"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "title": "API key",
+        "multiwoven_secret": true,
+        "order": 0
+      },
+      "base_url": {
+        "type": "string",
+        "title": "Base URL",
+        "order": 1
+      },
+      "config": {
+        "title": "Configuration",
+        "order": 2,
+        "additionalProperties": {
+          "type": "string"
+        }
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/firecrawl/icon.svg

@@ -0,0 +1,4 @@
+<svg width="2907" height="600" viewBox="0 0 2907 600" fill="none" xmlns="http://www.w3.org/2000/svg">
+<text x="0" y="599.93" font-size="600" dominant-baseline="text-after-edge">🔥</text>
+<path d="M768.456 483V119.364H993.953V166.594H823.321V277.389H977.796V324.442H823.321V483H768.456ZM1052.16 483V210.273H1105.25V483H1052.16ZM1078.97 168.192C1069.74 168.192 1061.81 165.114 1055.18 158.959C1048.67 152.685 1045.42 145.228 1045.42 136.587C1045.42 127.827 1048.67 120.37 1055.18 114.214C1061.81 107.941 1069.74 104.804 1078.97 104.804C1088.21 104.804 1096.08 107.941 1102.59 114.214C1109.22 120.37 1112.53 127.827 1112.53 136.587C1112.53 145.228 1109.22 152.685 1102.59 158.959C1096.08 165.114 1088.21 168.192 1078.97 168.192ZM1171.68 483V210.273H1222.99V253.597H1225.83C1230.8 238.919 1239.56 227.377 1252.11 218.973C1264.77 210.45 1279.1 206.189 1295.08 206.189C1298.39 206.189 1302.3 206.307 1306.8 206.544C1311.41 206.781 1315.02 207.077 1317.63 207.432V258.213C1315.5 257.621 1311.71 256.97 1306.26 256.26C1300.82 255.431 1295.37 255.017 1289.93 255.017C1277.38 255.017 1266.19 257.68 1256.37 263.007C1246.66 268.215 1238.97 275.495 1233.29 284.847C1227.61 294.08 1224.76 304.615 1224.76 316.452V483H1171.68ZM1466.35 488.504C1439.48 488.504 1416.33 482.763 1396.92 471.281C1377.63 459.681 1362.71 443.405 1352.18 422.453C1341.76 401.383 1336.55 376.703 1336.55 348.412C1336.55 320.476 1341.76 295.855 1352.18 274.548C1362.71 253.241 1377.39 236.61 1396.21 224.655C1415.15 212.699 1437.29 206.722 1462.62 206.722C1478.01 206.722 1492.92 209.267 1507.36 214.357C1521.8 219.446 1534.77 227.437 1546.25 238.327C1557.73 249.217 1566.78 263.362 1573.41 280.763C1580.04 298.045 1583.36 319.056 1583.36 343.795V362.616H1366.56V322.844H1531.33C1531.33 308.876 1528.49 296.506 1522.81 285.734C1517.13 274.844 1509.14 266.262 1498.84 259.989C1488.66 253.715 1476.7 250.578 1462.97 250.578C1448.06 250.578 1435.04 254.248 1423.91 261.587C1412.9 268.807 1404.38 278.277 1398.34 289.996C1392.42 301.596 1389.46 314.203 1389.46 327.815V358.888C1389.46 377.117 1392.66 392.624 1399.05 405.408C1405.56 418.192 1414.62 427.957 1426.22 434.705C1437.82 441.333 1451.37 444.648 1466.88 444.648C1476.94 444.648 1486.11 443.227 1494.4 440.386C1502.69 437.427 1509.85 433.047 1515.89 427.247C1521.92 421.447 1526.54 414.286 1529.73 405.763L1579.98 414.818C1575.96 429.615 1568.74 442.576 1558.32 453.703C1548.02 464.712 1535.06 473.294 1519.44 479.449C1503.93 485.486 1486.23 488.504 1466.35 488.504ZM1752.14 488.504C1725.74 488.504 1703.02 482.527 1683.96 470.571C1665.02 458.497 1650.46 441.866 1640.28 420.678C1630.1 399.489 1625.01 375.223 1625.01 347.879C1625.01 320.18 1630.22 295.737 1640.63 274.548C1651.05 253.241 1665.73 236.61 1684.67 224.655C1703.61 212.699 1725.92 206.722 1751.61 206.722C1772.32 206.722 1790.79 210.569 1807.01 218.263C1823.22 225.839 1836.3 236.492 1846.25 250.223C1856.31 263.954 1862.28 279.993 1864.18 298.341H1812.51C1809.67 285.557 1803.16 274.548 1792.98 265.315C1782.92 256.082 1769.42 251.466 1752.5 251.466C1737.7 251.466 1724.74 255.372 1713.61 263.185C1702.6 270.879 1694.02 281.887 1687.86 296.21C1681.71 310.415 1678.63 327.223 1678.63 346.636C1678.63 366.523 1681.65 383.687 1687.69 398.128C1693.72 412.569 1702.25 423.755 1713.26 431.686C1724.38 439.617 1737.46 443.582 1752.5 443.582C1762.56 443.582 1771.67 441.748 1779.84 438.078C1788.13 434.29 1795.05 428.904 1800.61 421.92C1806.3 414.937 1810.26 406.532 1812.51 396.707H1864.18C1862.28 414.345 1856.54 430.088 1846.96 443.938C1837.37 457.787 1824.52 468.677 1808.43 476.608C1792.45 484.539 1773.68 488.504 1752.14 488.504ZM1915.95 483V210.273H1967.27V253.597H1970.11C1975.08 238.919 1983.84 227.377 1996.39 218.973C2009.05 210.45 2023.37 206.189 2039.35 206.189C2042.67 206.189 2046.58 206.307 2051.07 206.544C2055.69 206.781 2059.3 207.077 2061.9 207.432V258.213C2059.77 257.621 2055.99 256.97 2050.54 256.26C2045.1 255.431 2039.65 255.017 2034.21 255.017C2021.66 255.017 2010.47 257.68 2000.65 263.007C1990.94 268.215 1983.25 275.495 1977.56 284.847C1971.88 294.08 1969.04 304.615 1969.04 316.452V483H1915.95ZM2179.77 489.037C2162.49 489.037 2146.87 485.841 2132.9 479.449C2118.93 472.938 2107.86 463.528 2099.7 451.217C2091.65 438.907 2087.62 423.814 2087.62 405.94C2087.62 390.552 2090.58 377.886 2096.5 367.943C2102.42 358 2110.41 350.128 2120.47 344.328C2130.53 338.528 2141.78 334.148 2154.21 331.189C2166.63 328.23 2179.3 325.981 2192.2 324.442C2208.54 322.548 2221.8 321.009 2231.98 319.825C2242.16 318.523 2249.55 316.452 2254.17 313.611C2258.79 310.77 2261.09 306.153 2261.09 299.761V298.518C2261.09 283.012 2256.71 270.997 2247.96 262.474C2239.31 253.952 2226.41 249.69 2209.25 249.69C2191.37 249.69 2177.29 253.656 2166.99 261.587C2156.81 269.399 2149.77 278.099 2145.86 287.688L2095.97 276.324C2101.89 259.752 2110.53 246.376 2121.89 236.196C2133.37 225.898 2146.57 218.44 2161.49 213.824C2176.4 209.089 2192.08 206.722 2208.54 206.722C2219.43 206.722 2230.97 208.024 2243.16 210.628C2255.47 213.114 2266.95 217.73 2277.61 224.477C2288.38 231.224 2297.2 240.872 2304.06 253.419C2310.93 265.848 2314.36 282.006 2314.36 301.892V483H2262.52V445.713H2260.38C2256.95 452.579 2251.8 459.326 2244.94 465.955C2238.07 472.583 2229.25 478.088 2218.48 482.467C2207.71 486.847 2194.81 489.037 2179.77 489.037ZM2191.31 446.423C2205.99 446.423 2218.54 443.523 2228.96 437.723C2239.49 431.923 2247.48 424.347 2252.93 414.996C2258.49 405.526 2261.27 395.405 2261.27 384.634V349.477C2259.38 351.371 2255.71 353.147 2250.26 354.804C2244.94 356.343 2238.84 357.704 2231.98 358.888C2225.11 359.953 2218.42 360.959 2211.91 361.906C2205.4 362.735 2199.96 363.445 2195.58 364.037C2185.28 365.339 2175.87 367.529 2167.34 370.607C2158.94 373.684 2152.19 378.123 2147.1 383.923C2142.13 389.605 2139.65 397.181 2139.65 406.651C2139.65 419.79 2144.5 429.733 2154.21 436.48C2163.91 443.109 2176.28 446.423 2191.31 446.423ZM2439.2 483L2358.94 210.273H2413.81L2467.25 410.557H2469.92L2523.54 210.273H2578.4L2631.67 409.669H2634.34L2687.42 210.273H2742.29L2662.21 483H2608.06L2552.66 286.089H2548.58L2493.18 483H2439.2ZM2840.76 119.364V483H2787.67V119.364H2840.76Z" fill="#36322F"/>
+</svg>

data/lib/multiwoven/integrations/source/generic_open_ai/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Generic OpenAI Spec Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/generic_open_ai-endpoint",
     "github_issue_label": "source-generic-open-ai-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/google_vertex_model/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Google Vertex",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/google_vertex-model",
     "github_issue_label": "source-vertex",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/http_model/config/meta.json

@@ -4,6 +4,7 @@
     "title": "HTTP Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/http-model-endpoint",
     "github_issue_label": "source-http-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/intuit_quick_books/config/meta.json

@@ -4,6 +4,7 @@
       "title": "Intuit QuickBooks",
       "connector_type": "source",
       "category": "Data Warehouse",
+      "sub_category": "Relational Database",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/intuit_quickbooks",
       "github_issue_label": "source-intuit-quickbooks",
       "icon": "icon.svg",

data/lib/multiwoven/integrations/source/maria_db/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Maria DB",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/maria_db",
     "github_issue_label": "source-maria-db",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/open_ai/config/meta.json

@@ -4,6 +4,7 @@
     "title": "OpenAI Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "LLM",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/open_ai-model",
     "github_issue_label": "source-open-ai-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/oracle_db/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Oracle DB",
     "connector_type": "source",
     "category": "Database",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/oracle",
     "github_issue_label": "source-oracle",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/pinecone_db/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Pinecone DB",
     "connector_type": "source",
     "category": "Database",
+    "sub_category": "Vector Database",
     "documentation_url": "https://docs.squared.ai/activation/vector_search/pinecone_db",
     "github_issue_label": "source-pinecone-db",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/postgresql/config/meta.json

@@ -4,6 +4,7 @@
     "title": "PostgreSQL",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/postgresql",
     "github_issue_label": "source-postgresql",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/qdrant/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Qdrant",
     "connector_type": "source",
     "category": "Database",
+    "sub_category": "Vector Database",
     "documentation_url": "https://docs.squared.ai/activation/vector_search/qdrant",
     "github_issue_label": "source-qdrant",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/redshift/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Amazon Redshift",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/redshift",
     "github_issue_label": "source-redshift",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/meta.json

@@ -6,6 +6,7 @@
     "connector_sub_type": "api",
     "direct_query_support": true,
     "category": "Retail",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/salesforce-consumer-goods-cloud",
     "github_issue_label": "source-salesforce-consumer-goods",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/sftp/config/meta.json

@@ -4,6 +4,7 @@
     "title": "SFTP",
     "connector_type": "source",
     "category": "File Storage",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sftp",
     "github_issue_label": "source-sftp",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/snowflake/config/meta.json

@@ -4,6 +4,7 @@
     "title": "Snowflake",
     "connector_type": "source",
     "category": "Data Warehouse",
+    "sub_category": "Relational Database",
     "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/snowflake",
     "github_issue_label": "source-snowflake",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/watsonx_ai/config/meta.json

@@ -4,6 +4,7 @@
     "title": "WatsonX AI Model Endpoint",
     "connector_type": "source",
     "category": "AI Model",
+    "sub_category": "AI_ML Service",
     "documentation_url": "https://docs.squared.ai/activation/ai-ml-sources/watsonx_ai-model",
     "github_issue_label": "source-watsonx-ai-model",
     "icon": "icon.svg",

data/lib/multiwoven/integrations/source/watsonx_data/config/meta.json

@@ -4,6 +4,7 @@
       "title": "WatsonX Data Endpoint",
       "connector_type": "source",
       "category": "Data Warehouse",
+      "sub_category": "Relational Database",
       "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/watsonx_data",
       "github_issue_label": "source-watsonx-data-endpoint",
       "icon": "icon.svg",

data/lib/multiwoven/integrations.rb

@@ -40,6 +40,7 @@ require "MailchimpMarketing"
 require "aws-sdk-bedrockruntime"
 require "pinecone"
 require "intuit-oauth"
+require "nokogiri"
 
 # Service
 require_relative "integrations/config"
@@ -88,6 +89,7 @@ require_relative "integrations/source/generic_open_ai/client"
 require_relative "integrations/source/intuit_quick_books/client"
 require_relative "integrations/source/pinecone_db/client"
 require_relative "integrations/source/qdrant/client"
+require_relative "integrations/source/firecrawl/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.29.1
+  version: 0.30.1
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-06-17 00:00:00.000000000 Z
+date: 2025-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -751,6 +751,10 @@ files:
 - lib/multiwoven/integrations/source/databrics_model/config/meta.json
 - lib/multiwoven/integrations/source/databrics_model/config/spec.json
 - lib/multiwoven/integrations/source/databrics_model/icon.svg
+- lib/multiwoven/integrations/source/firecrawl/client.rb
+- lib/multiwoven/integrations/source/firecrawl/config/meta.json
+- lib/multiwoven/integrations/source/firecrawl/config/spec.json
+- lib/multiwoven/integrations/source/firecrawl/icon.svg
 - lib/multiwoven/integrations/source/generic_open_ai/client.rb
 - lib/multiwoven/integrations/source/generic_open_ai/config/catalog.json
 - lib/multiwoven/integrations/source/generic_open_ai/config/meta.json