multiwoven-integrations 0.30.1 → 0.30.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77b60f06178a034c3b84ac94d582f500a25f2e987c6a3b813b8220e52bf2958c
-  data.tar.gz: abd9af3678499e495b4f6b6650db793aa9a29a1a7781ff84caad07d1826e1b88
+  metadata.gz: 9fd3ed70334fa8ca4cf73fa9c6e47f253e372d9b9358ee3a1004de5880e4656b
+  data.tar.gz: 25a4d2b8442bd87197f84cc7c36fe3bbb7864cc0f329312fba37d9742cbc8bee
 SHA512:
-  metadata.gz: f4535cf7f29ee0dec79736d292018fa18c5d0126e7603401f07dec97a10a8b63efb1f616afcf0b230cd2858f8a7979b2e191d6b169d6c2fb41ea06c9bf64eded
-  data.tar.gz: b4a3288ae74b92dfcf729d970db0a2cadd670aee1a5ee83f536bab6c42c07394a587b18a2e48fcf913257d0e6d6e97d6283aecf342c3b43b14ae0e53ac93dd95
+  metadata.gz: '08384a20d53b8b87472c509aedfaf37d68b3445a722686ba49b7fd86a4ef044eeeae7a60fa6cc1191f475b0c221ee6937a01ed635b5df784c1f5c494a9828cf5'
+  data.tar.gz: '085bb3fdb255ae339486d00e4747c9e97d5784e5b344dac3e5870c99229dfa71693e8d11831d9661cade3a879a42998e7c650638585be9c152e1b5a4d326702a'

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.30.1"
+    VERSION = "0.30.3"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/firecrawl/client.rb

@@ -18,12 +18,9 @@ module Multiwoven::Integrations::Source
         failure_status(e)
       end
 
-      def discover(connection_config)
-        connection_config = connection_config.with_indifferent_access
-        create_connection(connection_config)
-        response = execute_scrape(FIRECRAWL_SCRAPE_URL)
-        results = JSON.parse(response.body)
-        catalog = Catalog.new(streams: create_streams(results))
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
         catalog.to_multiwoven_message
       rescue StandardError => e
         handle_exception(e, { context: "FIRECRAWL:DISCOVER:EXCEPTION", type: "error" })
@@ -32,9 +29,8 @@ module Multiwoven::Integrations::Source
       def read(sync_config)
         connection_config = sync_config.source.connection_specification
         connection_config = connection_config.with_indifferent_access
-        query = sync_config.model.query
         url = create_connection(connection_config)
-        query(url, query)
+        query(url, nil, nil)
       rescue StandardError => e
         handle_exception(e, {
                            context: "FIRECRAWL:READ:EXCEPTION",
@@ -62,7 +58,15 @@ module Multiwoven::Integrations::Source
         FIRECRAWL_CRAWL_URL
       end
 
-      def query(url, _query)
+      def query(url, _query, limit = 1)
+        if limit.present?
+          if @config["includePaths"]&.any?
+            path = @config["includePaths"].first
+            @config["url"] = URI.join(@config["url"], path).to_s
+          end
+          @config.delete("includePaths")
+          @config[:limit] = limit
+        end
         request = execute_crawl(url)
         request = JSON.parse(request.body)
         crawl_url = get_request_url(request)
@@ -89,16 +93,6 @@ module Multiwoven::Integrations::Source
         )
       end
 
-      def execute_scrape(url)
-        send_request(
-          url: url,
-          http_method: HTTP_POST,
-          payload: JSON.parse({ "url": @base_url }.to_json),
-          headers: auth_headers(@api_key),
-          config: {}
-        )
-      end
-
       def crawl_activity
         send_request(
           url: FIRECRAWL_CRAWL_ACTIVE_URL,
@@ -148,37 +142,6 @@ module Multiwoven::Integrations::Source
         end
       end
 
-      def create_streams(records)
-        group_by_table(records).map do |r|
-          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
-        end
-      end
-
-      def group_by_table(response)
-        columns = response["data"].map do |key, value|
-          {
-            column_name: key,
-            data_type: "string",
-            is_nullable: value.nil?
-          }
-        end
-
-        if response["data"]["metadata"]["url"]
-          columns << {
-            column_name: "url",
-            data_type: "string",
-            is_nullable: response["data"]["metadata"]["url"].nil?
-          }
-        end
-
-        [
-          {
-            tablename: "scrape",
-            columns: columns
-          }
-        ]
-      end
-
       def build_url(url, id)
         format(url, id: id)
       end

data/lib/multiwoven/integrations/source/firecrawl/config/catalog.json

@@ -0,0 +1,26 @@
+{
+  "request_rate_limit": 15,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 5,
+  "streams": [
+    {
+      "name": "scrape",
+      "action": "fetch",
+      "json_schema": {
+        "type": "object",
+        "properties": {
+          "markdown": {
+            "type": "string"
+          },
+          "metadata": {
+            "type": "string"
+          },
+          "url": {
+            "type": "string"
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"]
+    }
+  ]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.30.1
+  version: 0.30.3
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-06-27 00:00:00.000000000 Z
+date: 2025-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -752,6 +752,7 @@ files:
 - lib/multiwoven/integrations/source/databrics_model/config/spec.json
 - lib/multiwoven/integrations/source/databrics_model/icon.svg
 - lib/multiwoven/integrations/source/firecrawl/client.rb
+- lib/multiwoven/integrations/source/firecrawl/config/catalog.json
 - lib/multiwoven/integrations/source/firecrawl/config/meta.json
 - lib/multiwoven/integrations/source/firecrawl/config/spec.json
 - lib/multiwoven/integrations/source/firecrawl/icon.svg