multiwoven-integrations 0.30.1 → 0.30.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77b60f06178a034c3b84ac94d582f500a25f2e987c6a3b813b8220e52bf2958c
-  data.tar.gz: abd9af3678499e495b4f6b6650db793aa9a29a1a7781ff84caad07d1826e1b88
+  metadata.gz: 51616b2e7b06336873aceb0c9e5731526dd7e57ba332eab190bc215d99c4ed8d
+  data.tar.gz: d376ae2826566eea31fd92dbd4e4ddf16034e1ef9085edf08f7b554744424583
 SHA512:
-  metadata.gz: f4535cf7f29ee0dec79736d292018fa18c5d0126e7603401f07dec97a10a8b63efb1f616afcf0b230cd2858f8a7979b2e191d6b169d6c2fb41ea06c9bf64eded
-  data.tar.gz: b4a3288ae74b92dfcf729d970db0a2cadd670aee1a5ee83f536bab6c42c07394a587b18a2e48fcf913257d0e6d6e97d6283aecf342c3b43b14ae0e53ac93dd95
+  metadata.gz: fc9778e2da0499ed9bec602005b3b69aadd8e038e251d77acc8407bde99dbf213c1ff0d6f937a832c1d4aa660dbae578de8042caf9c0720fca3ddcff2e0d762a
+  data.tar.gz: a8438815522f82a1bc6db95c6bd29c94a55f182bf459f90c3e01209f54b17e303d939280a63549f6c1eff5da4e57ca1f80e85b98646f6f935995f3a371f7b274

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.30.1"
+    VERSION = "0.30.2"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/firecrawl/client.rb

@@ -18,12 +18,9 @@ module Multiwoven::Integrations::Source
         failure_status(e)
       end
 
-      def discover(connection_config)
-        connection_config = connection_config.with_indifferent_access
-        create_connection(connection_config)
-        response = execute_scrape(FIRECRAWL_SCRAPE_URL)
-        results = JSON.parse(response.body)
-        catalog = Catalog.new(streams: create_streams(results))
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
         catalog.to_multiwoven_message
       rescue StandardError => e
         handle_exception(e, { context: "FIRECRAWL:DISCOVER:EXCEPTION", type: "error" })
@@ -32,9 +29,8 @@ module Multiwoven::Integrations::Source
       def read(sync_config)
         connection_config = sync_config.source.connection_specification
         connection_config = connection_config.with_indifferent_access
-        query = sync_config.model.query
         url = create_connection(connection_config)
-        query(url, query)
+        query(url, nil)
       rescue StandardError => e
         handle_exception(e, {
                            context: "FIRECRAWL:READ:EXCEPTION",
@@ -62,7 +58,16 @@ module Multiwoven::Integrations::Source
         FIRECRAWL_CRAWL_URL
       end
 
-      def query(url, _query)
+      def query(url, query)
+        has_limit = query.match(/LIMIT\s+(\d+)\s*$/i) if query.present?
+        if has_limit.present?
+          if @config["includePaths"]&.any?
+            path = @config["includePaths"].first
+            @config["url"] = URI.join(@config["url"], path).to_s
+          end
+          @config.delete("includePaths")
+          @config[:limit] = has_limit[1].to_i
+        end
         request = execute_crawl(url)
         request = JSON.parse(request.body)
         crawl_url = get_request_url(request)
@@ -89,16 +94,6 @@ module Multiwoven::Integrations::Source
         )
       end
 
-      def execute_scrape(url)
-        send_request(
-          url: url,
-          http_method: HTTP_POST,
-          payload: JSON.parse({ "url": @base_url }.to_json),
-          headers: auth_headers(@api_key),
-          config: {}
-        )
-      end
-
       def crawl_activity
         send_request(
           url: FIRECRAWL_CRAWL_ACTIVE_URL,
@@ -148,37 +143,6 @@ module Multiwoven::Integrations::Source
         end
       end
 
-      def create_streams(records)
-        group_by_table(records).map do |r|
-          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
-        end
-      end
-
-      def group_by_table(response)
-        columns = response["data"].map do |key, value|
-          {
-            column_name: key,
-            data_type: "string",
-            is_nullable: value.nil?
-          }
-        end
-
-        if response["data"]["metadata"]["url"]
-          columns << {
-            column_name: "url",
-            data_type: "string",
-            is_nullable: response["data"]["metadata"]["url"].nil?
-          }
-        end
-
-        [
-          {
-            tablename: "scrape",
-            columns: columns
-          }
-        ]
-      end
-
       def build_url(url, id)
         format(url, id: id)
       end

data/lib/multiwoven/integrations/source/firecrawl/config/catalog.json

@@ -0,0 +1,26 @@
+{
+  "request_rate_limit": 15,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 5,
+  "streams": [
+    {
+      "name": "scrape",
+      "action": "fetch",
+      "json_schema": {
+        "type": "object",
+        "properties": {
+          "markdown": {
+            "type": "string"
+          },
+          "metadata": {
+            "type": "string"
+          },
+          "url": {
+            "type": "string"
+          }
+        }
+      },
+      "supported_sync_modes": ["incremental"]
+    }
+  ]
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.30.1
+  version: 0.30.2
 platform: ruby
 authors:
 - Subin T P
@@ -752,6 +752,7 @@ files:
 - lib/multiwoven/integrations/source/databrics_model/config/spec.json
 - lib/multiwoven/integrations/source/databrics_model/icon.svg
 - lib/multiwoven/integrations/source/firecrawl/client.rb
+- lib/multiwoven/integrations/source/firecrawl/config/catalog.json
 - lib/multiwoven/integrations/source/firecrawl/config/meta.json
 - lib/multiwoven/integrations/source/firecrawl/config/spec.json
 - lib/multiwoven/integrations/source/firecrawl/icon.svg