context.dev 1.1.0 → 1.3.0

data/lib/context_dev/models/web_web_crawl_md_params.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module ContextDev
+  module Models
+    # @see ContextDev::Resources::Web#web_crawl_md
+    class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel
+      extend ContextDev::Internal::Type::RequestParameters::Converter
+      include ContextDev::Internal::Type::RequestParameters
+
+      # @!attribute url
+      #   The starting URL for the crawl (must include http:// or https:// protocol)
+      #
+      #   @return [String]
+      required :url, String
+
+      # @!attribute follow_subdomains
+      #   When true, follow links on subdomains of the starting URL's domain (e.g.
+      #   docs.example.com when starting from example.com). www and apex are always
+      #   treated as equivalent.
+      #
+      #   @return [Boolean, nil]
+      optional :follow_subdomains, ContextDev::Internal::Type::Boolean, api_name: :followSubdomains
+
+      # @!attribute include_images
+      #   Include image references in the Markdown output
+      #
+      #   @return [Boolean, nil]
+      optional :include_images, ContextDev::Internal::Type::Boolean, api_name: :includeImages
+
+      # @!attribute include_links
+      #   Preserve hyperlinks in the Markdown output
+      #
+      #   @return [Boolean, nil]
+      optional :include_links, ContextDev::Internal::Type::Boolean, api_name: :includeLinks
+
+      # @!attribute max_depth
+      #   Maximum link depth from the starting URL (0 = only the starting page)
+      #
+      #   @return [Integer, nil]
+      optional :max_depth, Integer, api_name: :maxDepth
+
+      # @!attribute max_pages
+      #   Maximum number of pages to crawl. Hard cap: 500.
+      #
+      #   @return [Integer, nil]
+      optional :max_pages, Integer, api_name: :maxPages
+
+      # @!attribute shorten_base64_images
+      #   Truncate base64-encoded image data in the Markdown output
+      #
+      #   @return [Boolean, nil]
+      optional :shorten_base64_images, ContextDev::Internal::Type::Boolean, api_name: :shortenBase64Images
+
+      # @!attribute url_regex
+      #   Regex pattern. Only URLs matching this pattern will be followed and scraped.
+      #
+      #   @return [String, nil]
+      optional :url_regex, String, api_name: :urlRegex
+
+      # @!attribute use_main_content_only
+      #   Extract only the main content, stripping headers, footers, sidebars, and
+      #   navigation
+      #
+      #   @return [Boolean, nil]
+      optional :use_main_content_only, ContextDev::Internal::Type::Boolean, api_name: :useMainContentOnly
+
+      # @!method initialize(url:, follow_subdomains: nil, include_images: nil, include_links: nil, max_depth: nil, max_pages: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
+      #   Some parameter documentations has been truncated, see
+      #   {ContextDev::Models::WebWebCrawlMdParams} for more details.
+      #
+      #   @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
+      #
+      #   @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex
+      #
+      #   @param include_images [Boolean] Include image references in the Markdown output
+      #
+      #   @param include_links [Boolean] Preserve hyperlinks in the Markdown output
+      #
+      #   @param max_depth [Integer] Maximum link depth from the starting URL (0 = only the starting page)
+      #
+      #   @param max_pages [Integer] Maximum number of pages to crawl. Hard cap: 500.
+      #
+      #   @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
+      #
+      #   @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
+      #
+      #   @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
+      #
+      #   @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
+    end
+  end
+end

data/lib/context_dev/models/web_web_crawl_md_response.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module ContextDev
+  module Models
+    # @see ContextDev::Resources::Web#web_crawl_md
+    class WebWebCrawlMdResponse < ContextDev::Internal::Type::BaseModel
+      # @!attribute metadata
+      #
+      #   @return [ContextDev::Models::WebWebCrawlMdResponse::Metadata]
+      required :metadata, -> { ContextDev::Models::WebWebCrawlMdResponse::Metadata }
+
+      # @!attribute results
+      #
+      #   @return [Array<ContextDev::Models::WebWebCrawlMdResponse::Result>]
+      required :results,
+               -> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::WebWebCrawlMdResponse::Result] }
+
+      # @!method initialize(metadata:, results:)
+      #   @param metadata [ContextDev::Models::WebWebCrawlMdResponse::Metadata]
+      #   @param results [Array<ContextDev::Models::WebWebCrawlMdResponse::Result>]
+
+      # @see ContextDev::Models::WebWebCrawlMdResponse#metadata
+      class Metadata < ContextDev::Internal::Type::BaseModel
+        # @!attribute max_crawl_depth
+        #   Maximum crawl depth reached during the crawl
+        #
+        #   @return [Integer]
+        required :max_crawl_depth, Integer, api_name: :maxCrawlDepth
+
+        # @!attribute num_failed
+        #   Number of pages that failed to crawl
+        #
+        #   @return [Integer]
+        required :num_failed, Integer, api_name: :numFailed
+
+        # @!attribute num_succeeded
+        #   Number of pages successfully crawled
+        #
+        #   @return [Integer]
+        required :num_succeeded, Integer, api_name: :numSucceeded
+
+        # @!attribute num_urls
+        #   Total number of URLs crawled
+        #
+        #   @return [Integer]
+        required :num_urls, Integer, api_name: :numUrls
+
+        # @!method initialize(max_crawl_depth:, num_failed:, num_succeeded:, num_urls:)
+        #   @param max_crawl_depth [Integer] Maximum crawl depth reached during the crawl
+        #
+        #   @param num_failed [Integer] Number of pages that failed to crawl
+        #
+        #   @param num_succeeded [Integer] Number of pages successfully crawled
+        #
+        #   @param num_urls [Integer] Total number of URLs crawled
+      end
+
+      class Result < ContextDev::Internal::Type::BaseModel
+        # @!attribute markdown
+        #   Extracted page content as Markdown (empty string on failure)
+        #
+        #   @return [String]
+        required :markdown, String
+
+        # @!attribute metadata
+        #
+        #   @return [ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata]
+        required :metadata, -> { ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata }
+
+        # @!method initialize(markdown:, metadata:)
+        #   @param markdown [String] Extracted page content as Markdown (empty string on failure)
+        #
+        #   @param metadata [ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata]
+
+        # @see ContextDev::Models::WebWebCrawlMdResponse::Result#metadata
+        class Metadata < ContextDev::Internal::Type::BaseModel
+          # @!attribute crawl_depth
+          #   Depth relative to the start URL. 0 = start URL, 1 = one link away.
+          #
+          #   @return [Integer]
+          required :crawl_depth, Integer, api_name: :crawlDepth
+
+          # @!attribute status_code
+          #   HTTP status code of the response
+          #
+          #   @return [Integer]
+          required :status_code, Integer, api_name: :statusCode
+
+          # @!attribute success
+          #   true if the page was fetched and parsed successfully
+          #
+          #   @return [Boolean]
+          required :success, ContextDev::Internal::Type::Boolean
+
+          # @!attribute title
+          #   The page's <title> content (empty string if unavailable)
+          #
+          #   @return [String]
+          required :title, String
+
+          # @!attribute url
+          #   The URL that was fetched
+          #
+          #   @return [String]
+          required :url, String
+
+          # @!method initialize(crawl_depth:, status_code:, success:, title:, url:)
+          #   @param crawl_depth [Integer] Depth relative to the start URL. 0 = start URL, 1 = one link away.
+          #
+          #   @param status_code [Integer] HTTP status code of the response
+          #
+          #   @param success [Boolean] true if the page was fetched and parsed successfully
+          #
+          #   @param title [String] The page's <title> content (empty string if unavailable)
+          #
+          #   @param url [String] The URL that was fetched
+        end
+      end
+    end
+  end
+end

data/lib/context_dev/models.rb

@@ -71,6 +71,8 @@ module ContextDev
 
   WebScreenshotParams = ContextDev::Models::WebScreenshotParams
 
+  WebWebCrawlMdParams = ContextDev::Models::WebWebCrawlMdParams
+
   WebWebScrapeHTMLParams = ContextDev::Models::WebWebScrapeHTMLParams
 
   WebWebScrapeImagesParams = ContextDev::Models::WebWebScrapeImagesParams

data/lib/context_dev/resources/style.rb

@@ -40,14 +40,12 @@ module ContextDev
       # Either 'domain' or 'directUrl' must be provided as a query parameter, but not
       # both.
       #
-      # @overload extract_styleguide(direct_url: nil, domain: nil, prioritize: nil, timeout_ms: nil, request_options: {})
+      # @overload extract_styleguide(direct_url: nil, domain: nil, timeout_ms: nil, request_options: {})
       #
       # @param direct_url [String] A specific URL to fetch the styleguide from directly, bypassing domain resolutio
       #
       # @param domain [String] Domain name to extract styleguide from (e.g., 'example.com', 'google.com'). The
       #
-      # @param prioritize [Symbol, ContextDev::Models::StyleExtractStyleguideParams::Prioritize] Optional parameter to prioritize screenshot capture for styleguide extraction. I
-      #
       # @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
       #
       # @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]

data/lib/context_dev/resources/web.rb

@@ -38,6 +38,49 @@ module ContextDev
         )
       end
 
+      # Some parameter documentations has been truncated, see
+      # {ContextDev::Models::WebWebCrawlMdParams} for more details.
+      #
+      # Performs a crawl starting from a given URL, extracts page content as Markdown,
+      # and returns results for all crawled pages. Only follows links within the same
+      # domain as the starting URL. Costs 1 credit per successful page crawled.
+      #
+      # @overload web_crawl_md(url:, follow_subdomains: nil, include_images: nil, include_links: nil, max_depth: nil, max_pages: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
+      #
+      # @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
+      #
+      # @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex
+      #
+      # @param include_images [Boolean] Include image references in the Markdown output
+      #
+      # @param include_links [Boolean] Preserve hyperlinks in the Markdown output
+      #
+      # @param max_depth [Integer] Maximum link depth from the starting URL (0 = only the starting page)
+      #
+      # @param max_pages [Integer] Maximum number of pages to crawl. Hard cap: 500.
+      #
+      # @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
+      #
+      # @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
+      #
+      # @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
+      #
+      # @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
+      #
+      # @return [ContextDev::Models::WebWebCrawlMdResponse]
+      #
+      # @see ContextDev::Models::WebWebCrawlMdParams
+      def web_crawl_md(params)
+        parsed, options = ContextDev::WebWebCrawlMdParams.dump_request(params)
+        @client.request(
+          method: :post,
+          path: "web/crawl",
+          body: parsed,
+          model: ContextDev::Models::WebWebCrawlMdResponse,
+          options: options
+        )
+      end
+
       # Scrapes the given URL and returns the raw HTML content of the page.
       #
       # @overload web_scrape_html(url:, request_options: {})

data/lib/context_dev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ContextDev
-  VERSION = "1.1.0"
+  VERSION = "1.3.0"
 end

data/lib/context_dev.rb

@@ -84,6 +84,8 @@ require_relative "context_dev/models/utility_prefetch_params"
 require_relative "context_dev/models/utility_prefetch_response"
 require_relative "context_dev/models/web_screenshot_params"
 require_relative "context_dev/models/web_screenshot_response"
+require_relative "context_dev/models/web_web_crawl_md_params"
+require_relative "context_dev/models/web_web_crawl_md_response"
 require_relative "context_dev/models/web_web_scrape_html_params"
 require_relative "context_dev/models/web_web_scrape_html_response"
 require_relative "context_dev/models/web_web_scrape_images_params"

data/rbi/context_dev/internal/util.rbi

@@ -148,12 +148,20 @@ module ContextDev
         end
       end
 
+      # https://www.rfc-editor.org/rfc/rfc3986.html#section-3.3
+      RFC_3986_NOT_PCHARS = T.let(/[^A-Za-z0-9\-._~!$&'()*+,;=:@]+/, Regexp)
+
       class << self
         # @api private
         sig { params(uri: URI::Generic).returns(String) }
         def uri_origin(uri)
         end
 
+        # @api private
+        sig { params(path: T.any(String, Integer)).returns(String) }
+        def encode_path(path)
+        end
+
         # @api private
         sig { params(path: T.any(String, T::Array[String])).returns(String) }
         def interpolate_path(path)

data/rbi/context_dev/models/style_extract_styleguide_params.rbi

@@ -30,27 +30,6 @@ module ContextDev
       sig { params(domain: String).void }
       attr_writer :domain
 
-      # Optional parameter to prioritize screenshot capture for styleguide extraction.
-      # If 'speed', optimizes for faster capture with basic quality. If 'quality',
-      # optimizes for higher quality with longer wait times. Defaults to 'quality' if
-      # not provided.
-      sig do
-        returns(
-          T.nilable(
-            ContextDev::StyleExtractStyleguideParams::Prioritize::OrSymbol
-          )
-        )
-      end
-      attr_reader :prioritize
-
-      sig do
-        params(
-          prioritize:
-            ContextDev::StyleExtractStyleguideParams::Prioritize::OrSymbol
-        ).void
-      end
-      attr_writer :prioritize
-
       # Optional timeout in milliseconds for the request. If the request takes longer
       # than this value, it will be aborted with a 408 status code. Maximum allowed
       # value is 300000ms (5 minutes).
@@ -64,8 +43,6 @@ module ContextDev
         params(
           direct_url: String,
           domain: String,
-          prioritize:
-            ContextDev::StyleExtractStyleguideParams::Prioritize::OrSymbol,
           timeout_ms: Integer,
           request_options: ContextDev::RequestOptions::OrHash
         ).returns(T.attached_class)
@@ -77,11 +54,6 @@ module ContextDev
         # Domain name to extract styleguide from (e.g., 'example.com', 'google.com'). The
         # domain will be automatically normalized and validated.
         domain: nil,
-        # Optional parameter to prioritize screenshot capture for styleguide extraction.
-        # If 'speed', optimizes for faster capture with basic quality. If 'quality',
-        # optimizes for higher quality with longer wait times. Defaults to 'quality' if
-        # not provided.
-        prioritize: nil,
         # Optional timeout in milliseconds for the request. If the request takes longer
         # than this value, it will be aborted with a 408 status code. Maximum allowed
         # value is 300000ms (5 minutes).
@@ -95,8 +67,6 @@ module ContextDev
           {
             direct_url: String,
             domain: String,
-            prioritize:
-              ContextDev::StyleExtractStyleguideParams::Prioritize::OrSymbol,
             timeout_ms: Integer,
             request_options: ContextDev::RequestOptions
           }
@@ -104,41 +74,6 @@ module ContextDev
       end
       def to_hash
       end
-
-      # Optional parameter to prioritize screenshot capture for styleguide extraction.
-      # If 'speed', optimizes for faster capture with basic quality. If 'quality',
-      # optimizes for higher quality with longer wait times. Defaults to 'quality' if
-      # not provided.
-      module Prioritize
-        extend ContextDev::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias do
-            T.all(Symbol, ContextDev::StyleExtractStyleguideParams::Prioritize)
-          end
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        SPEED =
-          T.let(
-            :speed,
-            ContextDev::StyleExtractStyleguideParams::Prioritize::TaggedSymbol
-          )
-        QUALITY =
-          T.let(
-            :quality,
-            ContextDev::StyleExtractStyleguideParams::Prioritize::TaggedSymbol
-          )
-
-        sig do
-          override.returns(
-            T::Array[
-              ContextDev::StyleExtractStyleguideParams::Prioritize::TaggedSymbol
-            ]
-          )
-        end
-        def self.values
-        end
-      end
     end
   end
 end