context.dev 1.2.0 → 1.4.0

data/lib/context_dev/models/web_web_crawl_md_params.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module ContextDev
+  module Models
+    # @see ContextDev::Resources::Web#web_crawl_md
+    class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel
+      extend ContextDev::Internal::Type::RequestParameters::Converter
+      include ContextDev::Internal::Type::RequestParameters
+
+      # @!attribute url
+      #   The starting URL for the crawl (must include http:// or https:// protocol)
+      #
+      #   @return [String]
+      required :url, String
+
+      # @!attribute follow_subdomains
+      #   When true, follow links on subdomains of the starting URL's domain (e.g.
+      #   docs.example.com when starting from example.com). www and apex are always
+      #   treated as equivalent.
+      #
+      #   @return [Boolean, nil]
+      optional :follow_subdomains, ContextDev::Internal::Type::Boolean, api_name: :followSubdomains
+
+      # @!attribute include_images
+      #   Include image references in the Markdown output
+      #
+      #   @return [Boolean, nil]
+      optional :include_images, ContextDev::Internal::Type::Boolean, api_name: :includeImages
+
+      # @!attribute include_links
+      #   Preserve hyperlinks in the Markdown output
+      #
+      #   @return [Boolean, nil]
+      optional :include_links, ContextDev::Internal::Type::Boolean, api_name: :includeLinks
+
+      # @!attribute max_depth
+      #   Maximum link depth from the starting URL (0 = only the starting page)
+      #
+      #   @return [Integer, nil]
+      optional :max_depth, Integer, api_name: :maxDepth
+
+      # @!attribute max_pages
+      #   Maximum number of pages to crawl. Hard cap: 500.
+      #
+      #   @return [Integer, nil]
+      optional :max_pages, Integer, api_name: :maxPages
+
+      # @!attribute shorten_base64_images
+      #   Truncate base64-encoded image data in the Markdown output
+      #
+      #   @return [Boolean, nil]
+      optional :shorten_base64_images, ContextDev::Internal::Type::Boolean, api_name: :shortenBase64Images
+
+      # @!attribute url_regex
+      #   Regex pattern. Only URLs matching this pattern will be followed and scraped.
+      #
+      #   @return [String, nil]
+      optional :url_regex, String, api_name: :urlRegex
+
+      # @!attribute use_main_content_only
+      #   Extract only the main content, stripping headers, footers, sidebars, and
+      #   navigation
+      #
+      #   @return [Boolean, nil]
+      optional :use_main_content_only, ContextDev::Internal::Type::Boolean, api_name: :useMainContentOnly
+
+      # @!method initialize(url:, follow_subdomains: nil, include_images: nil, include_links: nil, max_depth: nil, max_pages: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
+      #   Some parameter documentations has been truncated, see
+      #   {ContextDev::Models::WebWebCrawlMdParams} for more details.
+      #
+      #   @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
+      #
+      #   @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex
+      #
+      #   @param include_images [Boolean] Include image references in the Markdown output
+      #
+      #   @param include_links [Boolean] Preserve hyperlinks in the Markdown output
+      #
+      #   @param max_depth [Integer] Maximum link depth from the starting URL (0 = only the starting page)
+      #
+      #   @param max_pages [Integer] Maximum number of pages to crawl. Hard cap: 500.
+      #
+      #   @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
+      #
+      #   @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
+      #
+      #   @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
+      #
+      #   @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
+    end
+  end
+end

data/lib/context_dev/models/web_web_crawl_md_response.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module ContextDev
+  module Models
+    # @see ContextDev::Resources::Web#web_crawl_md
+    class WebWebCrawlMdResponse < ContextDev::Internal::Type::BaseModel
+      # @!attribute metadata
+      #
+      #   @return [ContextDev::Models::WebWebCrawlMdResponse::Metadata]
+      required :metadata, -> { ContextDev::Models::WebWebCrawlMdResponse::Metadata }
+
+      # @!attribute results
+      #
+      #   @return [Array<ContextDev::Models::WebWebCrawlMdResponse::Result>]
+      required :results,
+               -> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::WebWebCrawlMdResponse::Result] }
+
+      # @!method initialize(metadata:, results:)
+      #   @param metadata [ContextDev::Models::WebWebCrawlMdResponse::Metadata]
+      #   @param results [Array<ContextDev::Models::WebWebCrawlMdResponse::Result>]
+
+      # @see ContextDev::Models::WebWebCrawlMdResponse#metadata
+      class Metadata < ContextDev::Internal::Type::BaseModel
+        # @!attribute max_crawl_depth
+        #   Maximum crawl depth reached during the crawl
+        #
+        #   @return [Integer]
+        required :max_crawl_depth, Integer, api_name: :maxCrawlDepth
+
+        # @!attribute num_failed
+        #   Number of pages that failed to crawl
+        #
+        #   @return [Integer]
+        required :num_failed, Integer, api_name: :numFailed
+
+        # @!attribute num_succeeded
+        #   Number of pages successfully crawled
+        #
+        #   @return [Integer]
+        required :num_succeeded, Integer, api_name: :numSucceeded
+
+        # @!attribute num_urls
+        #   Total number of URLs crawled
+        #
+        #   @return [Integer]
+        required :num_urls, Integer, api_name: :numUrls
+
+        # @!method initialize(max_crawl_depth:, num_failed:, num_succeeded:, num_urls:)
+        #   @param max_crawl_depth [Integer] Maximum crawl depth reached during the crawl
+        #
+        #   @param num_failed [Integer] Number of pages that failed to crawl
+        #
+        #   @param num_succeeded [Integer] Number of pages successfully crawled
+        #
+        #   @param num_urls [Integer] Total number of URLs crawled
+      end
+
+      class Result < ContextDev::Internal::Type::BaseModel
+        # @!attribute markdown
+        #   Extracted page content as Markdown (empty string on failure)
+        #
+        #   @return [String]
+        required :markdown, String
+
+        # @!attribute metadata
+        #
+        #   @return [ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata]
+        required :metadata, -> { ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata }
+
+        # @!method initialize(markdown:, metadata:)
+        #   @param markdown [String] Extracted page content as Markdown (empty string on failure)
+        #
+        #   @param metadata [ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata]
+
+        # @see ContextDev::Models::WebWebCrawlMdResponse::Result#metadata
+        class Metadata < ContextDev::Internal::Type::BaseModel
+          # @!attribute crawl_depth
+          #   Depth relative to the start URL. 0 = start URL, 1 = one link away.
+          #
+          #   @return [Integer]
+          required :crawl_depth, Integer, api_name: :crawlDepth
+
+          # @!attribute status_code
+          #   HTTP status code of the response
+          #
+          #   @return [Integer]
+          required :status_code, Integer, api_name: :statusCode
+
+          # @!attribute success
+          #   true if the page was fetched and parsed successfully
+          #
+          #   @return [Boolean]
+          required :success, ContextDev::Internal::Type::Boolean
+
+          # @!attribute title
+          #   The page's <title> content (empty string if unavailable)
+          #
+          #   @return [String]
+          required :title, String
+
+          # @!attribute url
+          #   The URL that was fetched
+          #
+          #   @return [String]
+          required :url, String
+
+          # @!method initialize(crawl_depth:, status_code:, success:, title:, url:)
+          #   @param crawl_depth [Integer] Depth relative to the start URL. 0 = start URL, 1 = one link away.
+          #
+          #   @param status_code [Integer] HTTP status code of the response
+          #
+          #   @param success [Boolean] true if the page was fetched and parsed successfully
+          #
+          #   @param title [String] The page's <title> content (empty string if unavailable)
+          #
+          #   @param url [String] The URL that was fetched
+        end
+      end
+    end
+  end
+end

data/lib/context_dev/models.rb

@@ -71,6 +71,8 @@ module ContextDev
 
   WebScreenshotParams = ContextDev::Models::WebScreenshotParams
 
+  WebWebCrawlMdParams = ContextDev::Models::WebWebCrawlMdParams
+
   WebWebScrapeHTMLParams = ContextDev::Models::WebWebScrapeHTMLParams
 
   WebWebScrapeImagesParams = ContextDev::Models::WebWebScrapeImagesParams

data/lib/context_dev/resources/brand.rb

@@ -13,7 +13,7 @@ module ContextDev
       #
       # @param domain [String] Domain name to retrieve brand data for (e.g., 'example.com', 'google.com'). Cann
       #
-      # @param force_language [Symbol, ContextDev::Models::BrandRetrieveParams::ForceLanguage] Optional parameter to force the language of the retrieved brand data. Works with
+      # @param force_language [Symbol, ContextDev::Models::BrandRetrieveParams::ForceLanguage] Optional parameter to force the language of the retrieved brand data.
       #
       # @param max_speed [Boolean] Optional parameter to optimize the API call for maximum speed. When set to true,
       #

data/lib/context_dev/resources/web.rb

@@ -8,10 +8,13 @@ module ContextDev
       #
       # Capture a screenshot of a website. Supports both viewport (standard browser
       # view) and full-page screenshots. Can also screenshot specific page types (login,
-      # pricing, etc.) by using heuristics to find the appropriate URL. Returns a URL to
-      # the uploaded screenshot image hosted on our CDN.
+      # pricing, etc.) by using heuristics to find the appropriate URL. Either 'domain'
+      # or 'directUrl' must be provided as a query parameter, but not both. Returns a
+      # URL to the uploaded screenshot image hosted on our CDN.
       #
-      # @overload screenshot(domain:, full_screenshot: nil, page: nil, prioritize: nil, request_options: {})
+      # @overload screenshot(direct_url: nil, domain: nil, full_screenshot: nil, page: nil, prioritize: nil, request_options: {})
+      #
+      # @param direct_url [String] A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https
       #
       # @param domain [String] Domain name to take screenshot of (e.g., 'example.com', 'google.com'). The domai
       #
@@ -26,18 +29,61 @@ module ContextDev
       # @return [ContextDev::Models::WebScreenshotResponse]
       #
       # @see ContextDev::Models::WebScreenshotParams
-      def screenshot(params)
+      def screenshot(params = {})
         parsed, options = ContextDev::WebScreenshotParams.dump_request(params)
         query = ContextDev::Internal::Util.encode_query_params(parsed)
         @client.request(
           method: :get,
           path: "brand/screenshot",
-          query: query.transform_keys(full_screenshot: "fullScreenshot"),
+          query: query.transform_keys(direct_url: "directUrl", full_screenshot: "fullScreenshot"),
           model: ContextDev::Models::WebScreenshotResponse,
           options: options
         )
       end
 
+      # Some parameter documentations has been truncated, see
+      # {ContextDev::Models::WebWebCrawlMdParams} for more details.
+      #
+      # Performs a crawl starting from a given URL, extracts page content as Markdown,
+      # and returns results for all crawled pages. Only follows links within the same
+      # domain as the starting URL. Costs 1 credit per successful page crawled.
+      #
+      # @overload web_crawl_md(url:, follow_subdomains: nil, include_images: nil, include_links: nil, max_depth: nil, max_pages: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
+      #
+      # @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
+      #
+      # @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex
+      #
+      # @param include_images [Boolean] Include image references in the Markdown output
+      #
+      # @param include_links [Boolean] Preserve hyperlinks in the Markdown output
+      #
+      # @param max_depth [Integer] Maximum link depth from the starting URL (0 = only the starting page)
+      #
+      # @param max_pages [Integer] Maximum number of pages to crawl. Hard cap: 500.
+      #
+      # @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
+      #
+      # @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
+      #
+      # @param use_main_content_only [Boolean] Extract only the main content, stripping headers, footers, sidebars, and navigat
+      #
+      # @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
+      #
+      # @return [ContextDev::Models::WebWebCrawlMdResponse]
+      #
+      # @see ContextDev::Models::WebWebCrawlMdParams
+      def web_crawl_md(params)
+        parsed, options = ContextDev::WebWebCrawlMdParams.dump_request(params)
+        @client.request(
+          method: :post,
+          path: "web/crawl",
+          body: parsed,
+          model: ContextDev::Models::WebWebCrawlMdResponse,
+          options: options
+        )
+      end
+
       # Scrapes the given URL and returns the raw HTML content of the page.
       #
       # @overload web_scrape_html(url:, request_options: {})

data/lib/context_dev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ContextDev
-  VERSION = "1.2.0"
+  VERSION = "1.4.0"
 end

data/lib/context_dev.rb

@@ -84,6 +84,8 @@ require_relative "context_dev/models/utility_prefetch_params"
 require_relative "context_dev/models/utility_prefetch_response"
 require_relative "context_dev/models/web_screenshot_params"
 require_relative "context_dev/models/web_screenshot_response"
+require_relative "context_dev/models/web_web_crawl_md_params"
+require_relative "context_dev/models/web_web_crawl_md_response"
 require_relative "context_dev/models/web_web_scrape_html_params"
 require_relative "context_dev/models/web_web_scrape_html_response"
 require_relative "context_dev/models/web_web_scrape_images_params"