context.dev 1.1.0 → 1.3.0

data/rbi/context_dev/models/web_web_crawl_md_params.rbi

@@ -0,0 +1,137 @@
+# typed: strong
+
+module ContextDev
+  module Models
+    class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel
+      extend ContextDev::Internal::Type::RequestParameters::Converter
+      include ContextDev::Internal::Type::RequestParameters
+
+      OrHash =
+        T.type_alias do
+          T.any(ContextDev::WebWebCrawlMdParams, ContextDev::Internal::AnyHash)
+        end
+
+      # The starting URL for the crawl (must include http:// or https:// protocol)
+      sig { returns(String) }
+      attr_accessor :url
+
+      # When true, follow links on subdomains of the starting URL's domain (e.g.
+      # docs.example.com when starting from example.com). www and apex are always
+      # treated as equivalent.
+      sig { returns(T.nilable(T::Boolean)) }
+      attr_reader :follow_subdomains
+
+      sig { params(follow_subdomains: T::Boolean).void }
+      attr_writer :follow_subdomains
+
+      # Include image references in the Markdown output
+      sig { returns(T.nilable(T::Boolean)) }
+      attr_reader :include_images
+
+      sig { params(include_images: T::Boolean).void }
+      attr_writer :include_images
+
+      # Preserve hyperlinks in the Markdown output
+      sig { returns(T.nilable(T::Boolean)) }
+      attr_reader :include_links
+
+      sig { params(include_links: T::Boolean).void }
+      attr_writer :include_links
+
+      # Maximum link depth from the starting URL (0 = only the starting page)
+      sig { returns(T.nilable(Integer)) }
+      attr_reader :max_depth
+
+      sig { params(max_depth: Integer).void }
+      attr_writer :max_depth
+
+      # Maximum number of pages to crawl. Hard cap: 500.
+      sig { returns(T.nilable(Integer)) }
+      attr_reader :max_pages
+
+      sig { params(max_pages: Integer).void }
+      attr_writer :max_pages
+
+      # Truncate base64-encoded image data in the Markdown output
+      sig { returns(T.nilable(T::Boolean)) }
+      attr_reader :shorten_base64_images
+
+      sig { params(shorten_base64_images: T::Boolean).void }
+      attr_writer :shorten_base64_images
+
+      # Regex pattern. Only URLs matching this pattern will be followed and scraped.
+      sig { returns(T.nilable(String)) }
+      attr_reader :url_regex
+
+      sig { params(url_regex: String).void }
+      attr_writer :url_regex
+
+      # Extract only the main content, stripping headers, footers, sidebars, and
+      # navigation
+      sig { returns(T.nilable(T::Boolean)) }
+      attr_reader :use_main_content_only
+
+      sig { params(use_main_content_only: T::Boolean).void }
+      attr_writer :use_main_content_only
+
+      sig do
+        params(
+          url: String,
+          follow_subdomains: T::Boolean,
+          include_images: T::Boolean,
+          include_links: T::Boolean,
+          max_depth: Integer,
+          max_pages: Integer,
+          shorten_base64_images: T::Boolean,
+          url_regex: String,
+          use_main_content_only: T::Boolean,
+          request_options: ContextDev::RequestOptions::OrHash
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # The starting URL for the crawl (must include http:// or https:// protocol)
+        url:,
+        # When true, follow links on subdomains of the starting URL's domain (e.g.
+        # docs.example.com when starting from example.com). www and apex are always
+        # treated as equivalent.
+        follow_subdomains: nil,
+        # Include image references in the Markdown output
+        include_images: nil,
+        # Preserve hyperlinks in the Markdown output
+        include_links: nil,
+        # Maximum link depth from the starting URL (0 = only the starting page)
+        max_depth: nil,
+        # Maximum number of pages to crawl. Hard cap: 500.
+        max_pages: nil,
+        # Truncate base64-encoded image data in the Markdown output
+        shorten_base64_images: nil,
+        # Regex pattern. Only URLs matching this pattern will be followed and scraped.
+        url_regex: nil,
+        # Extract only the main content, stripping headers, footers, sidebars, and
+        # navigation
+        use_main_content_only: nil,
+        request_options: {}
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            url: String,
+            follow_subdomains: T::Boolean,
+            include_images: T::Boolean,
+            include_links: T::Boolean,
+            max_depth: Integer,
+            max_pages: Integer,
+            shorten_base64_images: T::Boolean,
+            url_regex: String,
+            use_main_content_only: T::Boolean,
+            request_options: ContextDev::RequestOptions
+          }
+        )
+      end
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/context_dev/models/web_web_crawl_md_response.rbi

@@ -0,0 +1,230 @@
+# typed: strong
+
+module ContextDev
+  module Models
+    class WebWebCrawlMdResponse < ContextDev::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(
+            ContextDev::Models::WebWebCrawlMdResponse,
+            ContextDev::Internal::AnyHash
+          )
+        end
+
+      sig { returns(ContextDev::Models::WebWebCrawlMdResponse::Metadata) }
+      attr_reader :metadata
+
+      sig do
+        params(
+          metadata: ContextDev::Models::WebWebCrawlMdResponse::Metadata::OrHash
+        ).void
+      end
+      attr_writer :metadata
+
+      sig do
+        returns(T::Array[ContextDev::Models::WebWebCrawlMdResponse::Result])
+      end
+      attr_accessor :results
+
+      sig do
+        params(
+          metadata: ContextDev::Models::WebWebCrawlMdResponse::Metadata::OrHash,
+          results:
+            T::Array[ContextDev::Models::WebWebCrawlMdResponse::Result::OrHash]
+        ).returns(T.attached_class)
+      end
+      def self.new(metadata:, results:)
+      end
+
+      sig do
+        override.returns(
+          {
+            metadata: ContextDev::Models::WebWebCrawlMdResponse::Metadata,
+            results: T::Array[ContextDev::Models::WebWebCrawlMdResponse::Result]
+          }
+        )
+      end
+      def to_hash
+      end
+
+      class Metadata < ContextDev::Internal::Type::BaseModel
+        OrHash =
+          T.type_alias do
+            T.any(
+              ContextDev::Models::WebWebCrawlMdResponse::Metadata,
+              ContextDev::Internal::AnyHash
+            )
+          end
+
+        # Maximum crawl depth reached during the crawl
+        sig { returns(Integer) }
+        attr_accessor :max_crawl_depth
+
+        # Number of pages that failed to crawl
+        sig { returns(Integer) }
+        attr_accessor :num_failed
+
+        # Number of pages successfully crawled
+        sig { returns(Integer) }
+        attr_accessor :num_succeeded
+
+        # Total number of URLs crawled
+        sig { returns(Integer) }
+        attr_accessor :num_urls
+
+        sig do
+          params(
+            max_crawl_depth: Integer,
+            num_failed: Integer,
+            num_succeeded: Integer,
+            num_urls: Integer
+          ).returns(T.attached_class)
+        end
+        def self.new(
+          # Maximum crawl depth reached during the crawl
+          max_crawl_depth:,
+          # Number of pages that failed to crawl
+          num_failed:,
+          # Number of pages successfully crawled
+          num_succeeded:,
+          # Total number of URLs crawled
+          num_urls:
+        )
+        end
+
+        sig do
+          override.returns(
+            {
+              max_crawl_depth: Integer,
+              num_failed: Integer,
+              num_succeeded: Integer,
+              num_urls: Integer
+            }
+          )
+        end
+        def to_hash
+        end
+      end
+
+      class Result < ContextDev::Internal::Type::BaseModel
+        OrHash =
+          T.type_alias do
+            T.any(
+              ContextDev::Models::WebWebCrawlMdResponse::Result,
+              ContextDev::Internal::AnyHash
+            )
+          end
+
+        # Extracted page content as Markdown (empty string on failure)
+        sig { returns(String) }
+        attr_accessor :markdown
+
+        sig do
+          returns(ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata)
+        end
+        attr_reader :metadata
+
+        sig do
+          params(
+            metadata:
+              ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::OrHash
+          ).void
+        end
+        attr_writer :metadata
+
+        sig do
+          params(
+            markdown: String,
+            metadata:
+              ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata::OrHash
+          ).returns(T.attached_class)
+        end
+        def self.new(
+          # Extracted page content as Markdown (empty string on failure)
+          markdown:,
+          metadata:
+        )
+        end
+
+        sig do
+          override.returns(
+            {
+              markdown: String,
+              metadata:
+                ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata
+            }
+          )
+        end
+        def to_hash
+        end
+
+        class Metadata < ContextDev::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                ContextDev::Models::WebWebCrawlMdResponse::Result::Metadata,
+                ContextDev::Internal::AnyHash
+              )
+            end
+
+          # Depth relative to the start URL. 0 = start URL, 1 = one link away.
+          sig { returns(Integer) }
+          attr_accessor :crawl_depth
+
+          # HTTP status code of the response
+          sig { returns(Integer) }
+          attr_accessor :status_code
+
+          # true if the page was fetched and parsed successfully
+          sig { returns(T::Boolean) }
+          attr_accessor :success
+
+          # The page's <title> content (empty string if unavailable)
+          sig { returns(String) }
+          attr_accessor :title
+
+          # The URL that was fetched
+          sig { returns(String) }
+          attr_accessor :url
+
+          sig do
+            params(
+              crawl_depth: Integer,
+              status_code: Integer,
+              success: T::Boolean,
+              title: String,
+              url: String
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            # Depth relative to the start URL. 0 = start URL, 1 = one link away.
+            crawl_depth:,
+            # HTTP status code of the response
+            status_code:,
+            # true if the page was fetched and parsed successfully
+            success:,
+            # The page's <title> content (empty string if unavailable)
+            title:,
+            # The URL that was fetched
+            url:
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                crawl_depth: Integer,
+                status_code: Integer,
+                success: T::Boolean,
+                title: String,
+                url: String
+              }
+            )
+          end
+          def to_hash
+          end
+        end
+      end
+    end
+  end
+end

data/rbi/context_dev/models.rbi

@@ -37,6 +37,8 @@ module ContextDev
 
   WebScreenshotParams = ContextDev::Models::WebScreenshotParams
 
+  WebWebCrawlMdParams = ContextDev::Models::WebWebCrawlMdParams
+
   WebWebScrapeHTMLParams = ContextDev::Models::WebWebScrapeHTMLParams
 
   WebWebScrapeImagesParams = ContextDev::Models::WebWebScrapeImagesParams

data/rbi/context_dev/resources/style.rbi

@@ -32,8 +32,6 @@ module ContextDev
         params(
           direct_url: String,
           domain: String,
-          prioritize:
-            ContextDev::StyleExtractStyleguideParams::Prioritize::OrSymbol,
           timeout_ms: Integer,
           request_options: ContextDev::RequestOptions::OrHash
         ).returns(ContextDev::Models::StyleExtractStyleguideResponse)
@@ -45,11 +43,6 @@ module ContextDev
         # Domain name to extract styleguide from (e.g., 'example.com', 'google.com'). The
         # domain will be automatically normalized and validated.
         domain: nil,
-        # Optional parameter to prioritize screenshot capture for styleguide extraction.
-        # If 'speed', optimizes for faster capture with basic quality. If 'quality',
-        # optimizes for higher quality with longer wait times. Defaults to 'quality' if
-        # not provided.
-        prioritize: nil,
         # Optional timeout in milliseconds for the request. If the request takes longer
         # than this value, it will be aborted with a 408 status code. Maximum allowed
         # value is 300000ms (5 minutes).

data/rbi/context_dev/resources/web.rbi

@@ -38,6 +38,49 @@ module ContextDev
       )
       end
 
+      # Performs a crawl starting from a given URL, extracts page content as Markdown,
+      # and returns results for all crawled pages. Only follows links within the same
+      # domain as the starting URL. Costs 1 credit per successful page crawled.
+      sig do
+        params(
+          url: String,
+          follow_subdomains: T::Boolean,
+          include_images: T::Boolean,
+          include_links: T::Boolean,
+          max_depth: Integer,
+          max_pages: Integer,
+          shorten_base64_images: T::Boolean,
+          url_regex: String,
+          use_main_content_only: T::Boolean,
+          request_options: ContextDev::RequestOptions::OrHash
+        ).returns(ContextDev::Models::WebWebCrawlMdResponse)
+      end
+      def web_crawl_md(
+        # The starting URL for the crawl (must include http:// or https:// protocol)
+        url:,
+        # When true, follow links on subdomains of the starting URL's domain (e.g.
+        # docs.example.com when starting from example.com). www and apex are always
+        # treated as equivalent.
+        follow_subdomains: nil,
+        # Include image references in the Markdown output
+        include_images: nil,
+        # Preserve hyperlinks in the Markdown output
+        include_links: nil,
+        # Maximum link depth from the starting URL (0 = only the starting page)
+        max_depth: nil,
+        # Maximum number of pages to crawl. Hard cap: 500.
+        max_pages: nil,
+        # Truncate base64-encoded image data in the Markdown output
+        shorten_base64_images: nil,
+        # Regex pattern. Only URLs matching this pattern will be followed and scraped.
+        url_regex: nil,
+        # Extract only the main content, stripping headers, footers, sidebars, and
+        # navigation
+        use_main_content_only: nil,
+        request_options: {}
+      )
+      end
+
       # Scrapes the given URL and returns the raw HTML content of the page.
       sig do
         params(

data/sig/context_dev/internal/util.rbs

@@ -45,8 +45,12 @@ module ContextDev
         -> top?
       } -> top?
 
+      RFC_3986_NOT_PCHARS: Regexp
+
       def self?.uri_origin: (URI::Generic uri) -> String
 
+      def self?.encode_path: (String | Integer path) -> String
+
       def self?.interpolate_path: (String | ::Array[String] path) -> String
 
       def self?.decode_query: (String? query) -> ::Hash[String, ::Array[String]]

data/sig/context_dev/models/style_extract_styleguide_params.rbs

@@ -1,12 +1,7 @@
 module ContextDev
   module Models
     type style_extract_styleguide_params =
-      {
-        direct_url: String,
-        domain: String,
-        prioritize: ContextDev::Models::StyleExtractStyleguideParams::prioritize,
-        timeout_ms: Integer
-      }
+      { direct_url: String, domain: String, timeout_ms: Integer }
       & ContextDev::Internal::Type::request_parameters
 
     class StyleExtractStyleguideParams < ContextDev::Internal::Type::BaseModel
@@ -21,12 +16,6 @@ module ContextDev
 
       def domain=: (String) -> String
 
-      attr_reader prioritize: ContextDev::Models::StyleExtractStyleguideParams::prioritize?
-
-      def prioritize=: (
-        ContextDev::Models::StyleExtractStyleguideParams::prioritize
-      ) -> ContextDev::Models::StyleExtractStyleguideParams::prioritize
-
       attr_reader timeout_ms: Integer?
 
       def timeout_ms=: (Integer) -> Integer
@@ -34,7 +23,6 @@ module ContextDev
       def initialize: (
         ?direct_url: String,
         ?domain: String,
-        ?prioritize: ContextDev::Models::StyleExtractStyleguideParams::prioritize,
         ?timeout_ms: Integer,
         ?request_options: ContextDev::request_opts
       ) -> void
@@ -42,21 +30,9 @@ module ContextDev
       def to_hash: -> {
         direct_url: String,
         domain: String,
-        prioritize: ContextDev::Models::StyleExtractStyleguideParams::prioritize,
         timeout_ms: Integer,
         request_options: ContextDev::RequestOptions
       }
-
-      type prioritize = :speed | :quality
-
-      module Prioritize
-        extend ContextDev::Internal::Type::Enum
-
-        SPEED: :speed
-        QUALITY: :quality
-
-        def self?.values: -> ::Array[ContextDev::Models::StyleExtractStyleguideParams::prioritize]
-      end
     end
   end
 end