context.dev 2.6.0 → 2.7.0

data/rbi/context_dev/models/crawl_controls.rbi

@@ -0,0 +1,252 @@
+# typed: strong
+
+module ContextDev
+  module Models
+    class CrawlControls < ContextDev::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(ContextDev::CrawlControls, ContextDev::Internal::AnyHash)
+        end
+
+      # Whether links to subdomains were followed. Always false for a sitemap crawl.
+      sig { returns(T::Boolean) }
+      attr_accessor :follow_subdomains
+
+      # Link depth limit. Always 0 for a sitemap crawl, which never follows links off
+      # its URLs; null when a `start_url` crawl set no limit.
+      sig { returns(T.nilable(Integer)) }
+      attr_accessor :max_depth
+
+      # The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs
+      # its sitemap actually lists, up to this many, so `input.reserved` is often lower.
+      sig { returns(Integer) }
+      attr_accessor :max_pages
+
+      # Where the crawl started.
+      sig { returns(ContextDev::CrawlControls::Source::Variants) }
+      attr_accessor :source
+
+      # RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
+      sig { returns(T.nilable(String)) }
+      attr_accessor :url_pattern
+
+      # The crawl controls as submitted, so the limits requested can be compared against
+      # what the crawl reached.
+      sig do
+        params(
+          follow_subdomains: T::Boolean,
+          max_depth: T.nilable(Integer),
+          max_pages: Integer,
+          source:
+            T.any(
+              ContextDev::CrawlControls::Source::UnionMember0::OrHash,
+              ContextDev::CrawlControls::Source::UnionMember1::OrHash
+            ),
+          url_pattern: T.nilable(String)
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # Whether links to subdomains were followed. Always false for a sitemap crawl.
+        follow_subdomains:,
+        # Link depth limit. Always 0 for a sitemap crawl, which never follows links off
+        # its URLs; null when a `start_url` crawl set no limit.
+        max_depth:,
+        # The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs
+        # its sitemap actually lists, up to this many, so `input.reserved` is often lower.
+        max_pages:,
+        # Where the crawl started.
+        source:,
+        # RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
+        url_pattern:
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            follow_subdomains: T::Boolean,
+            max_depth: T.nilable(Integer),
+            max_pages: Integer,
+            source: ContextDev::CrawlControls::Source::Variants,
+            url_pattern: T.nilable(String)
+          }
+        )
+      end
+      def to_hash
+      end
+
+      # Where the crawl started.
+      module Source
+        extend ContextDev::Internal::Type::Union
+
+        Variants =
+          T.type_alias do
+            T.any(
+              ContextDev::CrawlControls::Source::UnionMember0,
+              ContextDev::CrawlControls::Source::UnionMember1
+            )
+          end
+
+        class UnionMember0 < ContextDev::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                ContextDev::CrawlControls::Source::UnionMember0,
+                ContextDev::Internal::AnyHash
+              )
+            end
+
+          sig do
+            returns(
+              ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol
+            )
+          end
+          attr_accessor :type
+
+          # Page the crawl started from.
+          sig { returns(String) }
+          attr_accessor :url
+
+          sig do
+            params(
+              type:
+                ContextDev::CrawlControls::Source::UnionMember0::Type::OrSymbol,
+              url: String
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            type:,
+            # Page the crawl started from.
+            url:
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                type:
+                  ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol,
+                url: String
+              }
+            )
+          end
+          def to_hash
+          end
+
+          module Type
+            extend ContextDev::Internal::Type::Enum
+
+            TaggedSymbol =
+              T.type_alias do
+                T.all(
+                  Symbol,
+                  ContextDev::CrawlControls::Source::UnionMember0::Type
+                )
+              end
+            OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+            START_URL =
+              T.let(
+                :start_url,
+                ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol
+              )
+
+            sig do
+              override.returns(
+                T::Array[
+                  ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol
+                ]
+              )
+            end
+            def self.values
+            end
+          end
+        end
+
+        class UnionMember1 < ContextDev::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                ContextDev::CrawlControls::Source::UnionMember1,
+                ContextDev::Internal::AnyHash
+              )
+            end
+
+          # Domain whose sitemap supplied the pages.
+          sig { returns(String) }
+          attr_accessor :domain
+
+          sig do
+            returns(
+              ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol
+            )
+          end
+          attr_accessor :type
+
+          sig do
+            params(
+              domain: String,
+              type:
+                ContextDev::CrawlControls::Source::UnionMember1::Type::OrSymbol
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            # Domain whose sitemap supplied the pages.
+            domain:,
+            type:
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                domain: String,
+                type:
+                  ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol
+              }
+            )
+          end
+          def to_hash
+          end
+
+          module Type
+            extend ContextDev::Internal::Type::Enum
+
+            TaggedSymbol =
+              T.type_alias do
+                T.all(
+                  Symbol,
+                  ContextDev::CrawlControls::Source::UnionMember1::Type
+                )
+              end
+            OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+            SITEMAP =
+              T.let(
+                :sitemap,
+                ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol
+              )
+
+            sig do
+              override.returns(
+                T::Array[
+                  ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol
+                ]
+              )
+            end
+            def self.values
+            end
+          end
+        end
+
+        sig do
+          override.returns(
+            T::Array[ContextDev::CrawlControls::Source::Variants]
+          )
+        end
+        def self.variants
+        end
+      end
+    end
+  end
+end

data/rbi/context_dev/models/{error.rbi → failure.rbi}

@@ -2,24 +2,27 @@
 
 module ContextDev
   module Models
-    class Error < ContextDev::Internal::Type::BaseModel
+    class Failure < ContextDev::Internal::Type::BaseModel
       OrHash =
-        T.type_alias { T.any(ContextDev::Error, ContextDev::Internal::AnyHash) }
+        T.type_alias do
+          T.any(ContextDev::Failure, ContextDev::Internal::AnyHash)
+        end
 
-      # Batch error code.
+      # Why the batch itself stopped.
       sig { returns(String) }
       attr_accessor :code
 
-      # Batch error message.
+      # Human-readable explanation.
       sig { returns(String) }
       attr_accessor :message
 
-      # Why the batch failed.
+      # A failure of the batch as a whole, distinct from the per-page failures in
+      # `page_errors`.
       sig { params(code: String, message: String).returns(T.attached_class) }
       def self.new(
-        # Batch error code.
+        # Why the batch itself stopped.
         code:,
-        # Batch error message.
+        # Human-readable explanation.
         message:
       )
       end

data/rbi/context_dev/models/intake.rbi

@@ -0,0 +1,88 @@
+# typed: strong
+
+module ContextDev
+  module Models
+    class Intake < ContextDev::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(ContextDev::Intake, ContextDev::Internal::AnyHash)
+        end
+
+      # URLs dropped before reserving because another entry resolved to the same page.
+      # Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than
+      # once.
+      sig { returns(Integer) }
+      attr_accessor :duplicates
+
+      # URLs from your list rejected as unusable; the same ones are itemised in
+      # `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable
+      # page is rejected outright with a 400 rather than accepted with an empty list.
+      sig { returns(T.nilable(Integer)) }
+      attr_accessor :invalid
+
+      # Pages credits were reserved for. Everything else — progress, the refund, the
+      # completion percentage — is measured against this.
+      sig { returns(Integer) }
+      attr_accessor :reserved
+
+      # Whether `reserved` is an upper bound the batch may finish under. True only for a
+      # crawl that follows links, whose reachable page count is unknowable until it
+      # runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact
+      # page count.
+      sig { returns(T::Boolean) }
+      attr_accessor :reserved_is_ceiling
+
+      # URLs in the list you sent, before validation and de-duplication. Null for a
+      # crawl, which is given a source rather than a list.
+      sig { returns(T.nilable(Integer)) }
+      attr_accessor :submitted
+
+      # What submission took in, and what it charged for.
+      sig do
+        params(
+          duplicates: Integer,
+          invalid: T.nilable(Integer),
+          reserved: Integer,
+          reserved_is_ceiling: T::Boolean,
+          submitted: T.nilable(Integer)
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # URLs dropped before reserving because another entry resolved to the same page.
+        # Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than
+        # once.
+        duplicates:,
+        # URLs from your list rejected as unusable; the same ones are itemised in
+        # `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable
+        # page is rejected outright with a 400 rather than accepted with an empty list.
+        invalid:,
+        # Pages credits were reserved for. Everything else — progress, the refund, the
+        # completion percentage — is measured against this.
+        reserved:,
+        # Whether `reserved` is an upper bound the batch may finish under. True only for a
+        # crawl that follows links, whose reachable page count is unknowable until it
+        # runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact
+        # page count.
+        reserved_is_ceiling:,
+        # URLs in the list you sent, before validation and de-duplication. Null for a
+        # crawl, which is given a source rather than a list.
+        submitted:
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            duplicates: Integer,
+            invalid: T.nilable(Integer),
+            reserved: Integer,
+            reserved_is_ceiling: T::Boolean,
+            submitted: T.nilable(Integer)
+          }
+        )
+      end
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/context_dev/models/{error_count.rbi → page_error_count.rbi}

@@ -2,10 +2,10 @@
 
 module ContextDev
   module Models
-    class ErrorCount < ContextDev::Internal::Type::BaseModel
+    class PageErrorCount < ContextDev::Internal::Type::BaseModel
       OrHash =
         T.type_alias do
-          T.any(ContextDev::ErrorCount, ContextDev::Internal::AnyHash)
+          T.any(ContextDev::PageErrorCount, ContextDev::Internal::AnyHash)
         end
 
       # Error code for these failures.

data/rbi/context_dev/models/web_screenshot_params.rbi

@@ -27,9 +27,8 @@ module ContextDev
       end
       attr_writer :color_scheme
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       sig do
         returns(T.nilable(ContextDev::WebScreenshotParams::Country::OrSymbol))
       end
@@ -201,9 +200,8 @@ module ContextDev
         # Optional parameter to choose the site's visual theme in the screenshot. Use
         # 'light' or 'dark' when the site offers both appearances.
         color_scheme: nil,
-        # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-        # residential proxy exit location. Must be one of Context.dev's supported
-        # countries. When provided, Context.dev fetches the target page from that country.
+        # Fetch the target page through a residential proxy in this country (ISO 3166-1
+        # alpha-2).
         country: nil,
         # A specific URL to screenshot directly, bypassing domain resolution (e.g.,
         # 'https://example.com/pricing'). When provided, the screenshot is taken of this
@@ -322,9 +320,8 @@ module ContextDev
         end
       end
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       module Country
         extend ContextDev::Internal::Type::Enum
 

data/rbi/context_dev/models/web_web_crawl_md_params.rbi

@@ -15,9 +15,8 @@ module ContextDev
       sig { returns(String) }
       attr_accessor :url
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       sig do
         returns(T.nilable(ContextDev::WebWebCrawlMdParams::Country::OrSymbol))
       end
@@ -213,9 +212,8 @@ module ContextDev
       def self.new(
         # The starting URL for the crawl (must include http:// or https:// protocol)
         url:,
-        # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-        # residential proxy exit location. Must be one of Context.dev's supported
-        # countries. When provided, Context.dev fetches the target page from that country.
+        # Fetch the target page through a residential proxy in this country (ISO 3166-1
+        # alpha-2).
         country: nil,
         # CSS selectors to remove before each crawled page is converted to Markdown.
         # Applied after includeSelectors. Exclusion takes precedence: an element matching
@@ -313,9 +311,8 @@ module ContextDev
       def to_hash
       end
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       module Country
         extend ContextDev::Internal::Type::Enum
 

data/rbi/context_dev/models/web_web_scrape_html_params.rbi

@@ -35,9 +35,8 @@ module ContextDev
       end
       attr_accessor :actions
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       sig do
         returns(
           T.nilable(ContextDev::WebWebScrapeHTMLParams::Country::OrSymbol)
@@ -246,9 +245,8 @@ module ContextDev
         # content is captured. Requires a paid plan. Send a JSON array in the query
         # parameter. Maximum: 5 actions.
         actions: nil,
-        # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-        # residential proxy exit location. Must be one of Context.dev's supported
-        # countries. When provided, Context.dev fetches the target page from that country.
+        # Fetch the target page through a residential proxy in this country (ISO 3166-1
+        # alpha-2).
         country: nil,
         # CSS selectors to remove from the result. Applied after includeSelectors.
         # Exclusion takes precedence: an element matching both is removed. Examples:
@@ -417,9 +415,8 @@ module ContextDev
         end
       end
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       module Country
         extend ContextDev::Internal::Type::Enum
 

data/rbi/context_dev/models/web_web_scrape_md_params.rbi

@@ -33,9 +33,8 @@ module ContextDev
       end
       attr_accessor :actions
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       sig do
         returns(T.nilable(ContextDev::WebWebScrapeMdParams::Country::OrSymbol))
       end
@@ -328,9 +327,8 @@ module ContextDev
         # content is captured. Requires a paid plan. Send a JSON array in the query
         # parameter. Maximum: 5 actions.
         actions: nil,
-        # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-        # residential proxy exit location. Must be one of Context.dev's supported
-        # countries. When provided, Context.dev fetches the target page from that country.
+        # Fetch the target page through a residential proxy in this country (ISO 3166-1
+        # alpha-2).
         country: nil,
         # CSS selectors to remove before conversion to Markdown. Applied after
         # includeSelectors. Exclusion takes precedence: an element matching both is
@@ -520,9 +518,8 @@ module ContextDev
         end
       end
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       module Country
         extend ContextDev::Internal::Type::Enum
 

data/rbi/context_dev/models.rbi

@@ -20,14 +20,16 @@ module ContextDev
   BrandRetrieveSimplifiedParams =
     ContextDev::Models::BrandRetrieveSimplifiedParams
 
-  Error = ContextDev::Models::Error
+  CrawlControls = ContextDev::Models::CrawlControls
 
-  ErrorCount = ContextDev::Models::ErrorCount
+  Failure = ContextDev::Models::Failure
 
   IndustryRetrieveNaicsParams = ContextDev::Models::IndustryRetrieveNaicsParams
 
   IndustryRetrieveSicParams = ContextDev::Models::IndustryRetrieveSicParams
 
+  Intake = ContextDev::Models::Intake
+
   MonitorCreateParams = ContextDev::Models::MonitorCreateParams
 
   MonitorDeleteParams = ContextDev::Models::MonitorDeleteParams
@@ -56,6 +58,8 @@ module ContextDev
 
   MonitorUpdateParams = ContextDev::Models::MonitorUpdateParams
 
+  PageErrorCount = ContextDev::Models::PageErrorCount
+
   ParseHandleParams = ContextDev::Models::ParseHandleParams
 
   UtilityPrefetchParams = ContextDev::Models::UtilityPrefetchParams

data/rbi/context_dev/resources/batch.rbi

@@ -3,9 +3,7 @@
 module ContextDev
   module Resources
     class Batch
-      # Check progress and get download links when the batch finishes. Also returns the
-      # rejected-URL list and webhook signing secret from submission, so nothing is lost
-      # if the submit response was dropped.
+      # Check progress, and get download links once the batch finishes.
       sig do
         params(
           batch_id: String,
@@ -66,9 +64,8 @@ module ContextDev
       )
       end
 
-      # Page through the result records of a finished batch as JSON, in the same order
-      # as the downloadable result files. Use this instead of downloading and parsing
-      # the NDJSON files yourself.
+      # Page through a finished batch's results as JSON instead of downloading the
+      # NDJSON files.
       sig do
         params(
           batch_id: String,

data/rbi/context_dev/resources/web.rbi

@@ -216,9 +216,8 @@ module ContextDev
         # Optional parameter to choose the site's visual theme in the screenshot. Use
         # 'light' or 'dark' when the site offers both appearances.
         color_scheme: nil,
-        # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-        # residential proxy exit location. Must be one of Context.dev's supported
-        # countries. When provided, Context.dev fetches the target page from that country.
+        # Fetch the target page through a residential proxy in this country (ISO 3166-1
+        # alpha-2).
         country: nil,
         # A specific URL to screenshot directly, bypassing domain resolution (e.g.,
         # 'https://example.com/pricing'). When provided, the screenshot is taken of this
@@ -355,9 +354,8 @@ module ContextDev
       def web_crawl_md(
         # The starting URL for the crawl (must include http:// or https:// protocol)
         url:,
-        # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-        # residential proxy exit location. Must be one of Context.dev's supported
-        # countries. When provided, Context.dev fetches the target page from that country.
+        # Fetch the target page through a residential proxy in this country (ISO 3166-1
+        # alpha-2).
         country: nil,
         # CSS selectors to remove before each crawled page is converted to Markdown.
         # Applied after includeSelectors. Exclusion takes precedence: an element matching
@@ -473,9 +471,8 @@ module ContextDev
         # content is captured. Requires a paid plan. Send a JSON array in the query
         # parameter. Maximum: 5 actions.
         actions: nil,
-        # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-        # residential proxy exit location. Must be one of Context.dev's supported
-        # countries. When provided, Context.dev fetches the target page from that country.
+        # Fetch the target page through a residential proxy in this country (ISO 3166-1
+        # alpha-2).
         country: nil,
         # CSS selectors to remove from the result. Applied after includeSelectors.
         # Exclusion takes precedence: an element matching both is removed. Examples:
@@ -673,9 +670,8 @@ module ContextDev
         # content is captured. Requires a paid plan. Send a JSON array in the query
         # parameter. Maximum: 5 actions.
         actions: nil,
-        # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-        # residential proxy exit location. Must be one of Context.dev's supported
-        # countries. When provided, Context.dev fetches the target page from that country.
+        # Fetch the target page through a residential proxy in this country (ISO 3166-1
+        # alpha-2).
         country: nil,
         # CSS selectors to remove before conversion to Markdown. Applied after
         # includeSelectors. Exclusion takes precedence: an element matching both is