context.dev 2.6.0 → 2.7.0

data/lib/context_dev/models/crawl_controls.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module ContextDev
+  module Models
+    class CrawlControls < ContextDev::Internal::Type::BaseModel
+      # @!attribute follow_subdomains
+      #   Whether links to subdomains were followed. Always false for a sitemap crawl.
+      #
+      #   @return [Boolean]
+      required :follow_subdomains, ContextDev::Internal::Type::Boolean
+
+      # @!attribute max_depth
+      #   Link depth limit. Always 0 for a sitemap crawl, which never follows links off
+      #   its URLs; null when a `start_url` crawl set no limit.
+      #
+      #   @return [Integer, nil]
+      required :max_depth, Integer, nil?: true
+
+      # @!attribute max_pages
+      #   The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs
+      #   its sitemap actually lists, up to this many, so `input.reserved` is often lower.
+      #
+      #   @return [Integer]
+      required :max_pages, Integer
+
+      # @!attribute source
+      #   Where the crawl started.
+      #
+      #   @return [ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1]
+      required :source, union: -> { ContextDev::CrawlControls::Source }
+
+      # @!attribute url_pattern
+      #   RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
+      #
+      #   @return [String, nil]
+      required :url_pattern, String, nil?: true
+
+      # @!method initialize(follow_subdomains:, max_depth:, max_pages:, source:, url_pattern:)
+      #   Some parameter documentations has been truncated, see
+      #   {ContextDev::Models::CrawlControls} for more details.
+      #
+      #   The crawl controls as submitted, so the limits requested can be compared against
+      #   what the crawl reached.
+      #
+      #   @param follow_subdomains [Boolean] Whether links to subdomains were followed. Always false for a sitemap crawl.
+      #
+      #   @param max_depth [Integer, nil] Link depth limit. Always 0 for a sitemap crawl, which never follows links off it
+      #
+      #   @param max_pages [Integer] The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs it
+      #
+      #   @param source [ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1] Where the crawl started.
+      #
+      #   @param url_pattern [String, nil] RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
+
+      # Where the crawl started.
+      #
+      # @see ContextDev::Models::CrawlControls#source
+      module Source
+        extend ContextDev::Internal::Type::Union
+
+        variant -> { ContextDev::CrawlControls::Source::UnionMember0 }
+
+        variant -> { ContextDev::CrawlControls::Source::UnionMember1 }
+
+        class UnionMember0 < ContextDev::Internal::Type::BaseModel
+          # @!attribute type
+          #
+          #   @return [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember0::Type]
+          required :type, enum: -> { ContextDev::CrawlControls::Source::UnionMember0::Type }
+
+          # @!attribute url
+          #   Page the crawl started from.
+          #
+          #   @return [String]
+          required :url, String
+
+          # @!method initialize(type:, url:)
+          #   @param type [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember0::Type]
+          #
+          #   @param url [String] Page the crawl started from.
+
+          # @see ContextDev::Models::CrawlControls::Source::UnionMember0#type
+          module Type
+            extend ContextDev::Internal::Type::Enum
+
+            START_URL = :start_url
+
+            # @!method self.values
+            #   @return [Array<Symbol>]
+          end
+        end
+
+        class UnionMember1 < ContextDev::Internal::Type::BaseModel
+          # @!attribute domain
+          #   Domain whose sitemap supplied the pages.
+          #
+          #   @return [String]
+          required :domain, String
+
+          # @!attribute type
+          #
+          #   @return [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember1::Type]
+          required :type, enum: -> { ContextDev::CrawlControls::Source::UnionMember1::Type }
+
+          # @!method initialize(domain:, type:)
+          #   @param domain [String] Domain whose sitemap supplied the pages.
+          #
+          #   @param type [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember1::Type]
+
+          # @see ContextDev::Models::CrawlControls::Source::UnionMember1#type
+          module Type
+            extend ContextDev::Internal::Type::Enum
+
+            SITEMAP = :sitemap
+
+            # @!method self.values
+            #   @return [Array<Symbol>]
+          end
+        end
+
+        # @!method self.variants
+        #   @return [Array(ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1)]
+      end
+    end
+  end
+end

data/lib/context_dev/models/failure.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ContextDev
+  module Models
+    class Failure < ContextDev::Internal::Type::BaseModel
+      # @!attribute code
+      #   Why the batch itself stopped.
+      #
+      #   @return [String]
+      required :code, String
+
+      # @!attribute message
+      #   Human-readable explanation.
+      #
+      #   @return [String]
+      required :message, String
+
+      # @!method initialize(code:, message:)
+      #   A failure of the batch as a whole, distinct from the per-page failures in
+      #   `page_errors`.
+      #
+      #   @param code [String] Why the batch itself stopped.
+      #
+      #   @param message [String] Human-readable explanation.
+    end
+  end
+end

data/lib/context_dev/models/intake.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module ContextDev
+  module Models
+    class Intake < ContextDev::Internal::Type::BaseModel
+      # @!attribute duplicates
+      #   URLs dropped before reserving because another entry resolved to the same page.
+      #   Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than
+      #   once.
+      #
+      #   @return [Integer]
+      required :duplicates, Integer
+
+      # @!attribute invalid
+      #   URLs from your list rejected as unusable; the same ones are itemised in
+      #   `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable
+      #   page is rejected outright with a 400 rather than accepted with an empty list.
+      #
+      #   @return [Integer, nil]
+      required :invalid, Integer, nil?: true
+
+      # @!attribute reserved
+      #   Pages credits were reserved for. Everything else — progress, the refund, the
+      #   completion percentage — is measured against this.
+      #
+      #   @return [Integer]
+      required :reserved, Integer
+
+      # @!attribute reserved_is_ceiling
+      #   Whether `reserved` is an upper bound the batch may finish under. True only for a
+      #   crawl that follows links, whose reachable page count is unknowable until it
+      #   runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact
+      #   page count.
+      #
+      #   @return [Boolean]
+      required :reserved_is_ceiling, ContextDev::Internal::Type::Boolean
+
+      # @!attribute submitted
+      #   URLs in the list you sent, before validation and de-duplication. Null for a
+      #   crawl, which is given a source rather than a list.
+      #
+      #   @return [Integer, nil]
+      required :submitted, Integer, nil?: true
+
+      # @!method initialize(duplicates:, invalid:, reserved:, reserved_is_ceiling:, submitted:)
+      #   Some parameter documentations has been truncated, see
+      #   {ContextDev::Models::Intake} for more details.
+      #
+      #   What submission took in, and what it charged for.
+      #
+      #   @param duplicates [Integer] URLs dropped before reserving because another entry resolved to the same page. N
+      #
+      #   @param invalid [Integer, nil] URLs from your list rejected as unusable; the same ones are itemised in `invalid
+      #
+      #   @param reserved [Integer] Pages credits were reserved for. Everything else — progress, the refund, the com
+      #
+      #   @param reserved_is_ceiling [Boolean] Whether `reserved` is an upper bound the batch may finish under. True only for a
+      #
+      #   @param submitted [Integer, nil] URLs in the list you sent, before validation and de-duplication. Null for a craw
+    end
+  end
+end

data/lib/context_dev/models/{error_count.rb → page_error_count.rb}

@@ -2,7 +2,7 @@
 
 module ContextDev
   module Models
-    class ErrorCount < ContextDev::Internal::Type::BaseModel
+    class PageErrorCount < ContextDev::Internal::Type::BaseModel
       # @!attribute code
       #   Error code for these failures.
       #

data/lib/context_dev/models/web_screenshot_params.rb

@@ -15,9 +15,8 @@ module ContextDev
       optional :color_scheme, enum: -> { ContextDev::WebScreenshotParams::ColorScheme }
 
       # @!attribute country
-      #   Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      #   residential proxy exit location. Must be one of Context.dev's supported
-      #   countries. When provided, Context.dev fetches the target page from that country.
+      #   Fetch the target page through a residential proxy in this country (ISO 3166-1
+      #   alpha-2).
       #
       #   @return [Symbol, ContextDev::Models::WebScreenshotParams::Country, nil]
       optional :country, enum: -> { ContextDev::WebScreenshotParams::Country }
@@ -128,7 +127,7 @@ module ContextDev
       #
       #   @param color_scheme [Symbol, ContextDev::Models::WebScreenshotParams::ColorScheme] Optional parameter to choose the site's visual theme in the screenshot. Use 'lig
       #
-      #   @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      #   @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
       #
       #   @param direct_url [String] A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https
       #
@@ -168,9 +167,8 @@ module ContextDev
         #   @return [Array<Symbol>]
       end
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       module Country
         extend ContextDev::Internal::Type::Enum
 

data/lib/context_dev/models/web_web_crawl_md_params.rb

@@ -14,9 +14,8 @@ module ContextDev
       required :url, String
 
       # @!attribute country
-      #   Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      #   residential proxy exit location. Must be one of Context.dev's supported
-      #   countries. When provided, Context.dev fetches the target page from that country.
+      #   Fetch the target page through a residential proxy in this country (ISO 3166-1
+      #   alpha-2).
       #
       #   @return [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country, nil]
       optional :country, enum: -> { ContextDev::WebWebCrawlMdParams::Country }
@@ -164,7 +163,7 @@ module ContextDev
       #
       #   @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
       #
-      #   @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      #   @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
       #
       #   @param exclude_selectors [Array<String>] CSS selectors to remove before each crawled page is converted to Markdown. Appli
       #
@@ -206,9 +205,8 @@ module ContextDev
       #
       #   @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       module Country
         extend ContextDev::Internal::Type::Enum
 

data/lib/context_dev/models/web_web_scrape_html_params.rb

@@ -24,9 +24,8 @@ module ContextDev
                nil?: true
 
       # @!attribute country
-      #   Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      #   residential proxy exit location. Must be one of Context.dev's supported
-      #   countries. When provided, Context.dev fetches the target page from that country.
+      #   Fetch the target page through a residential proxy in this country (ISO 3166-1
+      #   alpha-2).
       #
       #   @return [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country, nil]
       optional :country, enum: -> { ContextDev::WebWebScrapeHTMLParams::Country }
@@ -131,7 +130,7 @@ module ContextDev
       #
       #   @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
       #
-      #   @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      #   @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
       #
       #   @param exclude_selectors [Array<String>, nil] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi
       #
@@ -212,9 +211,8 @@ module ContextDev
         #   @return [Array(ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform)]
       end
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       module Country
         extend ContextDev::Internal::Type::Enum
 

data/lib/context_dev/models/web_web_scrape_md_params.rb

@@ -25,9 +25,8 @@ module ContextDev
                nil?: true
 
       # @!attribute country
-      #   Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      #   residential proxy exit location. Must be one of Context.dev's supported
-      #   countries. When provided, Context.dev fetches the target page from that country.
+      #   Fetch the target page through a residential proxy in this country (ISO 3166-1
+      #   alpha-2).
       #
       #   @return [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country, nil]
       optional :country, enum: -> { ContextDev::WebWebScrapeMdParams::Country }
@@ -150,7 +149,7 @@ module ContextDev
       #
       #   @param actions [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
       #
-      #   @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      #   @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
       #
       #   @param exclude_selectors [Array<String>, nil] CSS selectors to remove before conversion to Markdown. Applied after includeSele
       #
@@ -237,9 +236,8 @@ module ContextDev
         #   @return [Array(ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform)]
       end
 
-      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
-      # residential proxy exit location. Must be one of Context.dev's supported
-      # countries. When provided, Context.dev fetches the target page from that country.
+      # Fetch the target page through a residential proxy in this country (ISO 3166-1
+      # alpha-2).
       module Country
         extend ContextDev::Internal::Type::Enum
 

data/lib/context_dev/models.rb

@@ -57,14 +57,16 @@ module ContextDev
 
   BrandRetrieveSimplifiedParams = ContextDev::Models::BrandRetrieveSimplifiedParams
 
-  Error = ContextDev::Models::Error
+  CrawlControls = ContextDev::Models::CrawlControls
 
-  ErrorCount = ContextDev::Models::ErrorCount
+  Failure = ContextDev::Models::Failure
 
   IndustryRetrieveNaicsParams = ContextDev::Models::IndustryRetrieveNaicsParams
 
   IndustryRetrieveSicParams = ContextDev::Models::IndustryRetrieveSicParams
 
+  Intake = ContextDev::Models::Intake
+
   MonitorCreateParams = ContextDev::Models::MonitorCreateParams
 
   MonitorDeleteParams = ContextDev::Models::MonitorDeleteParams
@@ -91,6 +93,8 @@ module ContextDev
 
   MonitorUpdateParams = ContextDev::Models::MonitorUpdateParams
 
+  PageErrorCount = ContextDev::Models::PageErrorCount
+
   ParseHandleParams = ContextDev::Models::ParseHandleParams
 
   UtilityPrefetchParams = ContextDev::Models::UtilityPrefetchParams

data/lib/context_dev/resources/batch.rb

@@ -3,9 +3,7 @@
 module ContextDev
   module Resources
     class Batch
-      # Check progress and get download links when the batch finishes. Also returns the
-      # rejected-URL list and webhook signing secret from submission, so nothing is lost
-      # if the submit response was dropped.
+      # Check progress, and get download links once the batch finishes.
       #
       # @overload retrieve(batch_id, request_options: {})
       #
@@ -86,9 +84,8 @@ module ContextDev
       # Some parameter documentations has been truncated, see
       # {ContextDev::Models::BatchGetResultsParams} for more details.
       #
-      # Page through the result records of a finished batch as JSON, in the same order
-      # as the downloadable result files. Use this instead of downloading and parsing
-      # the NDJSON files yourself.
+      # Page through a finished batch's results as JSON instead of downloading the
+      # NDJSON files.
       #
       # @overload get_results(batch_id, cursor: nil, limit: nil, request_options: {})
       #

data/lib/context_dev/resources/web.rb

@@ -180,7 +180,7 @@ module ContextDev
       #
       # @param color_scheme [Symbol, ContextDev::Models::WebScreenshotParams::ColorScheme] Optional parameter to choose the site's visual theme in the screenshot. Use 'lig
       #
-      # @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      # @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
       #
       # @param direct_url [String] A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https
       #
@@ -285,7 +285,7 @@ module ContextDev
       #
       # @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
       #
-      # @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      # @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
       #
       # @param exclude_selectors [Array<String>] CSS selectors to remove before each crawled page is converted to Markdown. Appli
       #
@@ -353,7 +353,7 @@ module ContextDev
       #
       # @param actions [Array<ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
       #
-      # @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      # @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
       #
       # @param exclude_selectors [Array<String>, nil] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi
       #
@@ -481,7 +481,7 @@ module ContextDev
       #
       # @param actions [Array<ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform>, nil] Optional browser actions executed in array order after the page loads and before
       #
-      # @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      # @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al
       #
       # @param exclude_selectors [Array<String>, nil] CSS selectors to remove before conversion to Markdown. Applied after includeSele
       #

data/lib/context_dev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ContextDev
-  VERSION = "2.6.0"
+  VERSION = "2.7.0"
 end

data/lib/context_dev.rb

@@ -70,12 +70,13 @@ require_relative "context_dev/models/brand_retrieve_params"
 require_relative "context_dev/models/brand_retrieve_response"
 require_relative "context_dev/models/brand_retrieve_simplified_params"
 require_relative "context_dev/models/brand_retrieve_simplified_response"
-require_relative "context_dev/models/error"
-require_relative "context_dev/models/error_count"
+require_relative "context_dev/models/crawl_controls"
+require_relative "context_dev/models/failure"
 require_relative "context_dev/models/industry_retrieve_naics_params"
 require_relative "context_dev/models/industry_retrieve_naics_response"
 require_relative "context_dev/models/industry_retrieve_sic_params"
 require_relative "context_dev/models/industry_retrieve_sic_response"
+require_relative "context_dev/models/intake"
 require_relative "context_dev/models/monitor_create_params"
 require_relative "context_dev/models/monitor_create_response"
 require_relative "context_dev/models/monitor_delete_params"
@@ -102,6 +103,7 @@ require_relative "context_dev/models/monitor_run_params"
 require_relative "context_dev/models/monitor_run_response"
 require_relative "context_dev/models/monitor_update_params"
 require_relative "context_dev/models/monitor_update_response"
+require_relative "context_dev/models/page_error_count"
 require_relative "context_dev/models/parse_handle_params"
 require_relative "context_dev/models/parse_handle_response"
 require_relative "context_dev/models/utility_prefetch_params"