context.dev 2.12.0 → 2.14.0

data/lib/context_dev/resources/web.rb

@@ -566,18 +566,21 @@ module ContextDev
       # Some parameter documentations has been truncated, see
       # {ContextDev::Models::WebWebScrapeSitemapParams} for more details.
       #
-      # Crawl an entire website's sitemap and return all discovered page URLs. Pass
-      # `search` to have the crawled sitemap filtered down to the pages about a phrase
-      # (for example `pricing and plans` or `api authentication docs`), most relevant
-      # first — a searched crawl scans the whole sitemap and costs 2 credits instead
-      # of 1.
+      # Crawl an entire website's sitemap and return all discovered page URLs. Set
+      # `includeSubdomains=true` to also discover public pages and sitemaps on child
+      # hosts such as `docs.example.com` or `brand.example.com`. Pass `search` to have
+      # the discovered URLs filtered down to the pages about a phrase (for example
+      # `pricing and plans` or `api authentication docs`), most relevant first — a
+      # searched crawl scans the whole sitemap and costs 2 credits instead of 1.
       #
-      # @overload web_scrape_sitemap(domain:, headers: nil, max_links: nil, search: nil, sitemap_url: nil, tags: nil, timeout_ms: nil, url_regex: nil, zdr: nil, request_options: {})
+      # @overload web_scrape_sitemap(domain:, headers: nil, include_subdomains: nil, max_links: nil, search: nil, sitemap_url: nil, tags: nil, timeout_ms: nil, url_regex: nil, zdr: nil, request_options: {})
       #
       # @param domain [String] Domain to build a sitemap for
       #
       # @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob
       #
+      # @param include_subdomains [Boolean] When true, discover and include public pages and sitemaps on subdomains of the r
+      #
       # @param max_links [Integer] Maximum number of links to return from the sitemap crawl. Defaults to 10,000. Mi
       #
       # @param search [String] Optional search phrase. When provided, the crawled sitemap is filtered to the pa
@@ -604,6 +607,7 @@ module ContextDev
           method: :get,
           path: "web/scrape/sitemap",
           query: query.transform_keys(
+            include_subdomains: "includeSubdomains",
             max_links: "maxLinks",
             sitemap_url: "sitemapUrl",
             timeout_ms: "timeoutMS",

data/lib/context_dev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ContextDev
-  VERSION = "2.12.0"
+  VERSION = "2.14.0"
 end

data/rbi/context_dev/models/ai_extract_product_response.rbi

@@ -11,6 +11,22 @@ module ContextDev
           )
         end
 
+      # Cache outcome for this response. Composite responses are hits only when every
+      # cache-controlled fetch contributing to the output was a hit; age_ms is the
+      # oldest contributing hit.
+      sig do
+        returns(ContextDev::Models::AIExtractProductResponse::CacheMetadata)
+      end
+      attr_reader :cache_metadata
+
+      sig do
+        params(
+          cache_metadata:
+            ContextDev::Models::AIExtractProductResponse::CacheMetadata::OrHash
+        ).void
+      end
+      attr_writer :cache_metadata
+
       # Whether the given URL is a product detail page
       sig { returns(T.nilable(T::Boolean)) }
       attr_reader :is_product_page
@@ -65,6 +81,8 @@ module ContextDev
 
       sig do
         params(
+          cache_metadata:
+            ContextDev::Models::AIExtractProductResponse::CacheMetadata::OrHash,
           is_product_page: T::Boolean,
           key_metadata:
             ContextDev::Models::AIExtractProductResponse::KeyMetadata::OrHash,
@@ -79,6 +97,10 @@ module ContextDev
         ).returns(T.attached_class)
       end
       def self.new(
+        # Cache outcome for this response. Composite responses are hits only when every
+        # cache-controlled fetch contributing to the output was a hit; age_ms is the
+        # oldest contributing hit.
+        cache_metadata:,
         # Whether the given URL is a product detail page
         is_product_page: nil,
         # Metadata about the API key used for the request. Included in every response
@@ -94,6 +116,8 @@ module ContextDev
       sig do
         override.returns(
           {
+            cache_metadata:
+              ContextDev::Models::AIExtractProductResponse::CacheMetadata,
             is_product_page: T::Boolean,
             key_metadata:
               ContextDev::Models::AIExtractProductResponse::KeyMetadata,
@@ -109,6 +133,101 @@ module ContextDev
       def to_hash
       end
 
+      class CacheMetadata < ContextDev::Internal::Type::BaseModel
+        OrHash =
+          T.type_alias do
+            T.any(
+              ContextDev::Models::AIExtractProductResponse::CacheMetadata,
+              ContextDev::Internal::AnyHash
+            )
+          end
+
+        # Age of the cached data in milliseconds. Zero for miss and zdr responses.
+        sig { returns(Integer) }
+        attr_accessor :age_ms
+
+        # Whether the response was served from cache, required fresh work, or honored
+        # zero-data-retention cache bypass.
+        sig do
+          returns(
+            ContextDev::Models::AIExtractProductResponse::CacheMetadata::Status::TaggedSymbol
+          )
+        end
+        attr_accessor :status
+
+        # Cache outcome for this response. Composite responses are hits only when every
+        # cache-controlled fetch contributing to the output was a hit; age_ms is the
+        # oldest contributing hit.
+        sig do
+          params(
+            age_ms: Integer,
+            status:
+              ContextDev::Models::AIExtractProductResponse::CacheMetadata::Status::OrSymbol
+          ).returns(T.attached_class)
+        end
+        def self.new(
+          # Age of the cached data in milliseconds. Zero for miss and zdr responses.
+          age_ms:,
+          # Whether the response was served from cache, required fresh work, or honored
+          # zero-data-retention cache bypass.
+          status:
+        )
+        end
+
+        sig do
+          override.returns(
+            {
+              age_ms: Integer,
+              status:
+                ContextDev::Models::AIExtractProductResponse::CacheMetadata::Status::TaggedSymbol
+            }
+          )
+        end
+        def to_hash
+        end
+
+        # Whether the response was served from cache, required fresh work, or honored
+        # zero-data-retention cache bypass.
+        module Status
+          extend ContextDev::Internal::Type::Enum
+
+          TaggedSymbol =
+            T.type_alias do
+              T.all(
+                Symbol,
+                ContextDev::Models::AIExtractProductResponse::CacheMetadata::Status
+              )
+            end
+          OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+          HIT =
+            T.let(
+              :hit,
+              ContextDev::Models::AIExtractProductResponse::CacheMetadata::Status::TaggedSymbol
+            )
+          MISS =
+            T.let(
+              :miss,
+              ContextDev::Models::AIExtractProductResponse::CacheMetadata::Status::TaggedSymbol
+            )
+          ZDR =
+            T.let(
+              :zdr,
+              ContextDev::Models::AIExtractProductResponse::CacheMetadata::Status::TaggedSymbol
+            )
+
+          sig do
+            override.returns(
+              T::Array[
+                ContextDev::Models::AIExtractProductResponse::CacheMetadata::Status::TaggedSymbol
+              ]
+            )
+          end
+          def self.values
+          end
+        end
+      end
+
       class KeyMetadata < ContextDev::Internal::Type::BaseModel
         OrHash =
           T.type_alias do

data/rbi/context_dev/models/ai_extract_products_response.rbi

@@ -11,6 +11,22 @@ module ContextDev
           )
         end
 
+      # Cache outcome for this response. Composite responses are hits only when every
+      # cache-controlled fetch contributing to the output was a hit; age_ms is the
+      # oldest contributing hit.
+      sig do
+        returns(ContextDev::Models::AIExtractProductsResponse::CacheMetadata)
+      end
+      attr_reader :cache_metadata
+
+      sig do
+        params(
+          cache_metadata:
+            ContextDev::Models::AIExtractProductsResponse::CacheMetadata::OrHash
+        ).void
+      end
+      attr_writer :cache_metadata
+
       # Metadata about the API key used for the request. Included in every response
       # whenever a valid API key is provided, even when the response status is not 200.
       sig do
@@ -50,6 +66,8 @@ module ContextDev
 
       sig do
         params(
+          cache_metadata:
+            ContextDev::Models::AIExtractProductsResponse::CacheMetadata::OrHash,
           key_metadata:
             ContextDev::Models::AIExtractProductsResponse::KeyMetadata::OrHash,
           products:
@@ -59,6 +77,10 @@ module ContextDev
         ).returns(T.attached_class)
       end
       def self.new(
+        # Cache outcome for this response. Composite responses are hits only when every
+        # cache-controlled fetch contributing to the output was a hit; age_ms is the
+        # oldest contributing hit.
+        cache_metadata:,
         # Metadata about the API key used for the request. Included in every response
         # whenever a valid API key is provided, even when the response status is not 200.
         key_metadata: nil,
@@ -70,6 +92,8 @@ module ContextDev
       sig do
         override.returns(
           {
+            cache_metadata:
+              ContextDev::Models::AIExtractProductsResponse::CacheMetadata,
             key_metadata:
               ContextDev::Models::AIExtractProductsResponse::KeyMetadata,
             products:
@@ -80,6 +104,101 @@ module ContextDev
       def to_hash
       end
 
+      class CacheMetadata < ContextDev::Internal::Type::BaseModel
+        OrHash =
+          T.type_alias do
+            T.any(
+              ContextDev::Models::AIExtractProductsResponse::CacheMetadata,
+              ContextDev::Internal::AnyHash
+            )
+          end
+
+        # Age of the cached data in milliseconds. Zero for miss and zdr responses.
+        sig { returns(Integer) }
+        attr_accessor :age_ms
+
+        # Whether the response was served from cache, required fresh work, or honored
+        # zero-data-retention cache bypass.
+        sig do
+          returns(
+            ContextDev::Models::AIExtractProductsResponse::CacheMetadata::Status::TaggedSymbol
+          )
+        end
+        attr_accessor :status
+
+        # Cache outcome for this response. Composite responses are hits only when every
+        # cache-controlled fetch contributing to the output was a hit; age_ms is the
+        # oldest contributing hit.
+        sig do
+          params(
+            age_ms: Integer,
+            status:
+              ContextDev::Models::AIExtractProductsResponse::CacheMetadata::Status::OrSymbol
+          ).returns(T.attached_class)
+        end
+        def self.new(
+          # Age of the cached data in milliseconds. Zero for miss and zdr responses.
+          age_ms:,
+          # Whether the response was served from cache, required fresh work, or honored
+          # zero-data-retention cache bypass.
+          status:
+        )
+        end
+
+        sig do
+          override.returns(
+            {
+              age_ms: Integer,
+              status:
+                ContextDev::Models::AIExtractProductsResponse::CacheMetadata::Status::TaggedSymbol
+            }
+          )
+        end
+        def to_hash
+        end
+
+        # Whether the response was served from cache, required fresh work, or honored
+        # zero-data-retention cache bypass.
+        module Status
+          extend ContextDev::Internal::Type::Enum
+
+          TaggedSymbol =
+            T.type_alias do
+              T.all(
+                Symbol,
+                ContextDev::Models::AIExtractProductsResponse::CacheMetadata::Status
+              )
+            end
+          OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+          HIT =
+            T.let(
+              :hit,
+              ContextDev::Models::AIExtractProductsResponse::CacheMetadata::Status::TaggedSymbol
+            )
+          MISS =
+            T.let(
+              :miss,
+              ContextDev::Models::AIExtractProductsResponse::CacheMetadata::Status::TaggedSymbol
+            )
+          ZDR =
+            T.let(
+              :zdr,
+              ContextDev::Models::AIExtractProductsResponse::CacheMetadata::Status::TaggedSymbol
+            )
+
+          sig do
+            override.returns(
+              T::Array[
+                ContextDev::Models::AIExtractProductsResponse::CacheMetadata::Status::TaggedSymbol
+              ]
+            )
+          end
+          def self.values
+          end
+        end
+      end
+
       class KeyMetadata < ContextDev::Internal::Type::BaseModel
         OrHash =
           T.type_alias do

data/rbi/context_dev/models/batch_get_results_response.rbi

@@ -130,6 +130,24 @@ module ContextDev
               )
             end
 
+          # Cache outcome for this response. Composite responses are hits only when every
+          # cache-controlled fetch contributing to the output was a hit; age_ms is the
+          # oldest contributing hit.
+          sig do
+            returns(
+              ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata
+            )
+          end
+          attr_reader :cache_metadata
+
+          sig do
+            params(
+              cache_metadata:
+                ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::OrHash
+            ).void
+          end
+          attr_writer :cache_metadata
+
           # URL the content was read from, after redirects.
           sig { returns(String) }
           attr_accessor :final_url
@@ -202,6 +220,8 @@ module ContextDev
           # A page the batch fetched successfully.
           sig do
             params(
+              cache_metadata:
+                ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::OrHash,
               final_url: String,
               http_status: T.nilable(Integer),
               metadata:
@@ -216,6 +236,10 @@ module ContextDev
             ).returns(T.attached_class)
           end
           def self.new(
+            # Cache outcome for this response. Composite responses are hits only when every
+            # cache-controlled fetch contributing to the output was a hit; age_ms is the
+            # oldest contributing hit.
+            cache_metadata:,
             # URL the content was read from, after redirects.
             final_url:,
             # HTTP status of the final response, when known.
@@ -244,6 +268,8 @@ module ContextDev
           sig do
             override.returns(
               {
+                cache_metadata:
+                  ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata,
                 final_url: String,
                 http_status: T.nilable(Integer),
                 metadata:
@@ -261,6 +287,101 @@ module ContextDev
           def to_hash
           end
 
+          class CacheMetadata < ContextDev::Internal::Type::BaseModel
+            OrHash =
+              T.type_alias do
+                T.any(
+                  ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata,
+                  ContextDev::Internal::AnyHash
+                )
+              end
+
+            # Age of the cached data in milliseconds. Zero for miss and zdr responses.
+            sig { returns(Integer) }
+            attr_accessor :age_ms
+
+            # Whether the response was served from cache, required fresh work, or honored
+            # zero-data-retention cache bypass.
+            sig do
+              returns(
+                ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::Status::TaggedSymbol
+              )
+            end
+            attr_accessor :status
+
+            # Cache outcome for this response. Composite responses are hits only when every
+            # cache-controlled fetch contributing to the output was a hit; age_ms is the
+            # oldest contributing hit.
+            sig do
+              params(
+                age_ms: Integer,
+                status:
+                  ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::Status::OrSymbol
+              ).returns(T.attached_class)
+            end
+            def self.new(
+              # Age of the cached data in milliseconds. Zero for miss and zdr responses.
+              age_ms:,
+              # Whether the response was served from cache, required fresh work, or honored
+              # zero-data-retention cache bypass.
+              status:
+            )
+            end
+
+            sig do
+              override.returns(
+                {
+                  age_ms: Integer,
+                  status:
+                    ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::Status::TaggedSymbol
+                }
+              )
+            end
+            def to_hash
+            end
+
+            # Whether the response was served from cache, required fresh work, or honored
+            # zero-data-retention cache bypass.
+            module Status
+              extend ContextDev::Internal::Type::Enum
+
+              TaggedSymbol =
+                T.type_alias do
+                  T.all(
+                    Symbol,
+                    ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::Status
+                  )
+                end
+              OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+              HIT =
+                T.let(
+                  :hit,
+                  ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::Status::TaggedSymbol
+                )
+              MISS =
+                T.let(
+                  :miss,
+                  ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::Status::TaggedSymbol
+                )
+              ZDR =
+                T.let(
+                  :zdr,
+                  ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::Status::TaggedSymbol
+                )
+
+              sig do
+                override.returns(
+                  T::Array[
+                    ContextDev::Models::BatchGetResultsResponse::Data::Ok::CacheMetadata::Status::TaggedSymbol
+                  ]
+                )
+              end
+              def self.values
+              end
+            end
+          end
+
           class Metadata < ContextDev::Internal::Type::BaseModel
             OrHash =
               T.type_alias do

data/rbi/context_dev/models/batch_submit_response.rbi

@@ -15,6 +15,20 @@ module ContextDev
       sig { returns(String) }
       attr_accessor :id
 
+      # Cache outcome for this response. Composite responses are hits only when every
+      # cache-controlled fetch contributing to the output was a hit; age_ms is the
+      # oldest contributing hit.
+      sig { returns(ContextDev::Models::BatchSubmitResponse::CacheMetadata) }
+      attr_reader :cache_metadata
+
+      sig do
+        params(
+          cache_metadata:
+            ContextDev::Models::BatchSubmitResponse::CacheMetadata::OrHash
+        ).void
+      end
+      attr_writer :cache_metadata
+
       # The crawl controls as submitted, so the limits requested can be compared against
       # what the crawl reached.
       sig { returns(T.nilable(ContextDev::CrawlControls)) }
@@ -98,6 +112,8 @@ module ContextDev
       sig do
         params(
           id: String,
+          cache_metadata:
+            ContextDev::Models::BatchSubmitResponse::CacheMetadata::OrHash,
           crawl: T.nilable(ContextDev::CrawlControls::OrHash),
           created_at: String,
           credits: ContextDev::Models::BatchSubmitResponse::Credits::OrHash,
@@ -118,6 +134,10 @@ module ContextDev
       def self.new(
         # Batch ID. Poll GET /batch/{batch_id} with it.
         id:,
+        # Cache outcome for this response. Composite responses are hits only when every
+        # cache-controlled fetch contributing to the output was a hit; age_ms is the
+        # oldest contributing hit.
+        cache_metadata:,
         # The crawl controls as submitted, so the limits requested can be compared against
         # what the crawl reached.
         crawl:,
@@ -149,6 +169,8 @@ module ContextDev
         override.returns(
           {
             id: String,
+            cache_metadata:
+              ContextDev::Models::BatchSubmitResponse::CacheMetadata,
             crawl: T.nilable(ContextDev::CrawlControls),
             created_at: String,
             credits: ContextDev::Models::BatchSubmitResponse::Credits,
@@ -169,6 +191,101 @@ module ContextDev
       def to_hash
       end
 
+      class CacheMetadata < ContextDev::Internal::Type::BaseModel
+        OrHash =
+          T.type_alias do
+            T.any(
+              ContextDev::Models::BatchSubmitResponse::CacheMetadata,
+              ContextDev::Internal::AnyHash
+            )
+          end
+
+        # Age of the cached data in milliseconds. Zero for miss and zdr responses.
+        sig { returns(Integer) }
+        attr_accessor :age_ms
+
+        # Whether the response was served from cache, required fresh work, or honored
+        # zero-data-retention cache bypass.
+        sig do
+          returns(
+            ContextDev::Models::BatchSubmitResponse::CacheMetadata::Status::TaggedSymbol
+          )
+        end
+        attr_accessor :status
+
+        # Cache outcome for this response. Composite responses are hits only when every
+        # cache-controlled fetch contributing to the output was a hit; age_ms is the
+        # oldest contributing hit.
+        sig do
+          params(
+            age_ms: Integer,
+            status:
+              ContextDev::Models::BatchSubmitResponse::CacheMetadata::Status::OrSymbol
+          ).returns(T.attached_class)
+        end
+        def self.new(
+          # Age of the cached data in milliseconds. Zero for miss and zdr responses.
+          age_ms:,
+          # Whether the response was served from cache, required fresh work, or honored
+          # zero-data-retention cache bypass.
+          status:
+        )
+        end
+
+        sig do
+          override.returns(
+            {
+              age_ms: Integer,
+              status:
+                ContextDev::Models::BatchSubmitResponse::CacheMetadata::Status::TaggedSymbol
+            }
+          )
+        end
+        def to_hash
+        end
+
+        # Whether the response was served from cache, required fresh work, or honored
+        # zero-data-retention cache bypass.
+        module Status
+          extend ContextDev::Internal::Type::Enum
+
+          TaggedSymbol =
+            T.type_alias do
+              T.all(
+                Symbol,
+                ContextDev::Models::BatchSubmitResponse::CacheMetadata::Status
+              )
+            end
+          OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+          HIT =
+            T.let(
+              :hit,
+              ContextDev::Models::BatchSubmitResponse::CacheMetadata::Status::TaggedSymbol
+            )
+          MISS =
+            T.let(
+              :miss,
+              ContextDev::Models::BatchSubmitResponse::CacheMetadata::Status::TaggedSymbol
+            )
+          ZDR =
+            T.let(
+              :zdr,
+              ContextDev::Models::BatchSubmitResponse::CacheMetadata::Status::TaggedSymbol
+            )
+
+          sig do
+            override.returns(
+              T::Array[
+                ContextDev::Models::BatchSubmitResponse::CacheMetadata::Status::TaggedSymbol
+              ]
+            )
+          end
+          def self.values
+          end
+        end
+      end
+
       class Credits < ContextDev::Internal::Type::BaseModel
         OrHash =
           T.type_alias do