context.dev 2.5.0 → 2.7.0

data/rbi/context_dev/models/crawl_controls.rbi

@@ -0,0 +1,252 @@
+# typed: strong
+
+module ContextDev
+  module Models
+    class CrawlControls < ContextDev::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(ContextDev::CrawlControls, ContextDev::Internal::AnyHash)
+        end
+
+      # Whether links to subdomains were followed. Always false for a sitemap crawl.
+      sig { returns(T::Boolean) }
+      attr_accessor :follow_subdomains
+
+      # Link depth limit. Always 0 for a sitemap crawl, which never follows links off
+      # its URLs; null when a `start_url` crawl set no limit.
+      sig { returns(T.nilable(Integer)) }
+      attr_accessor :max_depth
+
+      # The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs
+      # its sitemap actually lists, up to this many, so `input.reserved` is often lower.
+      sig { returns(Integer) }
+      attr_accessor :max_pages
+
+      # Where the crawl started.
+      sig { returns(ContextDev::CrawlControls::Source::Variants) }
+      attr_accessor :source
+
+      # RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
+      sig { returns(T.nilable(String)) }
+      attr_accessor :url_pattern
+
+      # The crawl controls as submitted, so the limits requested can be compared against
+      # what the crawl reached.
+      sig do
+        params(
+          follow_subdomains: T::Boolean,
+          max_depth: T.nilable(Integer),
+          max_pages: Integer,
+          source:
+            T.any(
+              ContextDev::CrawlControls::Source::UnionMember0::OrHash,
+              ContextDev::CrawlControls::Source::UnionMember1::OrHash
+            ),
+          url_pattern: T.nilable(String)
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # Whether links to subdomains were followed. Always false for a sitemap crawl.
+        follow_subdomains:,
+        # Link depth limit. Always 0 for a sitemap crawl, which never follows links off
+        # its URLs; null when a `start_url` crawl set no limit.
+        max_depth:,
+        # The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs
+        # its sitemap actually lists, up to this many, so `input.reserved` is often lower.
+        max_pages:,
+        # Where the crawl started.
+        source:,
+        # RE2 pattern URLs had to match to be crawled. Null when the crawl set none.
+        url_pattern:
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            follow_subdomains: T::Boolean,
+            max_depth: T.nilable(Integer),
+            max_pages: Integer,
+            source: ContextDev::CrawlControls::Source::Variants,
+            url_pattern: T.nilable(String)
+          }
+        )
+      end
+      def to_hash
+      end
+
+      # Where the crawl started.
+      module Source
+        extend ContextDev::Internal::Type::Union
+
+        Variants =
+          T.type_alias do
+            T.any(
+              ContextDev::CrawlControls::Source::UnionMember0,
+              ContextDev::CrawlControls::Source::UnionMember1
+            )
+          end
+
+        class UnionMember0 < ContextDev::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                ContextDev::CrawlControls::Source::UnionMember0,
+                ContextDev::Internal::AnyHash
+              )
+            end
+
+          sig do
+            returns(
+              ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol
+            )
+          end
+          attr_accessor :type
+
+          # Page the crawl started from.
+          sig { returns(String) }
+          attr_accessor :url
+
+          sig do
+            params(
+              type:
+                ContextDev::CrawlControls::Source::UnionMember0::Type::OrSymbol,
+              url: String
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            type:,
+            # Page the crawl started from.
+            url:
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                type:
+                  ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol,
+                url: String
+              }
+            )
+          end
+          def to_hash
+          end
+
+          module Type
+            extend ContextDev::Internal::Type::Enum
+
+            TaggedSymbol =
+              T.type_alias do
+                T.all(
+                  Symbol,
+                  ContextDev::CrawlControls::Source::UnionMember0::Type
+                )
+              end
+            OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+            START_URL =
+              T.let(
+                :start_url,
+                ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol
+              )
+
+            sig do
+              override.returns(
+                T::Array[
+                  ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol
+                ]
+              )
+            end
+            def self.values
+            end
+          end
+        end
+
+        class UnionMember1 < ContextDev::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                ContextDev::CrawlControls::Source::UnionMember1,
+                ContextDev::Internal::AnyHash
+              )
+            end
+
+          # Domain whose sitemap supplied the pages.
+          sig { returns(String) }
+          attr_accessor :domain
+
+          sig do
+            returns(
+              ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol
+            )
+          end
+          attr_accessor :type
+
+          sig do
+            params(
+              domain: String,
+              type:
+                ContextDev::CrawlControls::Source::UnionMember1::Type::OrSymbol
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            # Domain whose sitemap supplied the pages.
+            domain:,
+            type:
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                domain: String,
+                type:
+                  ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol
+              }
+            )
+          end
+          def to_hash
+          end
+
+          module Type
+            extend ContextDev::Internal::Type::Enum
+
+            TaggedSymbol =
+              T.type_alias do
+                T.all(
+                  Symbol,
+                  ContextDev::CrawlControls::Source::UnionMember1::Type
+                )
+              end
+            OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+            SITEMAP =
+              T.let(
+                :sitemap,
+                ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol
+              )
+
+            sig do
+              override.returns(
+                T::Array[
+                  ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol
+                ]
+              )
+            end
+            def self.values
+            end
+          end
+        end
+
+        sig do
+          override.returns(
+            T::Array[ContextDev::CrawlControls::Source::Variants]
+          )
+        end
+        def self.variants
+        end
+      end
+    end
+  end
+end

data/rbi/context_dev/models/failure.rbi

@@ -0,0 +1,35 @@
+# typed: strong
+
+module ContextDev
+  module Models
+    class Failure < ContextDev::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(ContextDev::Failure, ContextDev::Internal::AnyHash)
+        end
+
+      # Why the batch itself stopped.
+      sig { returns(String) }
+      attr_accessor :code
+
+      # Human-readable explanation.
+      sig { returns(String) }
+      attr_accessor :message
+
+      # A failure of the batch as a whole, distinct from the per-page failures in
+      # `page_errors`.
+      sig { params(code: String, message: String).returns(T.attached_class) }
+      def self.new(
+        # Why the batch itself stopped.
+        code:,
+        # Human-readable explanation.
+        message:
+      )
+      end
+
+      sig { override.returns({ code: String, message: String }) }
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/context_dev/models/intake.rbi

@@ -0,0 +1,88 @@
+# typed: strong
+
+module ContextDev
+  module Models
+    class Intake < ContextDev::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(ContextDev::Intake, ContextDev::Internal::AnyHash)
+        end
+
+      # URLs dropped before reserving because another entry resolved to the same page.
+      # Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than
+      # once.
+      sig { returns(Integer) }
+      attr_accessor :duplicates
+
+      # URLs from your list rejected as unusable; the same ones are itemised in
+      # `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable
+      # page is rejected outright with a 400 rather than accepted with an empty list.
+      sig { returns(T.nilable(Integer)) }
+      attr_accessor :invalid
+
+      # Pages credits were reserved for. Everything else — progress, the refund, the
+      # completion percentage — is measured against this.
+      sig { returns(Integer) }
+      attr_accessor :reserved
+
+      # Whether `reserved` is an upper bound the batch may finish under. True only for a
+      # crawl that follows links, whose reachable page count is unknowable until it
+      # runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact
+      # page count.
+      sig { returns(T::Boolean) }
+      attr_accessor :reserved_is_ceiling
+
+      # URLs in the list you sent, before validation and de-duplication. Null for a
+      # crawl, which is given a source rather than a list.
+      sig { returns(T.nilable(Integer)) }
+      attr_accessor :submitted
+
+      # What submission took in, and what it charged for.
+      sig do
+        params(
+          duplicates: Integer,
+          invalid: T.nilable(Integer),
+          reserved: Integer,
+          reserved_is_ceiling: T::Boolean,
+          submitted: T.nilable(Integer)
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # URLs dropped before reserving because another entry resolved to the same page.
+        # Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than
+        # once.
+        duplicates:,
+        # URLs from your list rejected as unusable; the same ones are itemised in
+        # `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable
+        # page is rejected outright with a 400 rather than accepted with an empty list.
+        invalid:,
+        # Pages credits were reserved for. Everything else — progress, the refund, the
+        # completion percentage — is measured against this.
+        reserved:,
+        # Whether `reserved` is an upper bound the batch may finish under. True only for a
+        # crawl that follows links, whose reachable page count is unknowable until it
+        # runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact
+        # page count.
+        reserved_is_ceiling:,
+        # URLs in the list you sent, before validation and de-duplication. Null for a
+        # crawl, which is given a source rather than a list.
+        submitted:
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            duplicates: Integer,
+            invalid: T.nilable(Integer),
+            reserved: Integer,
+            reserved_is_ceiling: T::Boolean,
+            submitted: T.nilable(Integer)
+          }
+        )
+      end
+      def to_hash
+      end
+    end
+  end
+end