context.dev 2.10.0 → 2.12.0

data/rbi/context_dev/models/web_web_crawl_md_params.rbi

@@ -150,7 +150,9 @@ module ContextDev
       sig { params(timeout_ms: Integer).void }
       attr_writer :timeout_ms
 
-      # Regex pattern. Only URLs matching this pattern will be followed and scraped.
+      # Regex pattern. Only URLs matching this pattern will be followed and scraped. An
+      # automatic prefix scope in the form ^<starting URL> follows a redirect of the
+      # starting page.
       sig { returns(T.nilable(String)) }
       attr_reader :url_regex
 
@@ -165,8 +167,8 @@ module ContextDev
       sig { params(use_main_content_only: T::Boolean).void }
       attr_writer :use_main_content_only
 
-      # Optional browser wait time in milliseconds after initial page load for each
-      # crawled page. Min: 0. Max: 30000 (30 seconds).
+      # Browser wait time in milliseconds after initial page load for each crawled page.
+      # Defaults to 3500 (3.5 seconds). Min: 0. Max: 30000 (30 seconds).
       sig { returns(T.nilable(Integer)) }
       attr_reader :wait_for_ms
 
@@ -263,13 +265,15 @@ module ContextDev
         # than this value, it will be aborted with a 408 status code. Maximum allowed
         # value is 300000ms (5 minutes).
         timeout_ms: nil,
-        # Regex pattern. Only URLs matching this pattern will be followed and scraped.
+        # Regex pattern. Only URLs matching this pattern will be followed and scraped. An
+        # automatic prefix scope in the form ^<starting URL> follows a redirect of the
+        # starting page.
         url_regex: nil,
         # Extract only the main content, stripping headers, footers, sidebars, and
         # navigation
         use_main_content_only: nil,
-        # Optional browser wait time in milliseconds after initial page load for each
-        # crawled page. Min: 0. Max: 30000 (30 seconds).
+        # Browser wait time in milliseconds after initial page load for each crawled page.
+        # Defaults to 3500 (3.5 seconds). Min: 0. Max: 30000 (30 seconds).
         wait_for_ms: nil,
         # Set to enabled to bypass shared caches and omit request and response content
         # from retained usage logs. Requires zero data retention to be enabled for your

data/rbi/context_dev/models/web_web_scrape_html_params.rbi

@@ -27,7 +27,8 @@ module ContextDev
             T::Array[
               T.any(
                 ContextDev::WebWebScrapeHTMLParams::Action::Wait,
-                ContextDev::WebWebScrapeHTMLParams::Action::Perform
+                ContextDev::WebWebScrapeHTMLParams::Action::Perform,
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll
               )
             ]
           )
@@ -155,7 +156,8 @@ module ContextDev
               T::Array[
                 T.any(
                   ContextDev::WebWebScrapeHTMLParams::Action::Wait::OrHash,
-                  ContextDev::WebWebScrapeHTMLParams::Action::Perform::OrHash
+                  ContextDev::WebWebScrapeHTMLParams::Action::Perform::OrHash,
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::OrHash
                 )
               ]
             ),
@@ -242,7 +244,8 @@ module ContextDev
                 T::Array[
                   T.any(
                     ContextDev::WebWebScrapeHTMLParams::Action::Wait,
-                    ContextDev::WebWebScrapeHTMLParams::Action::Perform
+                    ContextDev::WebWebScrapeHTMLParams::Action::Perform,
+                    ContextDev::WebWebScrapeHTMLParams::Action::Scroll
                   )
                 ]
               ),
@@ -275,7 +278,8 @@ module ContextDev
           T.type_alias do
             T.any(
               ContextDev::WebWebScrapeHTMLParams::Action::Wait,
-              ContextDev::WebWebScrapeHTMLParams::Action::Perform
+              ContextDev::WebWebScrapeHTMLParams::Action::Perform,
+              ContextDev::WebWebScrapeHTMLParams::Action::Scroll
             )
           end
 
@@ -331,6 +335,216 @@ module ContextDev
           end
         end
 
+        class Scroll < ContextDev::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll,
+                ContextDev::Internal::AnyHash
+              )
+            end
+
+          sig { returns(Symbol) }
+          attr_accessor :do_
+
+          # Pixels per scroll, one visible viewport, or the current scroll boundary.
+          # Defaults to viewport.
+          sig do
+            returns(
+              T.nilable(
+                T.any(
+                  Integer,
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::OrSymbol
+                )
+              )
+            )
+          end
+          attr_reader :amount
+
+          sig do
+            params(
+              amount:
+                T.any(
+                  Integer,
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::OrSymbol
+                )
+            ).void
+          end
+          attr_writer :amount
+
+          # CSS selector for the first matching scroll container. Defaults to the page.
+          sig { returns(T.nilable(String)) }
+          attr_reader :container
+
+          sig { params(container: String).void }
+          attr_writer :container
+
+          # Direction to scroll. Defaults to down.
+          sig do
+            returns(
+              T.nilable(
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::OrSymbol
+              )
+            )
+          end
+          attr_reader :direction
+
+          sig do
+            params(
+              direction:
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::OrSymbol
+            ).void
+          end
+          attr_writer :direction
+
+          # Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
+          # changing. Defaults to 1.
+          sig { returns(T.nilable(Integer)) }
+          attr_reader :max_scrolls
+
+          sig { params(max_scrolls: Integer).void }
+          attr_writer :max_scrolls
+
+          # Scroll the page or a selected scrollable container, waiting adaptively for
+          # content and dimensions to settle after each iteration.
+          sig do
+            params(
+              amount:
+                T.any(
+                  Integer,
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::OrSymbol
+                ),
+              container: String,
+              direction:
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::OrSymbol,
+              max_scrolls: Integer,
+              do_: Symbol
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            # Pixels per scroll, one visible viewport, or the current scroll boundary.
+            # Defaults to viewport.
+            amount: nil,
+            # CSS selector for the first matching scroll container. Defaults to the page.
+            container: nil,
+            # Direction to scroll. Defaults to down.
+            direction: nil,
+            # Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
+            # changing. Defaults to 1.
+            max_scrolls: nil,
+            do_: :scroll
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                do_: Symbol,
+                amount:
+                  T.any(
+                    Integer,
+                    ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::OrSymbol
+                  ),
+                container: String,
+                direction:
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::OrSymbol,
+                max_scrolls: Integer
+              }
+            )
+          end
+          def to_hash
+          end
+
+          # Pixels per scroll, one visible viewport, or the current scroll boundary.
+          # Defaults to viewport.
+          module Amount
+            extend ContextDev::Internal::Type::Union
+
+            Variants =
+              T.type_alias do
+                T.any(
+                  Integer,
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::TaggedSymbol
+                )
+              end
+
+            sig do
+              override.returns(
+                T::Array[
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::Variants
+                ]
+              )
+            end
+            def self.variants
+            end
+
+            TaggedSymbol =
+              T.type_alias do
+                T.all(
+                  Symbol,
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount
+                )
+              end
+            OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+            VIEWPORT =
+              T.let(
+                :viewport,
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::TaggedSymbol
+              )
+            MAX =
+              T.let(
+                :max,
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Amount::TaggedSymbol
+              )
+          end
+
+          # Direction to scroll. Defaults to down.
+          module Direction
+            extend ContextDev::Internal::Type::Enum
+
+            TaggedSymbol =
+              T.type_alias do
+                T.all(
+                  Symbol,
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction
+                )
+              end
+            OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+            UP =
+              T.let(
+                :up,
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::TaggedSymbol
+              )
+            DOWN =
+              T.let(
+                :down,
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::TaggedSymbol
+              )
+            LEFT =
+              T.let(
+                :left,
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::TaggedSymbol
+              )
+            RIGHT =
+              T.let(
+                :right,
+                ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::TaggedSymbol
+              )
+
+            sig do
+              override.returns(
+                T::Array[
+                  ContextDev::WebWebScrapeHTMLParams::Action::Scroll::Direction::TaggedSymbol
+                ]
+              )
+            end
+            def self.values
+            end
+          end
+        end
+
         sig do
           override.returns(
             T::Array[ContextDev::WebWebScrapeHTMLParams::Action::Variants]

data/rbi/context_dev/models/web_web_scrape_images_params.rbi

@@ -27,7 +27,8 @@ module ContextDev
             T::Array[
               T.any(
                 ContextDev::WebWebScrapeImagesParams::Action::Wait,
-                ContextDev::WebWebScrapeImagesParams::Action::Perform
+                ContextDev::WebWebScrapeImagesParams::Action::Perform,
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll
               )
             ]
           )
@@ -105,7 +106,8 @@ module ContextDev
               T::Array[
                 T.any(
                   ContextDev::WebWebScrapeImagesParams::Action::Wait::OrHash,
-                  ContextDev::WebWebScrapeImagesParams::Action::Perform::OrHash
+                  ContextDev::WebWebScrapeImagesParams::Action::Perform::OrHash,
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::OrHash
                 )
               ]
             ),
@@ -166,7 +168,8 @@ module ContextDev
                 T::Array[
                   T.any(
                     ContextDev::WebWebScrapeImagesParams::Action::Wait,
-                    ContextDev::WebWebScrapeImagesParams::Action::Perform
+                    ContextDev::WebWebScrapeImagesParams::Action::Perform,
+                    ContextDev::WebWebScrapeImagesParams::Action::Scroll
                   )
                 ]
               ),
@@ -194,7 +197,8 @@ module ContextDev
           T.type_alias do
             T.any(
               ContextDev::WebWebScrapeImagesParams::Action::Wait,
-              ContextDev::WebWebScrapeImagesParams::Action::Perform
+              ContextDev::WebWebScrapeImagesParams::Action::Perform,
+              ContextDev::WebWebScrapeImagesParams::Action::Scroll
             )
           end
 
@@ -250,6 +254,216 @@ module ContextDev
           end
         end
 
+        class Scroll < ContextDev::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll,
+                ContextDev::Internal::AnyHash
+              )
+            end
+
+          sig { returns(Symbol) }
+          attr_accessor :do_
+
+          # Pixels per scroll, one visible viewport, or the current scroll boundary.
+          # Defaults to viewport.
+          sig do
+            returns(
+              T.nilable(
+                T.any(
+                  Integer,
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::OrSymbol
+                )
+              )
+            )
+          end
+          attr_reader :amount
+
+          sig do
+            params(
+              amount:
+                T.any(
+                  Integer,
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::OrSymbol
+                )
+            ).void
+          end
+          attr_writer :amount
+
+          # CSS selector for the first matching scroll container. Defaults to the page.
+          sig { returns(T.nilable(String)) }
+          attr_reader :container
+
+          sig { params(container: String).void }
+          attr_writer :container
+
+          # Direction to scroll. Defaults to down.
+          sig do
+            returns(
+              T.nilable(
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::OrSymbol
+              )
+            )
+          end
+          attr_reader :direction
+
+          sig do
+            params(
+              direction:
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::OrSymbol
+            ).void
+          end
+          attr_writer :direction
+
+          # Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
+          # changing. Defaults to 1.
+          sig { returns(T.nilable(Integer)) }
+          attr_reader :max_scrolls
+
+          sig { params(max_scrolls: Integer).void }
+          attr_writer :max_scrolls
+
+          # Scroll the page or a selected scrollable container, waiting adaptively for
+          # content and dimensions to settle after each iteration.
+          sig do
+            params(
+              amount:
+                T.any(
+                  Integer,
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::OrSymbol
+                ),
+              container: String,
+              direction:
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::OrSymbol,
+              max_scrolls: Integer,
+              do_: Symbol
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            # Pixels per scroll, one visible viewport, or the current scroll boundary.
+            # Defaults to viewport.
+            amount: nil,
+            # CSS selector for the first matching scroll container. Defaults to the page.
+            container: nil,
+            # Direction to scroll. Defaults to down.
+            direction: nil,
+            # Maximum scroll iterations. Stops early when scrolling and scrollable extent stop
+            # changing. Defaults to 1.
+            max_scrolls: nil,
+            do_: :scroll
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                do_: Symbol,
+                amount:
+                  T.any(
+                    Integer,
+                    ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::OrSymbol
+                  ),
+                container: String,
+                direction:
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::OrSymbol,
+                max_scrolls: Integer
+              }
+            )
+          end
+          def to_hash
+          end
+
+          # Pixels per scroll, one visible viewport, or the current scroll boundary.
+          # Defaults to viewport.
+          module Amount
+            extend ContextDev::Internal::Type::Union
+
+            Variants =
+              T.type_alias do
+                T.any(
+                  Integer,
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::TaggedSymbol
+                )
+              end
+
+            sig do
+              override.returns(
+                T::Array[
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::Variants
+                ]
+              )
+            end
+            def self.variants
+            end
+
+            TaggedSymbol =
+              T.type_alias do
+                T.all(
+                  Symbol,
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount
+                )
+              end
+            OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+            VIEWPORT =
+              T.let(
+                :viewport,
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::TaggedSymbol
+              )
+            MAX =
+              T.let(
+                :max,
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Amount::TaggedSymbol
+              )
+          end
+
+          # Direction to scroll. Defaults to down.
+          module Direction
+            extend ContextDev::Internal::Type::Enum
+
+            TaggedSymbol =
+              T.type_alias do
+                T.all(
+                  Symbol,
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction
+                )
+              end
+            OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+            UP =
+              T.let(
+                :up,
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::TaggedSymbol
+              )
+            DOWN =
+              T.let(
+                :down,
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::TaggedSymbol
+              )
+            LEFT =
+              T.let(
+                :left,
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::TaggedSymbol
+              )
+            RIGHT =
+              T.let(
+                :right,
+                ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::TaggedSymbol
+              )
+
+            sig do
+              override.returns(
+                T::Array[
+                  ContextDev::WebWebScrapeImagesParams::Action::Scroll::Direction::TaggedSymbol
+                ]
+              )
+            end
+            def self.values
+            end
+          end
+        end
+
         sig do
           override.returns(
             T::Array[ContextDev::WebWebScrapeImagesParams::Action::Variants]