context.dev 1.35.0 → 2.0.0

data/lib/context_dev/models/web_web_crawl_md_params.rb

@@ -13,6 +13,14 @@ module ContextDev
       #   @return [String]
       required :url, String
 
+      # @!attribute country
+      #   Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
+      #   residential proxy exit location. Must be one of Context.dev's supported
+      #   countries. When provided, Context.dev fetches the target page from that country.
+      #
+      #   @return [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country, nil]
+      optional :country, enum: -> { ContextDev::WebWebCrawlMdParams::Country }
+
       # @!attribute exclude_selectors
       #   CSS selectors to remove before each crawled page is converted to Markdown.
       #   Applied after includeSelectors. Exclusion takes precedence: an element matching
@@ -84,6 +92,14 @@ module ContextDev
       #   @return [ContextDev::Models::WebWebCrawlMdParams::Pdf, nil]
       optional :pdf, -> { ContextDev::WebWebCrawlMdParams::Pdf }
 
+      # @!attribute settle_animations
+      #   When true, waits briefly for CSS and transition animations to settle before
+      #   extracting each crawled page. Defaults to false. This adds a bit of latency in
+      #   exchange for more stable output on animated pages.
+      #
+      #   @return [Boolean, nil]
+      optional :settle_animations, ContextDev::Internal::Type::Boolean, api_name: :settleAnimations
+
       # @!attribute shorten_base64_images
       #   Truncate base64-encoded image data in the Markdown output
       #
@@ -127,12 +143,14 @@ module ContextDev
       #   @return [Integer, nil]
       optional :wait_for_ms, Integer, api_name: :waitForMs
 
-      # @!method initialize(url:, exclude_selectors: nil, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, include_selectors: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, shorten_base64_images: nil, stop_after_ms: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
+      # @!method initialize(url:, country: nil, exclude_selectors: nil, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, include_selectors: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, settle_animations: nil, shorten_base64_images: nil, stop_after_ms: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
       #   Some parameter documentations has been truncated, see
       #   {ContextDev::Models::WebWebCrawlMdParams} for more details.
       #
       #   @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
       #
+      #   @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r
+      #
       #   @param exclude_selectors [Array<String>] CSS selectors to remove before each crawled page is converted to Markdown. Appli
       #
       #   @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex
@@ -153,6 +171,8 @@ module ContextDev
       #
       #   @param pdf [ContextDev::Models::WebWebCrawlMdParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu
       #
+      #   @param settle_animations [Boolean] When true, waits briefly for CSS and transition animations to settle before extr
+      #
       #   @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
       #
       #   @param stop_after_ms [Integer] Soft time budget for the crawl in milliseconds. After each scrape, the crawler c
@@ -167,6 +187,221 @@ module ContextDev
       #
       #   @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
 
+      # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev
+      # residential proxy exit location. Must be one of Context.dev's supported
+      # countries. When provided, Context.dev fetches the target page from that country.
+      module Country
+        extend ContextDev::Internal::Type::Enum
+
+        AD = :ad
+        AE = :ae
+        AF = :af
+        AG = :ag
+        AI = :ai
+        AL = :al
+        AM = :am
+        AO = :ao
+        AR = :ar
+        AT = :at
+        AU = :au
+        AW = :aw
+        AZ = :az
+        BA = :ba
+        BB = :bb
+        BD = :bd
+        BE = :be
+        BF = :bf
+        BG = :bg
+        BH = :bh
+        BI = :bi
+        BJ = :bj
+        BM = :bm
+        BN = :bn
+        BO = :bo
+        BQ = :bq
+        BR = :br
+        BS = :bs
+        BW = :bw
+        BY = :by
+        BZ = :bz
+        CA = :ca
+        CD = :cd
+        CF = :cf
+        CG = :cg
+        CH = :ch
+        CI = :ci
+        CL = :cl
+        CM = :cm
+        CN = :cn
+        CO = :co
+        CR = :cr
+        CV = :cv
+        CW = :cw
+        CY = :cy
+        CZ = :cz
+        DE = :de
+        DJ = :dj
+        DK = :dk
+        DM = :dm
+        DO = :do
+        DZ = :dz
+        EC = :ec
+        EE = :ee
+        EG = :eg
+        ES = :es
+        ET = :et
+        FI = :fi
+        FJ = :fj
+        FR = :fr
+        GA = :ga
+        GB = :gb
+        GD = :gd
+        GE = :ge
+        GF = :gf
+        GG = :gg
+        GH = :gh
+        GM = :gm
+        GN = :gn
+        GP = :gp
+        GQ = :gq
+        GR = :gr
+        GT = :gt
+        GU = :gu
+        GW = :gw
+        GY = :gy
+        HK = :hk
+        HN = :hn
+        HR = :hr
+        HT = :ht
+        HU = :hu
+        ID = :id
+        IE = :ie
+        IL = :il
+        IM = :im
+        IN = :in
+        IQ = :iq
+        IR = :ir
+        IS = :is
+        IT = :it
+        JE = :je
+        JM = :jm
+        JO = :jo
+        JP = :jp
+        KE = :ke
+        KG = :kg
+        KH = :kh
+        KN = :kn
+        KR = :kr
+        KW = :kw
+        KY = :ky
+        KZ = :kz
+        LA = :la
+        LB = :lb
+        LC = :lc
+        LK = :lk
+        LR = :lr
+        LS = :ls
+        LT = :lt
+        LU = :lu
+        LV = :lv
+        LY = :ly
+        MA = :ma
+        MC = :mc
+        MD = :md
+        ME = :me
+        MF = :mf
+        MG = :mg
+        MK = :mk
+        ML = :ml
+        MM = :mm
+        MN = :mn
+        MO = :mo
+        MQ = :mq
+        MR = :mr
+        MT = :mt
+        MU = :mu
+        MV = :mv
+        MW = :mw
+        MX = :mx
+        MY = :my
+        MZ = :mz
+        NA = :na
+        NC = :nc
+        NE = :ne
+        NG = :ng
+        NI = :ni
+        NL = :nl
+        NO = :no
+        NP = :np
+        NZ = :nz
+        OM = :om
+        PA = :pa
+        PE = :pe
+        PF = :pf
+        PG = :pg
+        PH = :ph
+        PK = :pk
+        PL = :pl
+        PR = :pr
+        PS = :ps
+        PT = :pt
+        PY = :py
+        QA = :qa
+        RE = :re
+        RO = :ro
+        RS = :rs
+        RU = :ru
+        RW = :rw
+        SA = :sa
+        SC = :sc
+        SD = :sd
+        SE = :se
+        SG = :sg
+        SI = :si
+        SK = :sk
+        SL = :sl
+        SM = :sm
+        SN = :sn
+        SO = :so
+        SR = :sr
+        SS = :ss
+        ST = :st
+        SV = :sv
+        SX = :sx
+        SY = :sy
+        SZ = :sz
+        TC = :tc
+        TD = :td
+        TG = :tg
+        TH = :th
+        TJ = :tj
+        TL = :tl
+        TM = :tm
+        TN = :tn
+        TR = :tr
+        TT = :tt
+        TW = :tw
+        TZ = :tz
+        UA = :ua
+        UG = :ug
+        US = :us
+        UY = :uy
+        UZ = :uz
+        VC = :vc
+        VE = :ve
+        VG = :vg
+        VI = :vi
+        VN = :vn
+        YE = :ye
+        YT = :yt
+        ZA = :za
+        ZM = :zm
+        ZW = :zw
+
+        # @!method self.values
+        #   @return [Array<Symbol>]
+      end
+
       class Pdf < ContextDev::Internal::Type::BaseModel
         # @!attribute end_
         #   Last 1-based PDF page to parse. When omitted, parsing ends at the last page.

data/lib/context_dev/models/web_web_scrape_html_params.rb

@@ -13,6 +13,13 @@ module ContextDev
       #   @return [String]
       required :url, String
 
+      # @!attribute country
+      #   Two-letter ISO 3166-1 alpha-2 country code for the website request location.
+      #   When provided, Context.dev fetches the target page from that country.
+      #
+      #   @return [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country, nil]
+      optional :country, enum: -> { ContextDev::WebWebScrapeHTMLParams::Country }
+
       # @!attribute exclude_selectors
       #   CSS selectors to remove from the result. Applied after includeSelectors.
       #   Exclusion takes precedence: an element matching both is removed. Examples:
@@ -58,6 +65,14 @@ module ContextDev
       #   @return [ContextDev::Models::WebWebScrapeHTMLParams::Pdf, nil]
       optional :pdf, -> { ContextDev::WebWebScrapeHTMLParams::Pdf }
 
+      # @!attribute settle_animations
+      #   When true, waits briefly for CSS and transition animations to settle before
+      #   extracting HTML. Defaults to false. This adds a bit of latency in exchange for
+      #   more stable output on animated pages.
+      #
+      #   @return [Boolean, nil]
+      optional :settle_animations, ContextDev::Internal::Type::Boolean
+
       # @!attribute timeout_ms
       #   Optional timeout in milliseconds for the request. If the request takes longer
       #   than this value, it will be aborted with a 408 status code. Maximum allowed
@@ -80,12 +95,14 @@ module ContextDev
       #   @return [Integer, nil]
       optional :wait_for_ms, Integer
 
-      # @!method initialize(url:, exclude_selectors: nil, headers: nil, include_frames: nil, include_selectors: nil, max_age_ms: nil, pdf: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
+      # @!method initialize(url:, country: nil, exclude_selectors: nil, headers: nil, include_frames: nil, include_selectors: nil, max_age_ms: nil, pdf: nil, settle_animations: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {})
       #   Some parameter documentations has been truncated, see
       #   {ContextDev::Models::WebWebScrapeHTMLParams} for more details.
       #
       #   @param url [String] Full URL to scrape (must include http:// or https:// protocol)
       #
+      #   @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Two-letter ISO 3166-1 alpha-2 country code for the website request location. Whe
+      #
       #   @param exclude_selectors [Array<String>] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi
       #
       #   @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob
@@ -98,6 +115,8 @@ module ContextDev
       #
       #   @param pdf [ContextDev::Models::WebWebScrapeHTMLParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu
       #
+      #   @param settle_animations [Boolean] When true, waits briefly for CSS and transition animations to settle before extr
+      #
       #   @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th
       #
       #   @param use_main_content_only [Boolean] When true, return only the page's main content in the HTML response, excluding h
@@ -106,6 +125,220 @@ module ContextDev
       #
       #   @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
 
+      # Two-letter ISO 3166-1 alpha-2 country code for the website request location.
+      # When provided, Context.dev fetches the target page from that country.
+      module Country
+        extend ContextDev::Internal::Type::Enum
+
+        AD = :ad
+        AE = :ae
+        AF = :af
+        AG = :ag
+        AI = :ai
+        AL = :al
+        AM = :am
+        AO = :ao
+        AR = :ar
+        AT = :at
+        AU = :au
+        AW = :aw
+        AZ = :az
+        BA = :ba
+        BB = :bb
+        BD = :bd
+        BE = :be
+        BF = :bf
+        BG = :bg
+        BH = :bh
+        BI = :bi
+        BJ = :bj
+        BM = :bm
+        BN = :bn
+        BO = :bo
+        BQ = :bq
+        BR = :br
+        BS = :bs
+        BW = :bw
+        BY = :by
+        BZ = :bz
+        CA = :ca
+        CD = :cd
+        CF = :cf
+        CG = :cg
+        CH = :ch
+        CI = :ci
+        CL = :cl
+        CM = :cm
+        CN = :cn
+        CO = :co
+        CR = :cr
+        CV = :cv
+        CW = :cw
+        CY = :cy
+        CZ = :cz
+        DE = :de
+        DJ = :dj
+        DK = :dk
+        DM = :dm
+        DO = :do
+        DZ = :dz
+        EC = :ec
+        EE = :ee
+        EG = :eg
+        ES = :es
+        ET = :et
+        FI = :fi
+        FJ = :fj
+        FR = :fr
+        GA = :ga
+        GB = :gb
+        GD = :gd
+        GE = :ge
+        GF = :gf
+        GG = :gg
+        GH = :gh
+        GM = :gm
+        GN = :gn
+        GP = :gp
+        GQ = :gq
+        GR = :gr
+        GT = :gt
+        GU = :gu
+        GW = :gw
+        GY = :gy
+        HK = :hk
+        HN = :hn
+        HR = :hr
+        HT = :ht
+        HU = :hu
+        ID = :id
+        IE = :ie
+        IL = :il
+        IM = :im
+        IN = :in
+        IQ = :iq
+        IR = :ir
+        IS = :is
+        IT = :it
+        JE = :je
+        JM = :jm
+        JO = :jo
+        JP = :jp
+        KE = :ke
+        KG = :kg
+        KH = :kh
+        KN = :kn
+        KR = :kr
+        KW = :kw
+        KY = :ky
+        KZ = :kz
+        LA = :la
+        LB = :lb
+        LC = :lc
+        LK = :lk
+        LR = :lr
+        LS = :ls
+        LT = :lt
+        LU = :lu
+        LV = :lv
+        LY = :ly
+        MA = :ma
+        MC = :mc
+        MD = :md
+        ME = :me
+        MF = :mf
+        MG = :mg
+        MK = :mk
+        ML = :ml
+        MM = :mm
+        MN = :mn
+        MO = :mo
+        MQ = :mq
+        MR = :mr
+        MT = :mt
+        MU = :mu
+        MV = :mv
+        MW = :mw
+        MX = :mx
+        MY = :my
+        MZ = :mz
+        NA = :na
+        NC = :nc
+        NE = :ne
+        NG = :ng
+        NI = :ni
+        NL = :nl
+        NO = :no
+        NP = :np
+        NZ = :nz
+        OM = :om
+        PA = :pa
+        PE = :pe
+        PF = :pf
+        PG = :pg
+        PH = :ph
+        PK = :pk
+        PL = :pl
+        PR = :pr
+        PS = :ps
+        PT = :pt
+        PY = :py
+        QA = :qa
+        RE = :re
+        RO = :ro
+        RS = :rs
+        RU = :ru
+        RW = :rw
+        SA = :sa
+        SC = :sc
+        SD = :sd
+        SE = :se
+        SG = :sg
+        SI = :si
+        SK = :sk
+        SL = :sl
+        SM = :sm
+        SN = :sn
+        SO = :so
+        SR = :sr
+        SS = :ss
+        ST = :st
+        SV = :sv
+        SX = :sx
+        SY = :sy
+        SZ = :sz
+        TC = :tc
+        TD = :td
+        TG = :tg
+        TH = :th
+        TJ = :tj
+        TL = :tl
+        TM = :tm
+        TN = :tn
+        TR = :tr
+        TT = :tt
+        TW = :tw
+        TZ = :tz
+        UA = :ua
+        UG = :ug
+        US = :us
+        UY = :uy
+        UZ = :uz
+        VC = :vc
+        VE = :ve
+        VG = :vg
+        VI = :vi
+        VN = :vn
+        YE = :ye
+        YT = :yt
+        ZA = :za
+        ZM = :zm
+        ZW = :zw
+
+        # @!method self.values
+        #   @return [Array<Symbol>]
+      end
+
       class Pdf < ContextDev::Internal::Type::BaseModel
         # @!attribute end_
         #   Last 1-based PDF page to parse. When omitted, parsing ends at the last page.

data/lib/context_dev/models/web_web_scrape_html_response.rb

@@ -26,7 +26,9 @@ module ContextDev
 
       # @!attribute type
       #   Detected content type of the returned `html` field. Sitemaps and feeds are
-      #   surfaced as `xml`; ordinary pages are `html`.
+      #   surfaced as `xml`; ordinary pages are `html`. Excel workbooks are surfaced as
+      #   `xlsx`/`xls` with the extracted sheets as HTML tables; PowerPoint presentations
+      #   are surfaced as `pptx`/`ppt` with the extracted slides as HTML.
       #
       #   @return [Symbol, ContextDev::Models::WebWebScrapeHTMLResponse::Type]
       required :type, enum: -> { ContextDev::Models::WebWebScrapeHTMLResponse::Type }
@@ -320,7 +322,9 @@ module ContextDev
       end
 
       # Detected content type of the returned `html` field. Sitemaps and feeds are
-      # surfaced as `xml`; ordinary pages are `html`.
+      # surfaced as `xml`; ordinary pages are `html`. Excel workbooks are surfaced as
+      # `xlsx`/`xls` with the extracted sheets as HTML tables; PowerPoint presentations
+      # are surfaced as `pptx`/`ppt` with the extracted slides as HTML.
       #
       # @see ContextDev::Models::WebWebScrapeHTMLResponse#type
       module Type
@@ -336,6 +340,10 @@ module ContextDev
         PDF = :pdf
         DOCX = :docx
         DOC = :doc
+        XLSX = :xlsx
+        XLS = :xls
+        PPTX = :pptx
+        PPT = :ppt
 
         # @!method self.values
         #   @return [Array<Symbol>]

data/lib/context_dev/models/web_web_scrape_images_params.rb

@@ -13,6 +13,15 @@ module ContextDev
       #   @return [String]
       required :url, String
 
+      # @!attribute dedupe
+      #   When true, visually duplicate images are removed: every image is loaded and
+      #   perceptually hashed, and only the highest-resolution copy of each duplicate
+      #   group is kept. Images that cannot be downloaded or hashed are kept. Default:
+      #   false.
+      #
+      #   @return [Boolean, nil]
+      optional :dedupe, ContextDev::Internal::Type::Boolean
+
       # @!attribute enrichment
       #   Optional per-image processing, sent as deep-object query params such as
       #   enrichment[resolution]=true.
@@ -50,12 +59,14 @@ module ContextDev
       #   @return [Integer, nil]
       optional :wait_for_ms, Integer
 
-      # @!method initialize(url:, enrichment: nil, headers: nil, max_age_ms: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
+      # @!method initialize(url:, dedupe: nil, enrichment: nil, headers: nil, max_age_ms: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {})
       #   Some parameter documentations has been truncated, see
       #   {ContextDev::Models::WebWebScrapeImagesParams} for more details.
       #
       #   @param url [String] Page URL to inspect. Must include http:// or https://.
       #
+      #   @param dedupe [Boolean] When true, visually duplicate images are removed: every image is loaded and perc
+      #
       #   @param enrichment [ContextDev::Models::WebWebScrapeImagesParams::Enrichment] Optional per-image processing, sent as deep-object query params such as enrichme
       #
       #   @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob