enconvert 0.0.1 → 0.1.0

data/lib/enconvert/v2.rb

@@ -1,688 +1,762 @@
-# frozen_string_literal: true
-
-require "erb"
-require "json"
-
-require_relative "internal"
-require_relative "v2_results"
-
-# V2 API namespace, reached as `client.v2`.
-#
-# One method per V2 endpoint (21 total across six groups: perceive,
-# discover, lookup, distill, ingest, watch). Options are passed as
-# snake_case keyword arguments and serialized to the API's snake_case wire
-# format (a straight pass-through, since Ruby's own idiom already is
-# snake_case); responses are mapped back onto snake_case Struct readers.
-# User-data payloads (schemas, extracted data, tracked fields, diff
-# changes) pass through untouched as plain Ruby Hashes.
-#
-# All V2 endpoints require a private API key (public keys are rejected)
-# and are plan-gated: a disabled feature or exhausted monthly quota
-# raises QuotaError (HTTP 402).
-module Enconvert
-  class V2
-    include Internal
-
-    def initialize(transport)
-      @transport = transport
-    end
-
-    # ------------------------------------------------------------------
-    # Perceive — render a URL into agent-ready artifacts
-    # ------------------------------------------------------------------
-
-    # Render one URL into the requested outputs (markdown, screenshots, PDF,
-    # links, structured data, ...). Synchronous: returns the completed
-    # operation with 15-minute signed artifact URLs.
-    #
-    # opts accepts: outputs, extract, schema, wait_for, wait_timeout_ms,
-    # js_code, viewport, headers, cookies, auth, proxy_url, geolocation,
-    # action_chain, cache_mode, pdf_options, block_resources,
-    # respect_robots, mobile.
-    def perceive(url, **opts)
-      body = serialize_perceive_options(opts)
-      body[:url] = url
-      to_perceive_result(post("/v2/perceive", body))
-    end
-
-    # Re-fetch a perceive operation by id (`per_...`). Artifact URLs are
-    # freshly re-signed on every call.
-    def get_perceive_operation(operation_id)
-      to_perceive_result(get("/v2/perceive/#{encode_path(operation_id)}"))
-    end
-
-    # Perceive up to 1000 URLs with one shared options block. Small batches
-    # run inline (completed result); larger ones return status "queued" —
-    # poll get_perceive_batch with the job_id.
-    def perceive_batch(urls, output_mode: nil, **render_opts)
-      body = { urls: urls, options: serialize_perceive_options(render_opts) }
-      body[:output_mode] = output_mode unless output_mode.nil?
-      to_perceive_batch_result(post("/v2/perceive/batch", body))
-    end
-
-    # Poll a perceive batch by job_id. Items fill in as URLs complete.
-    def get_perceive_batch(job_id)
-      to_perceive_batch_result(get("/v2/perceive/batch/#{encode_path(job_id)}"))
-    end
-
-    # ------------------------------------------------------------------
-    # Discover — enumerate a site's URLs without rendering
-    # ------------------------------------------------------------------
-
-    # List a site's URLs via sitemap, HTTP crawl, or both. No browser
-    # rendering — fast and does not consume perceive quota.
-    #
-    # opts accepts: mode, max_urls, max_depth, include_patterns,
-    # exclude_patterns, same_domain_only, respect_robots.
-    def discover(url, **opts)
-      body = { url: url }
-      body[:mode] = opts[:mode] if opts.key?(:mode)
-      body[:max_urls] = opts[:max_urls] if opts.key?(:max_urls)
-      body[:max_depth] = opts[:max_depth] if opts.key?(:max_depth)
-      body[:include_patterns] = opts[:include_patterns] if opts.key?(:include_patterns)
-      body[:exclude_patterns] = opts[:exclude_patterns] if opts.key?(:exclude_patterns)
-      body[:same_domain_only] = opts[:same_domain_only] if opts.key?(:same_domain_only)
-      body[:respect_robots] = opts[:respect_robots] if opts.key?(:respect_robots)
-      to_discover_result(post("/v2/discover", body))
-    end
-
-    # ------------------------------------------------------------------
-    # Lookup — web search with optional auto-perceive
-    # ------------------------------------------------------------------
-
-    # Run a categorized web search. With perceive_top > 0, the top-N result
-    # URLs are auto-perceived (each consumes one perceive-quota unit) and
-    # carry their full PerceiveResult inline.
-    #
-    # opts accepts: category, country, locale, time_filter, num_results,
-    # page, location, autocorrect, perceive_top.
-    def lookup(query, **opts)
-      body = { query: query }
-      body[:category] = opts[:category] if opts.key?(:category)
-      body[:country] = opts[:country] if opts.key?(:country)
-      body[:locale] = opts[:locale] if opts.key?(:locale)
-      body[:time_filter] = opts[:time_filter] if opts.key?(:time_filter)
-      body[:num_results] = opts[:num_results] if opts.key?(:num_results)
-      body[:page] = opts[:page] if opts.key?(:page)
-      body[:location] = opts[:location] if opts.key?(:location)
-      body[:autocorrect] = opts[:autocorrect] if opts.key?(:autocorrect)
-      body[:perceive_top] = opts[:perceive_top] if opts.key?(:perceive_top)
-      to_lookup_result(post("/v2/lookup", body))
-    end
-
-    # ------------------------------------------------------------------
-    # Distill — schema-driven structured extraction
-    # ------------------------------------------------------------------
-
-    # Extract structured data matching `schema` from explicit URLs or from
-    # a discovered site. An optional css_schema answers fields for free;
-    # anything it misses escalates to the LLM tier (plan-gated).
-    #
-    # Provide exactly one of `urls` (non-empty Array) or `discover_from`
-    # (Hash with :url, :mode, :max_pages). `schema` is required.
-    def distill(urls: nil, discover_from: nil, schema: nil, css_schema: nil, wait_for: nil,
-                wait_timeout_ms: nil, headers: nil, cookies: nil, respect_robots: nil)
-      has_urls = urls.is_a?(Array) && !urls.empty?
-      has_discover = !discover_from.nil?
-      raise Enconvert::Error, "distill: provide exactly one of 'urls' or 'discover_from'" if has_urls == has_discover
-      raise Enconvert::Error, "distill: 'schema' is required and must be a Hash" unless schema.is_a?(Hash)
-
-      body = { schema: schema }
-      body[:urls] = urls if has_urls
-      if discover_from
-        df = { url: discover_from[:url] }
-        df[:mode] = discover_from[:mode] if discover_from.key?(:mode)
-        df[:max_pages] = discover_from[:max_pages] if discover_from.key?(:max_pages)
-        body[:discover_from] = df
-      end
-      body[:css_schema] = serialize_css_schema(css_schema) unless css_schema.nil?
-      body[:wait_for] = wait_for unless wait_for.nil?
-      body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
-      body[:headers] = headers unless headers.nil?
-      body[:cookies] = cookies unless cookies.nil?
-      body[:respect_robots] = respect_robots unless respect_robots.nil?
-      to_distill_result(post("/v2/distill", body))
-    end
-
-    # ------------------------------------------------------------------
-    # Ingest — site to RAG-ready JSONL chunks (always async)
-    # ------------------------------------------------------------------
-
-    # Start an ingest job: turn explicit URLs or a discovered site into
-    # chunked, RAG-ready JSONL. Always asynchronous — returns the queued
-    # job; poll get_ingest_job or configure webhook_url for completion.
-    #
-    # `mode` defaults to "urls" for validation purposes but is only sent on
-    # the wire when explicitly given (the API defaults it itself).
-    def ingest(mode: nil, url: nil, urls: nil, max_pages: nil, max_depth: nil, same_domain_only: nil,
-               include_patterns: nil, exclude_patterns: nil, respect_robots: nil, wait_for: nil,
-               wait_timeout_ms: nil, chunk: nil, webhook_url: nil)
-      effective_mode = mode || "urls"
-      if effective_mode == "urls"
-        raise Enconvert::Error, "ingest: mode 'urls' requires a non-empty 'urls' list" if urls.nil? || urls.empty?
-        raise Enconvert::Error, "ingest: mode 'urls' does not accept 'url'" unless url.nil?
-      else
-        raise Enconvert::Error, "ingest: mode '#{effective_mode}' requires a seed 'url'" if url.nil?
-        raise Enconvert::Error, "ingest: mode '#{effective_mode}' does not accept 'urls'" unless urls.nil?
-      end
-
-      body = {}
-      body[:mode] = mode unless mode.nil?
-      body[:url] = url unless url.nil?
-      body[:urls] = urls unless urls.nil?
-      body[:max_pages] = max_pages unless max_pages.nil?
-      body[:max_depth] = max_depth unless max_depth.nil?
-      body[:same_domain_only] = same_domain_only unless same_domain_only.nil?
-      body[:include_patterns] = include_patterns unless include_patterns.nil?
-      body[:exclude_patterns] = exclude_patterns unless exclude_patterns.nil?
-      body[:respect_robots] = respect_robots unless respect_robots.nil?
-      body[:wait_for] = wait_for unless wait_for.nil?
-      body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
-      if chunk
-        c = {}
-        c[:max_words] = chunk[:max_words] if chunk.key?(:max_words)
-        c[:sentence_overlap] = chunk[:sentence_overlap] if chunk.key?(:sentence_overlap)
-        body[:chunk] = c
-      end
-      body[:webhook_url] = webhook_url unless webhook_url.nil?
-      to_ingest_job(post("/v2/ingest", body))
-    end
-
-    # Ingest one or more uploaded FILES into RAG-ready JSONL chunks — the
-    # file counterpart of ingest(), sharing the same job lifecycle (mode
-    # "files"). PDF, DOCX, PPTX, XLSX, CSV, HTML, EPUB, TXT/MD and
-    # legacy/ODF office are accepted. Always asynchronous; poll
-    # get_ingest_job or configure a webhook.
-    #
-    # `files` accepts the same file-input union as convert_image (path
-    # string, IO-like/readable object, or { data:, filename: }); at least
-    # one file is required.
-    def ingest_files(files, chunk: nil, webhook_url: nil)
-      unless files.is_a?(Array) && !files.empty?
-        raise Enconvert::Error, "ingest_files: provide at least one file"
-      end
-
-      fields = files.map do |file|
-        part = to_file_part(file)
-        { name: "files", value: part.bytes, filename: part.filename, content_type: part.content_type }
-      end
-      if chunk
-        fields << { name: "max_words", value: chunk[:max_words] } if chunk.key?(:max_words)
-        fields << { name: "sentence_overlap", value: chunk[:sentence_overlap] } if chunk.key?(:sentence_overlap)
-      end
-      fields << { name: "webhook_url", value: webhook_url } unless webhook_url.nil?
-
-      body, boundary = build_multipart(fields)
-      resp = @transport.request(
-        :post, "/v2/ingest/files",
-        headers: { "Content-Type" => "multipart/form-data; boundary=#{boundary}" },
-        body: body
-      )
-      raise_for_status(resp)
-      to_ingest_job(JSON.parse(resp.body))
-    end
-
-    # List ingest jobs, newest first.
-    def list_ingest_jobs(skip: nil, limit: nil)
-      d = get("/v2/ingest#{list_query(skip, limit)}")
-      jobs = d["jobs"].is_a?(Array) ? d["jobs"] : []
-      IngestJobList.new(
-        jobs: jobs.map { |j| to_ingest_job_summary(j) },
-        skip: num(d["skip"]),
-        limit: num(d["limit"], 20),
-        has_more: d["has_more"] == true
-      )
-    end
-
-    # Get one ingest job by id (`ing_...`).
-    def get_ingest_job(job_id)
-      to_ingest_job(get("/v2/ingest/#{encode_path(job_id)}"))
-    end
-
-    # Cancel a queued/processing ingest job. Idempotent: canceling an
-    # already-terminal job returns it unchanged.
-    def cancel_ingest_job(job_id)
-      to_ingest_job(delete_request("/v2/ingest/#{encode_path(job_id)}"))
-    end
-
-    # Re-deliver the completion webhook of a completed job (409 if the job
-    # is not completed, 400 if it has no webhook configured).
-    def retry_ingest_webhook(job_id)
-      d = post("/v2/ingest/#{encode_path(job_id)}/retry-webhook", nil)
-      WebhookRetryResult.new(
-        job_id: str(d["job_id"]),
-        delivered: d["delivered"] == true,
-        attempts: num(d["attempts"]),
-        status_code: opt_num(d["status_code"]),
-        detail: str(d["detail"])
-      )
-    end
-
-    # Get (creating on first call) the project's webhook signing secret and
-    # the header/scheme details needed to verify deliveries.
-    def get_webhook_secret
-      to_webhook_secret(get("/v2/ingest/webhook-secret"))
-    end
-
-    # Rotate the webhook signing secret. Signatures made with the previous
-    # secret stop verifying immediately.
-    def rotate_webhook_secret
-      to_webhook_secret(post("/v2/ingest/webhook-secret/rotate", nil))
-    end
-
-    # ------------------------------------------------------------------
-    # Watch — recurring change monitoring
-    # ------------------------------------------------------------------
-
-    # Create a watcher that re-renders `url` on a fixed cadence (hourly
-    # floor) and notifies on changes via email and/or webhook.
-    def create_watcher(url, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
-                        webhook_url: nil, notify_email: nil)
-      body = { url: url }
-      body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
-      body[:diff_mode] = diff_mode unless diff_mode.nil?
-      body[:track_fields] = track_fields unless track_fields.nil?
-      body[:webhook_url] = webhook_url unless webhook_url.nil?
-      body[:notify_email] = notify_email unless notify_email.nil?
-      to_watcher(post("/v2/watch", body))
-    end
-
-    # List watchers, newest first.
-    def list_watchers(skip: nil, limit: nil)
-      d = get("/v2/watch#{list_query(skip, limit)}")
-      watchers = d["watchers"].is_a?(Array) ? d["watchers"] : []
-      WatcherList.new(
-        watchers: watchers.map { |w| to_watcher_summary(w) },
-        skip: num(d["skip"]),
-        limit: num(d["limit"], 20),
-        has_more: d["has_more"] == true
-      )
-    end
-
-    # Get one watcher by id (`wat_...`). Deleted watchers read as 404.
-    def get_watcher(watcher_id)
-      to_watcher(get("/v2/watch/#{encode_path(watcher_id)}"))
-    end
-
-    # Page through a watcher's check history, newest first.
-    def get_watcher_snapshots(watcher_id, limit: nil)
-      query = limit.nil? ? "" : "?limit=#{limit}"
-      d = get("/v2/watch/#{encode_path(watcher_id)}/snapshots#{query}")
-      snapshots = d["snapshots"].is_a?(Array) ? d["snapshots"] : []
-      WatcherSnapshotList.new(
-        watcher_id: str(d["watcher_id"]),
-        snapshots: snapshots.map { |s| to_watcher_snapshot(s) },
-        limit: num(d["limit"], 20)
-      )
-    end
-
-    # Update a watcher. At least one field is required. Set webhook_url to
-    # "" to clear the webhook; resuming a paused watcher re-checks the
-    # plan's watcher cap.
-    def update_watcher(watcher_id, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
-                        webhook_url: nil, notify_email: nil, status: nil)
-      body = {}
-      body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
-      body[:diff_mode] = diff_mode unless diff_mode.nil?
-      body[:track_fields] = track_fields unless track_fields.nil?
-      body[:webhook_url] = webhook_url unless webhook_url.nil?
-      body[:notify_email] = notify_email unless notify_email.nil?
-      body[:status] = status unless status.nil?
-      raise Enconvert::Error, "update_watcher: provide at least one field to update" if body.empty?
-
-      to_watcher(patch("/v2/watch/#{encode_path(watcher_id)}", body))
-    end
-
-    # Soft-delete a watcher (idempotent). Returns the tombstoned watcher
-    # with status "deleted".
-    def delete_watcher(watcher_id)
-      to_watcher(delete_request("/v2/watch/#{encode_path(watcher_id)}"))
-    end
-
-    private
-
-    # ------------------------------------------------------------------
-    # HTTP helpers
-    # ------------------------------------------------------------------
-
-    def json_request(method, path, body: nil, headers: {})
-      resp = @transport.request(method, path, headers: headers, body: body)
-      raise_for_status(resp)
-      JSON.parse(resp.body)
-    end
-
-    def post(path, body)
-      json_request(:post, path,
-                   body: body.nil? ? nil : JSON.generate(body),
-                   headers: { "Content-Type" => "application/json" })
-    end
-
-    def get(path)
-      json_request(:get, path)
-    end
-
-    def patch(path, body)
-      json_request(:patch, path, body: JSON.generate(body), headers: { "Content-Type" => "application/json" })
-    end
-
-    def delete_request(path)
-      json_request(:delete, path)
-    end
-
-    def encode_path(value)
-      ERB::Util.url_encode(value.to_s)
-    end
-
-    def list_query(skip, limit)
-      params = []
-      params << "skip=#{skip}" unless skip.nil?
-      params << "limit=#{limit}" unless limit.nil?
-      params.empty? ? "" : "?#{params.join('&')}"
-    end
-
-    # ------------------------------------------------------------------
-    # Request serializers
-    # ------------------------------------------------------------------
-
-    def serialize_perceive_options(o)
-      out = {}
-      out[:outputs] = o[:outputs] if o.key?(:outputs)
-      out[:extract] = o[:extract] if o.key?(:extract)
-      out[:schema] = o[:schema] if o.key?(:schema)
-      out[:wait_for] = o[:wait_for] if o.key?(:wait_for)
-      out[:wait_timeout_ms] = o[:wait_timeout_ms] if o.key?(:wait_timeout_ms)
-      out[:js_code] = o[:js_code] if o.key?(:js_code)
-      out[:viewport] = o[:viewport] if o.key?(:viewport)
-      out[:headers] = o[:headers] if o.key?(:headers)
-      out[:cookies] = o[:cookies] if o.key?(:cookies)
-      out[:auth] = o[:auth] if o.key?(:auth)
-      out[:proxy_url] = o[:proxy_url] if o.key?(:proxy_url)
-      out[:geolocation] = o[:geolocation] if o.key?(:geolocation)
-      out[:action_chain] = o[:action_chain] if o.key?(:action_chain)
-      out[:cache_mode] = o[:cache_mode] if o.key?(:cache_mode)
-      out[:pdf_options] = serialize_pdf_options(o[:pdf_options]) if o.key?(:pdf_options)
-      out[:block_resources] = o[:block_resources] if o.key?(:block_resources)
-      out[:respect_robots] = o[:respect_robots] if o.key?(:respect_robots)
-      out[:mobile] = o[:mobile] if o.key?(:mobile)
-      out
-    end
-
-    def serialize_css_field(f)
-      out = { name: f[:name], type: f[:type] }
-      out[:selector] = f[:selector] if f.key?(:selector)
-      out[:attribute] = f[:attribute] if f.key?(:attribute)
-      out[:pattern] = f[:pattern] if f.key?(:pattern)
-      out[:default] = f[:default] if f.key?(:default)
-      out[:transform] = f[:transform] if f.key?(:transform)
-      out[:fields] = f[:fields].map { |nf| serialize_css_field(nf) } if f.key?(:fields)
-      out
-    end
-
-    def serialize_css_schema(s)
-      out = { base_selector: s[:base_selector], fields: s[:fields].map { |f| serialize_css_field(f) } }
-      out[:name] = s[:name] if s.key?(:name)
-      out[:target_field] = s[:target_field] if s.key?(:target_field)
-      out
-    end
-
-    # ------------------------------------------------------------------
-    # Response mappers. Optional fields may be absent entirely
-    # (response_model_exclude_none) — every access is guarded.
-    # ------------------------------------------------------------------
-
-    def str(v, fallback = "")
-      v.is_a?(String) ? v : fallback
-    end
-
-    def opt_str(v)
-      v.is_a?(String) ? v : nil
-    end
-
-    def num(v, fallback = 0)
-      v.is_a?(Numeric) ? v : fallback
-    end
-
-    def opt_num(v)
-      v.is_a?(Numeric) ? v : nil
-    end
-
-    def str_arr(v)
-      v.is_a?(Array) ? v.select { |x| x.is_a?(String) } : []
-    end
-
-    def opt_obj(v)
-      v.is_a?(Hash) ? v : nil
-    end
-
-    def to_tokens(v)
-      d = v.is_a?(Hash) ? v : nil
-      V2Tokens.new(input: num(d && d["input"]), output: num(d && d["output"]))
-    end
-
-    def to_output_artifact(v)
-      d = v.is_a?(Hash) ? v : {}
-      V2OutputArtifact.new(
-        url: opt_str(d["url"]),
-        object_key: str(d["object_key"]),
-        size_bytes: num(d["size_bytes"]),
-        content_type: str(d["content_type"], "application/octet-stream"),
-        expires_in: num(d["expires_in"], 900)
-      )
-    end
-
-    def to_perceive_result(d)
-      raw_outputs = opt_obj(d["outputs"]) || {}
-      outputs = {}
-      raw_outputs.each { |name, artifact| outputs[name] = to_output_artifact(artifact) }
-      PerceiveResult.new(
-        operation_id: str(d["operation_id"]),
-        status: d["status"],
-        url: str(d["url"]),
-        url_final: opt_str(d["url_final"]),
-        content_hash: opt_str(d["content_hash"]),
-        render_quality: opt_num(d["render_quality"]),
-        cache_hit: d["cache_hit"] == true,
-        outputs: outputs,
-        structured: opt_obj(d["structured"]),
-        extraction_tier: opt_str(d["extraction_tier"]),
-        tokens: to_tokens(d["tokens"]),
-        cost_cents: num(d["cost_cents"]),
-        duration_ms: opt_num(d["duration_ms"]),
-        error: opt_str(d["error"]),
-        warnings: str_arr(d["warnings"])
-      )
-    end
-
-    def to_perceive_batch_result(d)
-      items = d["items"].is_a?(Array) ? d["items"] : []
-      PerceiveBatchResult.new(
-        job_id: str(d["job_id"]),
-        status: d["status"],
-        output_mode: opt_str(d["output_mode"]) || "manifest",
-        total: num(d["total"]),
-        completed: num(d["completed"]),
-        failed: num(d["failed"]),
-        pending: num(d["pending"]),
-        zip: d["zip"].nil? ? nil : to_output_artifact(d["zip"]),
-        items: items.map { |i| to_perceive_result(i) },
-        warnings: str_arr(d["warnings"])
-      )
-    end
-
-    def to_discover_result(d)
-      DiscoverResult.new(
-        url: str(d["url"]),
-        mode: d["mode"],
-        total: num(d["total"]),
-        urls: str_arr(d["urls"]),
-        pages_crawled: num(d["pages_crawled"]),
-        truncated: d["truncated"] == true,
-        robots_respected: d["robots_respected"] == true,
-        sources: opt_obj(d["sources"]) || {},
-        warnings: str_arr(d["warnings"])
-      )
-    end
-
-    def to_lookup_item(d)
-      perceive = opt_obj(d["perceive"])
-      LookupItem.new(
-        title: opt_str(d["title"]),
-        url: opt_str(d["url"]),
-        snippet: opt_str(d["snippet"]),
-        position: opt_num(d["position"]),
-        source: opt_str(d["source"]),
-        date: opt_str(d["date"]),
-        image_url: opt_str(d["image_url"]),
-        thumbnail_url: opt_str(d["thumbnail_url"]),
-        extra: opt_obj(d["extra"]) || {},
-        perceive: perceive ? to_perceive_result(perceive) : nil
-      )
-    end
-
-    def to_lookup_result(d)
-      results = d["results"].is_a?(Array) ? d["results"] : []
-      LookupResult.new(
-        lookup_id: opt_num(d["lookup_id"]),
-        query: str(d["query"]),
-        category: d["category"],
-        country: opt_str(d["country"]),
-        locale: opt_str(d["locale"]),
-        time_filter: opt_str(d["time_filter"]),
-        total: num(d["total"]),
-        results: results.map { |r| to_lookup_item(r) },
-        perceive_top: num(d["perceive_top"]),
-        perceive_operation_ids: str_arr(d["perceive_operation_ids"]),
-        answer_box: opt_obj(d["answer_box"]),
-        knowledge_graph: opt_obj(d["knowledge_graph"]),
-        credits: opt_num(d["credits"]),
-        cost_cents: num(d["cost_cents"]),
-        warnings: str_arr(d["warnings"])
-      )
-    end
-
-    def to_distill_item(d)
-      DistillItem.new(
-        url: str(d["url"]),
-        url_final: opt_str(d["url_final"]),
-        status: opt_str(d["status"]) || "completed",
-        data: opt_obj(d["data"]),
-        extraction_tier: opt_str(d["extraction_tier"]) || "none",
-        fields_from_css: num(d["fields_from_css"]),
-        fields_from_llm: num(d["fields_from_llm"]),
-        render_quality: opt_num(d["render_quality"]),
-        tokens: to_tokens(d["tokens"]),
-        cost_cents: num(d["cost_cents"]),
-        error: opt_str(d["error"]),
-        warnings: str_arr(d["warnings"])
-      )
-    end
-
-    def to_distill_result(d)
-      results = d["results"].is_a?(Array) ? d["results"] : []
-      DistillResult.new(
-        operation_id: str(d["operation_id"]),
-        total: num(d["total"]),
-        completed: num(d["completed"]),
-        failed: num(d["failed"]),
-        results: results.map { |r| to_distill_item(r) },
-        total_cost_cents: num(d["total_cost_cents"]),
-        warnings: str_arr(d["warnings"])
-      )
-    end
-
-    def to_ingest_job(d)
-      IngestJob.new(
-        job_id: str(d["job_id"]),
-        status: d["status"],
-        mode: d["mode"],
-        pages_discovered: num(d["pages_discovered"]),
-        pages_processed: num(d["pages_processed"]),
-        pages_failed: num(d["pages_failed"]),
-        total_chunks: num(d["total_chunks"]),
-        output_url: opt_str(d["output_url"]),
-        error_message: opt_str(d["error_message"]),
-        webhook_url: opt_str(d["webhook_url"]),
-        webhook_delivered: d["webhook_delivered"] == true,
-        created_at: opt_str(d["created_at"]),
-        completed_at: opt_str(d["completed_at"]),
-        warnings: str_arr(d["warnings"])
-      )
-    end
-
-    def to_ingest_job_summary(d)
-      IngestJobSummary.new(
-        job_id: str(d["job_id"]),
-        status: d["status"],
-        mode: d["mode"],
-        pages_discovered: num(d["pages_discovered"]),
-        pages_processed: num(d["pages_processed"]),
-        pages_failed: num(d["pages_failed"]),
-        total_chunks: num(d["total_chunks"]),
-        output_url: opt_str(d["output_url"]),
-        error_message: opt_str(d["error_message"]),
-        webhook_configured: d["webhook_configured"] == true,
-        webhook_delivered: d["webhook_delivered"] == true,
-        created_at: opt_str(d["created_at"]),
-        completed_at: opt_str(d["completed_at"])
-      )
-    end
-
-    def to_webhook_secret(d)
-      WebhookSecret.new(
-        secret: str(d["secret"]),
-        signature_header: str(d["signature_header"]),
-        timestamp_header: str(d["timestamp_header"]),
-        signature_scheme: str(d["signature_scheme"]),
-        replay_tolerance_seconds: num(d["replay_tolerance_seconds"]),
-        rotated: d["rotated"] == true
-      )
-    end
-
-    def to_watcher(d)
-      Watcher.new(
-        watcher_id: str(d["watcher_id"]),
-        url: str(d["url"]),
-        status: d["status"],
-        frequency_minutes: num(d["frequency_minutes"]),
-        diff_mode: d["diff_mode"],
-        track_fields: opt_obj(d["track_fields"]),
-        webhook_url: opt_str(d["webhook_url"]),
-        notify_email: d["notify_email"] != false,
-        consecutive_errors: num(d["consecutive_errors"]),
-        checks_count: num(d["checks_count"]),
-        last_check_at: opt_str(d["last_check_at"]),
-        next_check_at: opt_str(d["next_check_at"]),
-        last_change_at: opt_str(d["last_change_at"]),
-        created_at: opt_str(d["created_at"]),
-        updated_at: opt_str(d["updated_at"])
-      )
-    end
-
-    def to_watcher_summary(d)
-      WatcherSummary.new(
-        watcher_id: str(d["watcher_id"]),
-        url: str(d["url"]),
-        status: d["status"],
-        frequency_minutes: num(d["frequency_minutes"]),
-        checks_count: num(d["checks_count"]),
-        consecutive_errors: num(d["consecutive_errors"]),
-        last_check_at: opt_str(d["last_check_at"]),
-        next_check_at: opt_str(d["next_check_at"]),
-        last_change_at: opt_str(d["last_change_at"]),
-        created_at: opt_str(d["created_at"])
-      )
-    end
-
-    def to_watcher_snapshot(d)
-      changes = d["changes"].is_a?(Array) ? d["changes"].select { |c| c.is_a?(Hash) } : []
-      WatcherSnapshot.new(
-        checked_at: str(d["checked_at"]),
-        has_changes: d["has_changes"] == true,
-        similarity: opt_num(d["similarity"]),
-        render_quality: opt_num(d["render_quality"]),
-        change_count: num(d["change_count"]),
-        changes: changes
-      )
-    end
-  end
-end
+# frozen_string_literal: true
+
+require "erb"
+require "json"
+
+require_relative "internal"
+require_relative "v2_results"
+
+# V2 API namespace, reached as `client.v2`.
+#
+# One method per V2 endpoint (21 total across six groups: perceive,
+# discover, lookup, distill, ingest, watch). Options are passed as
+# snake_case keyword arguments and serialized to the API's snake_case wire
+# format (a straight pass-through, since Ruby's own idiom already is
+# snake_case); responses are mapped back onto snake_case Struct readers.
+# User-data payloads (schemas, extracted data, tracked fields, diff
+# changes) pass through untouched as plain Ruby Hashes.
+#
+# All V2 endpoints require a private API key (public keys are rejected)
+# and are plan-gated: a disabled feature or exhausted monthly quota
+# raises QuotaError (HTTP 402).
+module Enconvert
+  class V2
+    include Internal
+
+    def initialize(transport)
+      @transport = transport
+    end
+
+    # ------------------------------------------------------------------
+    # Perceive — render a URL into agent-ready artifacts
+    # ------------------------------------------------------------------
+
+    # Render one URL into the requested outputs (markdown, screenshots, PDF,
+    # links, structured data, ...). Synchronous: returns the completed
+    # operation with 15-minute signed artifact URLs.
+    #
+    # opts accepts: outputs, extract, schema, wait_for, wait_timeout_ms,
+    # js_code, viewport, headers, cookies, auth, proxy_url, geolocation,
+    # action_chain, cache_mode, pdf_options, block_resources,
+    # respect_robots, mobile, only_main_content, direct_download.
+    def perceive(url, **opts)
+      body = serialize_perceive_options(opts)
+      body[:url] = url
+      # direct_download is accepted by POST /v2/perceive only (the batch
+      # endpoint rejects it with 422), so it stays out of the shared
+      # options serializer.
+      body[:direct_download] = opts[:direct_download] if opts.key?(:direct_download)
+      to_perceive_result(post("/v2/perceive", body))
+    end
+
+    # Perceive with direct_download forced on: the response body IS the
+    # artifact bytes (`content`), with metadata in headers — no signed URL
+    # round-trip. Requires exactly one artifact-producing output
+    # ("structured" may ride along; it stays inline server-side and is not
+    # returned here). Accepts the same opts as perceive.
+    def perceive_direct(url, **opts)
+      require_one_artifact_output("perceive_direct", opts)
+      body = serialize_perceive_options(opts)
+      body[:url] = url
+      body[:direct_download] = true
+      resp = @transport.request(
+        :post, "/v2/perceive",
+        headers: { "Content-Type" => "application/json" },
+        body: JSON.generate(body)
+      )
+      raise_for_status(resp)
+      to_perceive_direct_result(resp)
+    end
+
+    # Re-fetch a perceive operation by id (`per_...`). Artifact URLs are
+    # freshly re-signed on every call.
+    def get_perceive_operation(operation_id)
+      to_perceive_result(get("/v2/perceive/#{encode_path(operation_id)}"))
+    end
+
+    # Download one stored artifact of an earlier perceive operation as raw
+    # bytes. `output` may be omitted when the operation produced exactly
+    # one artifact (400 otherwise, listing the available outputs). 410 once
+    # the artifact passed the plan's retention window.
+    def download_perceive_artifact(operation_id, output: nil)
+      query = "?direct_download=true"
+      query += "&output=#{encode_path(output)}" unless output.nil?
+      resp = @transport.request(:get, "/v2/perceive/#{encode_path(operation_id)}#{query}")
+      raise_for_status(resp)
+      to_perceive_direct_result(resp)
+    end
+
+    # Perceive up to 1000 URLs with one shared options block. Small batches
+    # run inline (completed result); larger ones return status "queued" —
+    # poll get_perceive_batch with the job_id.
+    def perceive_batch(urls, output_mode: nil, **render_opts)
+      body = { urls: urls, options: serialize_perceive_options(render_opts) }
+      body[:output_mode] = output_mode unless output_mode.nil?
+      to_perceive_batch_result(post("/v2/perceive/batch", body))
+    end
+
+    # Poll a perceive batch by job_id. Items fill in as URLs complete.
+    def get_perceive_batch(job_id)
+      to_perceive_batch_result(get("/v2/perceive/batch/#{encode_path(job_id)}"))
+    end
+
+    # ------------------------------------------------------------------
+    # Discover — enumerate a site's URLs without rendering
+    # ------------------------------------------------------------------
+
+    # List a site's URLs via sitemap, HTTP crawl, or both. No browser
+    # rendering — fast and does not consume perceive quota.
+    #
+    # opts accepts: mode, max_urls, max_depth, include_patterns,
+    # exclude_patterns, same_domain_only, respect_robots.
+    def discover(url, **opts)
+      body = { url: url }
+      body[:mode] = opts[:mode] if opts.key?(:mode)
+      body[:max_urls] = opts[:max_urls] if opts.key?(:max_urls)
+      body[:max_depth] = opts[:max_depth] if opts.key?(:max_depth)
+      body[:include_patterns] = opts[:include_patterns] if opts.key?(:include_patterns)
+      body[:exclude_patterns] = opts[:exclude_patterns] if opts.key?(:exclude_patterns)
+      body[:same_domain_only] = opts[:same_domain_only] if opts.key?(:same_domain_only)
+      body[:respect_robots] = opts[:respect_robots] if opts.key?(:respect_robots)
+      to_discover_result(post("/v2/discover", body))
+    end
+
+    # ------------------------------------------------------------------
+    # Lookup — web search with optional auto-perceive
+    # ------------------------------------------------------------------
+
+    # Run a categorized web search. With perceive_top > 0, the top-N result
+    # URLs are auto-perceived (each consumes one perceive-quota unit) and
+    # carry their full PerceiveResult inline.
+    #
+    # opts accepts: category, country, locale, time_filter, num_results,
+    # page, location, autocorrect, perceive_top.
+    def lookup(query, **opts)
+      body = { query: query }
+      body[:category] = opts[:category] if opts.key?(:category)
+      body[:country] = opts[:country] if opts.key?(:country)
+      body[:locale] = opts[:locale] if opts.key?(:locale)
+      body[:time_filter] = opts[:time_filter] if opts.key?(:time_filter)
+      body[:num_results] = opts[:num_results] if opts.key?(:num_results)
+      body[:page] = opts[:page] if opts.key?(:page)
+      body[:location] = opts[:location] if opts.key?(:location)
+      body[:autocorrect] = opts[:autocorrect] if opts.key?(:autocorrect)
+      body[:perceive_top] = opts[:perceive_top] if opts.key?(:perceive_top)
+      to_lookup_result(post("/v2/lookup", body))
+    end
+
+    # ------------------------------------------------------------------
+    # Distill — schema-driven structured extraction
+    # ------------------------------------------------------------------
+
+    # Extract structured data matching `schema` from explicit URLs or from
+    # a discovered site. An optional css_schema answers fields for free;
+    # anything it misses escalates to the LLM tier (plan-gated).
+    #
+    # Provide exactly one of `urls` (non-empty Array) or `discover_from`
+    # (Hash with :url, :mode, :max_pages). `schema` is required.
+    def distill(urls: nil, discover_from: nil, schema: nil, css_schema: nil, wait_for: nil,
+                wait_timeout_ms: nil, headers: nil, cookies: nil, respect_robots: nil)
+      has_urls = urls.is_a?(Array) && !urls.empty?
+      has_discover = !discover_from.nil?
+      raise Enconvert::Error, "distill: provide exactly one of 'urls' or 'discover_from'" if has_urls == has_discover
+      raise Enconvert::Error, "distill: 'schema' is required and must be a Hash" unless schema.is_a?(Hash)
+
+      body = { schema: schema }
+      body[:urls] = urls if has_urls
+      if discover_from
+        df = { url: discover_from[:url] }
+        df[:mode] = discover_from[:mode] if discover_from.key?(:mode)
+        df[:max_pages] = discover_from[:max_pages] if discover_from.key?(:max_pages)
+        body[:discover_from] = df
+      end
+      body[:css_schema] = serialize_css_schema(css_schema) unless css_schema.nil?
+      body[:wait_for] = wait_for unless wait_for.nil?
+      body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
+      body[:headers] = headers unless headers.nil?
+      body[:cookies] = cookies unless cookies.nil?
+      body[:respect_robots] = respect_robots unless respect_robots.nil?
+      to_distill_result(post("/v2/distill", body))
+    end
+
+    # ------------------------------------------------------------------
+    # Ingest — site to RAG-ready JSONL chunks (always async)
+    # ------------------------------------------------------------------
+
+    # Start an ingest job: turn explicit URLs or a discovered site into
+    # chunked, RAG-ready JSONL. Always asynchronous — returns the queued
+    # job; poll get_ingest_job or configure webhook_url for completion.
+    #
+    # `mode` defaults to "urls" for validation purposes but is only sent on
+    # the wire when explicitly given (the API defaults it itself).
+    def ingest(mode: nil, url: nil, urls: nil, max_pages: nil, max_depth: nil, same_domain_only: nil,
+               include_patterns: nil, exclude_patterns: nil, respect_robots: nil, wait_for: nil,
+               wait_timeout_ms: nil, chunk: nil, webhook_url: nil)
+      effective_mode = mode || "urls"
+      if effective_mode == "urls"
+        raise Enconvert::Error, "ingest: mode 'urls' requires a non-empty 'urls' list" if urls.nil? || urls.empty?
+        raise Enconvert::Error, "ingest: mode 'urls' does not accept 'url'" unless url.nil?
+      else
+        raise Enconvert::Error, "ingest: mode '#{effective_mode}' requires a seed 'url'" if url.nil?
+        raise Enconvert::Error, "ingest: mode '#{effective_mode}' does not accept 'urls'" unless urls.nil?
+      end
+
+      body = {}
+      body[:mode] = mode unless mode.nil?
+      body[:url] = url unless url.nil?
+      body[:urls] = urls unless urls.nil?
+      body[:max_pages] = max_pages unless max_pages.nil?
+      body[:max_depth] = max_depth unless max_depth.nil?
+      body[:same_domain_only] = same_domain_only unless same_domain_only.nil?
+      body[:include_patterns] = include_patterns unless include_patterns.nil?
+      body[:exclude_patterns] = exclude_patterns unless exclude_patterns.nil?
+      body[:respect_robots] = respect_robots unless respect_robots.nil?
+      body[:wait_for] = wait_for unless wait_for.nil?
+      body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
+      if chunk
+        c = {}
+        c[:max_words] = chunk[:max_words] if chunk.key?(:max_words)
+        c[:sentence_overlap] = chunk[:sentence_overlap] if chunk.key?(:sentence_overlap)
+        body[:chunk] = c
+      end
+      body[:webhook_url] = webhook_url unless webhook_url.nil?
+      to_ingest_job(post("/v2/ingest", body))
+    end
+
+    # Ingest one or more uploaded FILES into RAG-ready JSONL chunks — the
+    # file counterpart of ingest(), sharing the same job lifecycle (mode
+    # "files"). PDF, DOCX, PPTX, XLSX, CSV, HTML, EPUB, TXT/MD and
+    # legacy/ODF office are accepted. Always asynchronous; poll
+    # get_ingest_job or configure a webhook.
+    #
+    # `files` accepts the same file-input union as convert_image (path
+    # string, IO-like/readable object, or { data:, filename: }); at least
+    # one file is required.
+    def ingest_files(files, chunk: nil, webhook_url: nil)
+      unless files.is_a?(Array) && !files.empty?
+        raise Enconvert::Error, "ingest_files: provide at least one file"
+      end
+
+      fields = files.map do |file|
+        part = to_file_part(file)
+        { name: "files", value: part.bytes, filename: part.filename, content_type: part.content_type }
+      end
+      if chunk
+        fields << { name: "max_words", value: chunk[:max_words] } if chunk.key?(:max_words)
+        fields << { name: "sentence_overlap", value: chunk[:sentence_overlap] } if chunk.key?(:sentence_overlap)
+      end
+      fields << { name: "webhook_url", value: webhook_url } unless webhook_url.nil?
+
+      body, boundary = build_multipart(fields)
+      resp = @transport.request(
+        :post, "/v2/ingest/files",
+        headers: { "Content-Type" => "multipart/form-data; boundary=#{boundary}" },
+        body: body
+      )
+      raise_for_status(resp)
+      to_ingest_job(JSON.parse(resp.body))
+    end
+
+    # List ingest jobs, newest first.
+    def list_ingest_jobs(skip: nil, limit: nil)
+      d = get("/v2/ingest#{list_query(skip, limit)}")
+      jobs = d["jobs"].is_a?(Array) ? d["jobs"] : []
+      IngestJobList.new(
+        jobs: jobs.map { |j| to_ingest_job_summary(j) },
+        skip: num(d["skip"]),
+        limit: num(d["limit"], 20),
+        has_more: d["has_more"] == true
+      )
+    end
+
+    # Get one ingest job by id (`ing_...`).
+    def get_ingest_job(job_id)
+      to_ingest_job(get("/v2/ingest/#{encode_path(job_id)}"))
+    end
+
+    # Cancel a queued/processing ingest job. Idempotent: canceling an
+    # already-terminal job returns it unchanged.
+    def cancel_ingest_job(job_id)
+      to_ingest_job(delete_request("/v2/ingest/#{encode_path(job_id)}"))
+    end
+
+    # Re-deliver the completion webhook of a completed job (409 if the job
+    # is not completed, 400 if it has no webhook configured).
+    def retry_ingest_webhook(job_id)
+      d = post("/v2/ingest/#{encode_path(job_id)}/retry-webhook", nil)
+      WebhookRetryResult.new(
+        job_id: str(d["job_id"]),
+        delivered: d["delivered"] == true,
+        attempts: num(d["attempts"]),
+        status_code: opt_num(d["status_code"]),
+        detail: str(d["detail"])
+      )
+    end
+
+    # Get (creating on first call) the project's webhook signing secret and
+    # the header/scheme details needed to verify deliveries.
+    def get_webhook_secret
+      to_webhook_secret(get("/v2/ingest/webhook-secret"))
+    end
+
+    # Rotate the webhook signing secret. Signatures made with the previous
+    # secret stop verifying immediately.
+    def rotate_webhook_secret
+      to_webhook_secret(post("/v2/ingest/webhook-secret/rotate", nil))
+    end
+
+    # ------------------------------------------------------------------
+    # Watch — recurring change monitoring
+    # ------------------------------------------------------------------
+
+    # Create a watcher that re-renders `url` on a fixed cadence (hourly
+    # floor) and notifies on changes via email and/or webhook.
+    def create_watcher(url, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
+                        webhook_url: nil, notify_email: nil)
+      body = { url: url }
+      body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
+      body[:diff_mode] = diff_mode unless diff_mode.nil?
+      body[:track_fields] = track_fields unless track_fields.nil?
+      body[:webhook_url] = webhook_url unless webhook_url.nil?
+      body[:notify_email] = notify_email unless notify_email.nil?
+      to_watcher(post("/v2/watch", body))
+    end
+
+    # List watchers, newest first.
+    def list_watchers(skip: nil, limit: nil)
+      d = get("/v2/watch#{list_query(skip, limit)}")
+      watchers = d["watchers"].is_a?(Array) ? d["watchers"] : []
+      WatcherList.new(
+        watchers: watchers.map { |w| to_watcher_summary(w) },
+        skip: num(d["skip"]),
+        limit: num(d["limit"], 20),
+        has_more: d["has_more"] == true
+      )
+    end
+
+    # Get one watcher by id (`wat_...`). Deleted watchers read as 404.
+    def get_watcher(watcher_id)
+      to_watcher(get("/v2/watch/#{encode_path(watcher_id)}"))
+    end
+
+    # Page through a watcher's check history, newest first.
+    def get_watcher_snapshots(watcher_id, limit: nil)
+      query = limit.nil? ? "" : "?limit=#{limit}"
+      d = get("/v2/watch/#{encode_path(watcher_id)}/snapshots#{query}")
+      snapshots = d["snapshots"].is_a?(Array) ? d["snapshots"] : []
+      WatcherSnapshotList.new(
+        watcher_id: str(d["watcher_id"]),
+        snapshots: snapshots.map { |s| to_watcher_snapshot(s) },
+        limit: num(d["limit"], 20)
+      )
+    end
+
+    # Update a watcher. At least one field is required. Set webhook_url to
+    # "" to clear the webhook; resuming a paused watcher re-checks the
+    # plan's watcher cap.
+    def update_watcher(watcher_id, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
+                        webhook_url: nil, notify_email: nil, status: nil)
+      body = {}
+      body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
+      body[:diff_mode] = diff_mode unless diff_mode.nil?
+      body[:track_fields] = track_fields unless track_fields.nil?
+      body[:webhook_url] = webhook_url unless webhook_url.nil?
+      body[:notify_email] = notify_email unless notify_email.nil?
+      body[:status] = status unless status.nil?
+      raise Enconvert::Error, "update_watcher: provide at least one field to update" if body.empty?
+
+      to_watcher(patch("/v2/watch/#{encode_path(watcher_id)}", body))
+    end
+
+    # Soft-delete a watcher (idempotent). Returns the tombstoned watcher
+    # with status "deleted".
+    def delete_watcher(watcher_id)
+      to_watcher(delete_request("/v2/watch/#{encode_path(watcher_id)}"))
+    end
+
+    private
+
+    # ------------------------------------------------------------------
+    # HTTP helpers
+    # ------------------------------------------------------------------
+
+    def json_request(method, path, body: nil, headers: {})
+      resp = @transport.request(method, path, headers: headers, body: body)
+      raise_for_status(resp)
+      JSON.parse(resp.body)
+    end
+
+    def post(path, body)
+      json_request(:post, path,
+                   body: body.nil? ? nil : JSON.generate(body),
+                   headers: { "Content-Type" => "application/json" })
+    end
+
+    def get(path)
+      json_request(:get, path)
+    end
+
+    def patch(path, body)
+      json_request(:patch, path, body: JSON.generate(body), headers: { "Content-Type" => "application/json" })
+    end
+
+    def delete_request(path)
+      json_request(:delete, path)
+    end
+
+    def encode_path(value)
+      ERB::Util.url_encode(value.to_s)
+    end
+
+    def list_query(skip, limit)
+      params = []
+      params << "skip=#{skip}" unless skip.nil?
+      params << "limit=#{limit}" unless limit.nil?
+      params.empty? ? "" : "?#{params.join('&')}"
+    end
+
+    # ------------------------------------------------------------------
+    # Request serializers
+    # ------------------------------------------------------------------
+
+    def require_one_artifact_output(method_name, opts)
+      # Server default when outputs is omitted: ["markdown", "structured"].
+      effective = opts.key?(:outputs) ? Array(opts[:outputs]) : %w[markdown structured]
+      artifact_count = effective.count { |o| Enums::PERCEIVE_ARTIFACT_OUTPUTS.include?(o.to_s) }
+      return if artifact_count == 1
+
+      raise Enconvert::Error,
+            "#{method_name}: direct download requires exactly one artifact-producing output " \
+            "(#{Enums::PERCEIVE_ARTIFACT_OUTPUTS.join(', ')})"
+    end
+
+    def serialize_perceive_options(o)
+      out = {}
+      out[:outputs] = o[:outputs] if o.key?(:outputs)
+      out[:extract] = o[:extract] if o.key?(:extract)
+      out[:schema] = o[:schema] if o.key?(:schema)
+      out[:wait_for] = o[:wait_for] if o.key?(:wait_for)
+      out[:wait_timeout_ms] = o[:wait_timeout_ms] if o.key?(:wait_timeout_ms)
+      out[:js_code] = o[:js_code] if o.key?(:js_code)
+      out[:viewport] = o[:viewport] if o.key?(:viewport)
+      out[:headers] = o[:headers] if o.key?(:headers)
+      out[:cookies] = o[:cookies] if o.key?(:cookies)
+      out[:auth] = o[:auth] if o.key?(:auth)
+      out[:proxy_url] = o[:proxy_url] if o.key?(:proxy_url)
+      out[:geolocation] = o[:geolocation] if o.key?(:geolocation)
+      out[:action_chain] = o[:action_chain] if o.key?(:action_chain)
+      out[:cache_mode] = o[:cache_mode] if o.key?(:cache_mode)
+      out[:pdf_options] = serialize_pdf_options(o[:pdf_options]) if o.key?(:pdf_options)
+      out[:block_resources] = o[:block_resources] if o.key?(:block_resources)
+      out[:respect_robots] = o[:respect_robots] if o.key?(:respect_robots)
+      out[:mobile] = o[:mobile] if o.key?(:mobile)
+      out[:only_main_content] = o[:only_main_content] if o.key?(:only_main_content)
+      out
+    end
+
+    def serialize_css_field(f)
+      out = { name: f[:name], type: f[:type] }
+      out[:selector] = f[:selector] if f.key?(:selector)
+      out[:attribute] = f[:attribute] if f.key?(:attribute)
+      out[:pattern] = f[:pattern] if f.key?(:pattern)
+      out[:default] = f[:default] if f.key?(:default)
+      out[:transform] = f[:transform] if f.key?(:transform)
+      out[:fields] = f[:fields].map { |nf| serialize_css_field(nf) } if f.key?(:fields)
+      out
+    end
+
+    def serialize_css_schema(s)
+      out = { base_selector: s[:base_selector], fields: s[:fields].map { |f| serialize_css_field(f) } }
+      out[:name] = s[:name] if s.key?(:name)
+      out[:target_field] = s[:target_field] if s.key?(:target_field)
+      out
+    end
+
+    # ------------------------------------------------------------------
+    # Response mappers. Optional fields may be absent entirely
+    # (response_model_exclude_none) — every access is guarded.
+    # ------------------------------------------------------------------
+
+    def str(v, fallback = "")
+      v.is_a?(String) ? v : fallback
+    end
+
+    def opt_str(v)
+      v.is_a?(String) ? v : nil
+    end
+
+    def num(v, fallback = 0)
+      v.is_a?(Numeric) ? v : fallback
+    end
+
+    def opt_num(v)
+      v.is_a?(Numeric) ? v : nil
+    end
+
+    def str_arr(v)
+      v.is_a?(Array) ? v.select { |x| x.is_a?(String) } : []
+    end
+
+    def opt_obj(v)
+      v.is_a?(Hash) ? v : nil
+    end
+
+    def to_tokens(v)
+      d = v.is_a?(Hash) ? v : nil
+      V2Tokens.new(input: num(d && d["input"]), output: num(d && d["output"]))
+    end
+
+    def to_output_artifact(v)
+      d = v.is_a?(Hash) ? v : {}
+      V2OutputArtifact.new(
+        url: opt_str(d["url"]),
+        object_key: str(d["object_key"]),
+        size_bytes: num(d["size_bytes"]),
+        content_type: str(d["content_type"], "application/octet-stream"),
+        expires_in: num(d["expires_in"], 900)
+      )
+    end
+
+    def to_perceive_result(d)
+      raw_outputs = opt_obj(d["outputs"]) || {}
+      outputs = {}
+      raw_outputs.each { |name, artifact| outputs[name] = to_output_artifact(artifact) }
+      PerceiveResult.new(
+        operation_id: str(d["operation_id"]),
+        status: d["status"],
+        url: str(d["url"]),
+        url_final: opt_str(d["url_final"]),
+        content_hash: opt_str(d["content_hash"]),
+        render_quality: opt_num(d["render_quality"]),
+        cache_hit: d["cache_hit"] == true,
+        outputs: outputs,
+        structured: opt_obj(d["structured"]),
+        extraction_tier: opt_str(d["extraction_tier"]),
+        tokens: to_tokens(d["tokens"]),
+        cost_cents: num(d["cost_cents"]),
+        duration_ms: opt_num(d["duration_ms"]),
+        error: opt_str(d["error"]),
+        warnings: str_arr(d["warnings"]),
+        status_code: opt_num(d["status_code"]),
+        deductions: opt_obj(d["deductions"]) || {},
+        options_echo: opt_obj(d["options_echo"])
+      )
+    end
+
+    def filename_from_disposition(header)
+      return nil if header.nil?
+
+      match = header.match(/filename="?([^";]+)"?/i)
+      match && match[1]
+    end
+
+    # Maps a raw direct-download Net::HTTP response (body = artifact bytes,
+    # metadata in headers) onto a PerceiveDirectResult.
+    def to_perceive_direct_result(resp)
+      PerceiveDirectResult.new(
+        content: resp.body.to_s,
+        content_type: resp["Content-Type"] || "application/octet-stream",
+        filename: filename_from_disposition(resp["Content-Disposition"]),
+        operation_id: resp["X-Operation-Id"].to_s,
+        object_key: resp["X-Object-Key"].to_s,
+        cache_hit: resp["X-Cache-Hit"] == "true",
+        render_quality: resp["X-Render-Quality"]&.to_f,
+        source_status_code: resp["X-Source-Status-Code"]&.to_i,
+        content_hash: resp["X-Content-Hash"],
+        warnings_count: resp["X-Warnings-Count"].to_i
+      )
+    end
+
+    def to_perceive_batch_result(d)
+      items = d["items"].is_a?(Array) ? d["items"] : []
+      PerceiveBatchResult.new(
+        job_id: str(d["job_id"]),
+        status: d["status"],
+        output_mode: opt_str(d["output_mode"]) || "manifest",
+        total: num(d["total"]),
+        completed: num(d["completed"]),
+        failed: num(d["failed"]),
+        pending: num(d["pending"]),
+        zip: d["zip"].nil? ? nil : to_output_artifact(d["zip"]),
+        items: items.map { |i| to_perceive_result(i) },
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_discover_result(d)
+      DiscoverResult.new(
+        url: str(d["url"]),
+        mode: d["mode"],
+        total: num(d["total"]),
+        urls: str_arr(d["urls"]),
+        pages_crawled: num(d["pages_crawled"]),
+        truncated: d["truncated"] == true,
+        robots_respected: d["robots_respected"] == true,
+        sources: opt_obj(d["sources"]) || {},
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_lookup_item(d)
+      perceive = opt_obj(d["perceive"])
+      LookupItem.new(
+        title: opt_str(d["title"]),
+        url: opt_str(d["url"]),
+        snippet: opt_str(d["snippet"]),
+        position: opt_num(d["position"]),
+        source: opt_str(d["source"]),
+        date: opt_str(d["date"]),
+        image_url: opt_str(d["image_url"]),
+        thumbnail_url: opt_str(d["thumbnail_url"]),
+        extra: opt_obj(d["extra"]) || {},
+        perceive: perceive ? to_perceive_result(perceive) : nil
+      )
+    end
+
+    def to_lookup_result(d)
+      results = d["results"].is_a?(Array) ? d["results"] : []
+      LookupResult.new(
+        lookup_id: opt_num(d["lookup_id"]),
+        query: str(d["query"]),
+        category: d["category"],
+        country: opt_str(d["country"]),
+        locale: opt_str(d["locale"]),
+        time_filter: opt_str(d["time_filter"]),
+        total: num(d["total"]),
+        results: results.map { |r| to_lookup_item(r) },
+        perceive_top: num(d["perceive_top"]),
+        perceive_operation_ids: str_arr(d["perceive_operation_ids"]),
+        answer_box: opt_obj(d["answer_box"]),
+        knowledge_graph: opt_obj(d["knowledge_graph"]),
+        credits: opt_num(d["credits"]),
+        cost_cents: num(d["cost_cents"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_distill_item(d)
+      DistillItem.new(
+        url: str(d["url"]),
+        url_final: opt_str(d["url_final"]),
+        status: opt_str(d["status"]) || "completed",
+        data: opt_obj(d["data"]),
+        extraction_tier: opt_str(d["extraction_tier"]) || "none",
+        fields_from_css: num(d["fields_from_css"]),
+        fields_from_llm: num(d["fields_from_llm"]),
+        render_quality: opt_num(d["render_quality"]),
+        tokens: to_tokens(d["tokens"]),
+        cost_cents: num(d["cost_cents"]),
+        error: opt_str(d["error"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_distill_result(d)
+      results = d["results"].is_a?(Array) ? d["results"] : []
+      DistillResult.new(
+        operation_id: str(d["operation_id"]),
+        total: num(d["total"]),
+        completed: num(d["completed"]),
+        failed: num(d["failed"]),
+        results: results.map { |r| to_distill_item(r) },
+        total_cost_cents: num(d["total_cost_cents"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_ingest_job(d)
+      IngestJob.new(
+        job_id: str(d["job_id"]),
+        status: d["status"],
+        mode: d["mode"],
+        pages_discovered: num(d["pages_discovered"]),
+        pages_processed: num(d["pages_processed"]),
+        pages_failed: num(d["pages_failed"]),
+        total_chunks: num(d["total_chunks"]),
+        output_url: opt_str(d["output_url"]),
+        error_message: opt_str(d["error_message"]),
+        webhook_url: opt_str(d["webhook_url"]),
+        webhook_delivered: d["webhook_delivered"] == true,
+        created_at: opt_str(d["created_at"]),
+        completed_at: opt_str(d["completed_at"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_ingest_job_summary(d)
+      IngestJobSummary.new(
+        job_id: str(d["job_id"]),
+        status: d["status"],
+        mode: d["mode"],
+        pages_discovered: num(d["pages_discovered"]),
+        pages_processed: num(d["pages_processed"]),
+        pages_failed: num(d["pages_failed"]),
+        total_chunks: num(d["total_chunks"]),
+        output_url: opt_str(d["output_url"]),
+        error_message: opt_str(d["error_message"]),
+        webhook_configured: d["webhook_configured"] == true,
+        webhook_delivered: d["webhook_delivered"] == true,
+        created_at: opt_str(d["created_at"]),
+        completed_at: opt_str(d["completed_at"])
+      )
+    end
+
+    def to_webhook_secret(d)
+      WebhookSecret.new(
+        secret: str(d["secret"]),
+        signature_header: str(d["signature_header"]),
+        timestamp_header: str(d["timestamp_header"]),
+        signature_scheme: str(d["signature_scheme"]),
+        replay_tolerance_seconds: num(d["replay_tolerance_seconds"]),
+        rotated: d["rotated"] == true
+      )
+    end
+
+    def to_watcher(d)
+      Watcher.new(
+        watcher_id: str(d["watcher_id"]),
+        url: str(d["url"]),
+        status: d["status"],
+        frequency_minutes: num(d["frequency_minutes"]),
+        diff_mode: d["diff_mode"],
+        track_fields: opt_obj(d["track_fields"]),
+        webhook_url: opt_str(d["webhook_url"]),
+        notify_email: d["notify_email"] != false,
+        consecutive_errors: num(d["consecutive_errors"]),
+        checks_count: num(d["checks_count"]),
+        last_check_at: opt_str(d["last_check_at"]),
+        next_check_at: opt_str(d["next_check_at"]),
+        last_change_at: opt_str(d["last_change_at"]),
+        created_at: opt_str(d["created_at"]),
+        updated_at: opt_str(d["updated_at"])
+      )
+    end
+
+    def to_watcher_summary(d)
+      WatcherSummary.new(
+        watcher_id: str(d["watcher_id"]),
+        url: str(d["url"]),
+        status: d["status"],
+        frequency_minutes: num(d["frequency_minutes"]),
+        checks_count: num(d["checks_count"]),
+        consecutive_errors: num(d["consecutive_errors"]),
+        last_check_at: opt_str(d["last_check_at"]),
+        next_check_at: opt_str(d["next_check_at"]),
+        last_change_at: opt_str(d["last_change_at"]),
+        created_at: opt_str(d["created_at"])
+      )
+    end
+
+    def to_watcher_snapshot(d)
+      changes = d["changes"].is_a?(Array) ? d["changes"].select { |c| c.is_a?(Hash) } : []
+      WatcherSnapshot.new(
+        checked_at: str(d["checked_at"]),
+        has_changes: d["has_changes"] == true,
+        similarity: opt_num(d["similarity"]),
+        render_quality: opt_num(d["render_quality"]),
+        change_count: num(d["change_count"]),
+        changes: changes
+      )
+    end
+  end
+end