enconvert 0.0.1

data/lib/enconvert/v2.rb

@@ -0,0 +1,688 @@
+# frozen_string_literal: true
+
+require "erb"
+require "json"
+
+require_relative "internal"
+require_relative "v2_results"
+
+# V2 API namespace, reached as `client.v2`.
+#
+# One method per V2 endpoint (21 total across six groups: perceive,
+# discover, lookup, distill, ingest, watch). Options are passed as
+# snake_case keyword arguments and serialized to the API's snake_case wire
+# format (a straight pass-through, since Ruby's own idiom already is
+# snake_case); responses are mapped back onto snake_case Struct readers.
+# User-data payloads (schemas, extracted data, tracked fields, diff
+# changes) pass through untouched as plain Ruby Hashes.
+#
+# All V2 endpoints require a private API key (public keys are rejected)
+# and are plan-gated: a disabled feature or exhausted monthly quota
+# raises QuotaError (HTTP 402).
+module Enconvert
+  class V2
+    include Internal
+
+    def initialize(transport)
+      @transport = transport
+    end
+
+    # ------------------------------------------------------------------
+    # Perceive — render a URL into agent-ready artifacts
+    # ------------------------------------------------------------------
+
+    # Render one URL into the requested outputs (markdown, screenshots, PDF,
+    # links, structured data, ...). Synchronous: returns the completed
+    # operation with 15-minute signed artifact URLs.
+    #
+    # opts accepts: outputs, extract, schema, wait_for, wait_timeout_ms,
+    # js_code, viewport, headers, cookies, auth, proxy_url, geolocation,
+    # action_chain, cache_mode, pdf_options, block_resources,
+    # respect_robots, mobile.
+    def perceive(url, **opts)
+      body = serialize_perceive_options(opts)
+      body[:url] = url
+      to_perceive_result(post("/v2/perceive", body))
+    end
+
+    # Re-fetch a perceive operation by id (`per_...`). Artifact URLs are
+    # freshly re-signed on every call.
+    def get_perceive_operation(operation_id)
+      to_perceive_result(get("/v2/perceive/#{encode_path(operation_id)}"))
+    end
+
+    # Perceive up to 1000 URLs with one shared options block. Small batches
+    # run inline (completed result); larger ones return status "queued" —
+    # poll get_perceive_batch with the job_id.
+    def perceive_batch(urls, output_mode: nil, **render_opts)
+      body = { urls: urls, options: serialize_perceive_options(render_opts) }
+      body[:output_mode] = output_mode unless output_mode.nil?
+      to_perceive_batch_result(post("/v2/perceive/batch", body))
+    end
+
+    # Poll a perceive batch by job_id. Items fill in as URLs complete.
+    def get_perceive_batch(job_id)
+      to_perceive_batch_result(get("/v2/perceive/batch/#{encode_path(job_id)}"))
+    end
+
+    # ------------------------------------------------------------------
+    # Discover — enumerate a site's URLs without rendering
+    # ------------------------------------------------------------------
+
+    # List a site's URLs via sitemap, HTTP crawl, or both. No browser
+    # rendering — fast and does not consume perceive quota.
+    #
+    # opts accepts: mode, max_urls, max_depth, include_patterns,
+    # exclude_patterns, same_domain_only, respect_robots.
+    def discover(url, **opts)
+      body = { url: url }
+      body[:mode] = opts[:mode] if opts.key?(:mode)
+      body[:max_urls] = opts[:max_urls] if opts.key?(:max_urls)
+      body[:max_depth] = opts[:max_depth] if opts.key?(:max_depth)
+      body[:include_patterns] = opts[:include_patterns] if opts.key?(:include_patterns)
+      body[:exclude_patterns] = opts[:exclude_patterns] if opts.key?(:exclude_patterns)
+      body[:same_domain_only] = opts[:same_domain_only] if opts.key?(:same_domain_only)
+      body[:respect_robots] = opts[:respect_robots] if opts.key?(:respect_robots)
+      to_discover_result(post("/v2/discover", body))
+    end
+
+    # ------------------------------------------------------------------
+    # Lookup — web search with optional auto-perceive
+    # ------------------------------------------------------------------
+
+    # Run a categorized web search. With perceive_top > 0, the top-N result
+    # URLs are auto-perceived (each consumes one perceive-quota unit) and
+    # carry their full PerceiveResult inline.
+    #
+    # opts accepts: category, country, locale, time_filter, num_results,
+    # page, location, autocorrect, perceive_top.
+    def lookup(query, **opts)
+      body = { query: query }
+      body[:category] = opts[:category] if opts.key?(:category)
+      body[:country] = opts[:country] if opts.key?(:country)
+      body[:locale] = opts[:locale] if opts.key?(:locale)
+      body[:time_filter] = opts[:time_filter] if opts.key?(:time_filter)
+      body[:num_results] = opts[:num_results] if opts.key?(:num_results)
+      body[:page] = opts[:page] if opts.key?(:page)
+      body[:location] = opts[:location] if opts.key?(:location)
+      body[:autocorrect] = opts[:autocorrect] if opts.key?(:autocorrect)
+      body[:perceive_top] = opts[:perceive_top] if opts.key?(:perceive_top)
+      to_lookup_result(post("/v2/lookup", body))
+    end
+
+    # ------------------------------------------------------------------
+    # Distill — schema-driven structured extraction
+    # ------------------------------------------------------------------
+
+    # Extract structured data matching `schema` from explicit URLs or from
+    # a discovered site. An optional css_schema answers fields for free;
+    # anything it misses escalates to the LLM tier (plan-gated).
+    #
+    # Provide exactly one of `urls` (non-empty Array) or `discover_from`
+    # (Hash with :url, :mode, :max_pages). `schema` is required.
+    def distill(urls: nil, discover_from: nil, schema: nil, css_schema: nil, wait_for: nil,
+                wait_timeout_ms: nil, headers: nil, cookies: nil, respect_robots: nil)
+      has_urls = urls.is_a?(Array) && !urls.empty?
+      has_discover = !discover_from.nil?
+      raise Enconvert::Error, "distill: provide exactly one of 'urls' or 'discover_from'" if has_urls == has_discover
+      raise Enconvert::Error, "distill: 'schema' is required and must be a Hash" unless schema.is_a?(Hash)
+
+      body = { schema: schema }
+      body[:urls] = urls if has_urls
+      if discover_from
+        df = { url: discover_from[:url] }
+        df[:mode] = discover_from[:mode] if discover_from.key?(:mode)
+        df[:max_pages] = discover_from[:max_pages] if discover_from.key?(:max_pages)
+        body[:discover_from] = df
+      end
+      body[:css_schema] = serialize_css_schema(css_schema) unless css_schema.nil?
+      body[:wait_for] = wait_for unless wait_for.nil?
+      body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
+      body[:headers] = headers unless headers.nil?
+      body[:cookies] = cookies unless cookies.nil?
+      body[:respect_robots] = respect_robots unless respect_robots.nil?
+      to_distill_result(post("/v2/distill", body))
+    end
+
+    # ------------------------------------------------------------------
+    # Ingest — site to RAG-ready JSONL chunks (always async)
+    # ------------------------------------------------------------------
+
+    # Start an ingest job: turn explicit URLs or a discovered site into
+    # chunked, RAG-ready JSONL. Always asynchronous — returns the queued
+    # job; poll get_ingest_job or configure webhook_url for completion.
+    #
+    # `mode` defaults to "urls" for validation purposes but is only sent on
+    # the wire when explicitly given (the API defaults it itself).
+    def ingest(mode: nil, url: nil, urls: nil, max_pages: nil, max_depth: nil, same_domain_only: nil,
+               include_patterns: nil, exclude_patterns: nil, respect_robots: nil, wait_for: nil,
+               wait_timeout_ms: nil, chunk: nil, webhook_url: nil)
+      effective_mode = mode || "urls"
+      if effective_mode == "urls"
+        raise Enconvert::Error, "ingest: mode 'urls' requires a non-empty 'urls' list" if urls.nil? || urls.empty?
+        raise Enconvert::Error, "ingest: mode 'urls' does not accept 'url'" unless url.nil?
+      else
+        raise Enconvert::Error, "ingest: mode '#{effective_mode}' requires a seed 'url'" if url.nil?
+        raise Enconvert::Error, "ingest: mode '#{effective_mode}' does not accept 'urls'" unless urls.nil?
+      end
+
+      body = {}
+      body[:mode] = mode unless mode.nil?
+      body[:url] = url unless url.nil?
+      body[:urls] = urls unless urls.nil?
+      body[:max_pages] = max_pages unless max_pages.nil?
+      body[:max_depth] = max_depth unless max_depth.nil?
+      body[:same_domain_only] = same_domain_only unless same_domain_only.nil?
+      body[:include_patterns] = include_patterns unless include_patterns.nil?
+      body[:exclude_patterns] = exclude_patterns unless exclude_patterns.nil?
+      body[:respect_robots] = respect_robots unless respect_robots.nil?
+      body[:wait_for] = wait_for unless wait_for.nil?
+      body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
+      if chunk
+        c = {}
+        c[:max_words] = chunk[:max_words] if chunk.key?(:max_words)
+        c[:sentence_overlap] = chunk[:sentence_overlap] if chunk.key?(:sentence_overlap)
+        body[:chunk] = c
+      end
+      body[:webhook_url] = webhook_url unless webhook_url.nil?
+      to_ingest_job(post("/v2/ingest", body))
+    end
+
+    # Ingest one or more uploaded FILES into RAG-ready JSONL chunks — the
+    # file counterpart of ingest(), sharing the same job lifecycle (mode
+    # "files"). PDF, DOCX, PPTX, XLSX, CSV, HTML, EPUB, TXT/MD and
+    # legacy/ODF office are accepted. Always asynchronous; poll
+    # get_ingest_job or configure a webhook.
+    #
+    # `files` accepts the same file-input union as convert_image (path
+    # string, IO-like/readable object, or { data:, filename: }); at least
+    # one file is required.
+    def ingest_files(files, chunk: nil, webhook_url: nil)
+      unless files.is_a?(Array) && !files.empty?
+        raise Enconvert::Error, "ingest_files: provide at least one file"
+      end
+
+      fields = files.map do |file|
+        part = to_file_part(file)
+        { name: "files", value: part.bytes, filename: part.filename, content_type: part.content_type }
+      end
+      if chunk
+        fields << { name: "max_words", value: chunk[:max_words] } if chunk.key?(:max_words)
+        fields << { name: "sentence_overlap", value: chunk[:sentence_overlap] } if chunk.key?(:sentence_overlap)
+      end
+      fields << { name: "webhook_url", value: webhook_url } unless webhook_url.nil?
+
+      body, boundary = build_multipart(fields)
+      resp = @transport.request(
+        :post, "/v2/ingest/files",
+        headers: { "Content-Type" => "multipart/form-data; boundary=#{boundary}" },
+        body: body
+      )
+      raise_for_status(resp)
+      to_ingest_job(JSON.parse(resp.body))
+    end
+
+    # List ingest jobs, newest first.
+    def list_ingest_jobs(skip: nil, limit: nil)
+      d = get("/v2/ingest#{list_query(skip, limit)}")
+      jobs = d["jobs"].is_a?(Array) ? d["jobs"] : []
+      IngestJobList.new(
+        jobs: jobs.map { |j| to_ingest_job_summary(j) },
+        skip: num(d["skip"]),
+        limit: num(d["limit"], 20),
+        has_more: d["has_more"] == true
+      )
+    end
+
+    # Get one ingest job by id (`ing_...`).
+    def get_ingest_job(job_id)
+      to_ingest_job(get("/v2/ingest/#{encode_path(job_id)}"))
+    end
+
+    # Cancel a queued/processing ingest job. Idempotent: canceling an
+    # already-terminal job returns it unchanged.
+    def cancel_ingest_job(job_id)
+      to_ingest_job(delete_request("/v2/ingest/#{encode_path(job_id)}"))
+    end
+
+    # Re-deliver the completion webhook of a completed job (409 if the job
+    # is not completed, 400 if it has no webhook configured).
+    def retry_ingest_webhook(job_id)
+      d = post("/v2/ingest/#{encode_path(job_id)}/retry-webhook", nil)
+      WebhookRetryResult.new(
+        job_id: str(d["job_id"]),
+        delivered: d["delivered"] == true,
+        attempts: num(d["attempts"]),
+        status_code: opt_num(d["status_code"]),
+        detail: str(d["detail"])
+      )
+    end
+
+    # Get (creating on first call) the project's webhook signing secret and
+    # the header/scheme details needed to verify deliveries.
+    def get_webhook_secret
+      to_webhook_secret(get("/v2/ingest/webhook-secret"))
+    end
+
+    # Rotate the webhook signing secret. Signatures made with the previous
+    # secret stop verifying immediately.
+    def rotate_webhook_secret
+      to_webhook_secret(post("/v2/ingest/webhook-secret/rotate", nil))
+    end
+
+    # ------------------------------------------------------------------
+    # Watch — recurring change monitoring
+    # ------------------------------------------------------------------
+
+    # Create a watcher that re-renders `url` on a fixed cadence (hourly
+    # floor) and notifies on changes via email and/or webhook.
+    def create_watcher(url, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
+                        webhook_url: nil, notify_email: nil)
+      body = { url: url }
+      body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
+      body[:diff_mode] = diff_mode unless diff_mode.nil?
+      body[:track_fields] = track_fields unless track_fields.nil?
+      body[:webhook_url] = webhook_url unless webhook_url.nil?
+      body[:notify_email] = notify_email unless notify_email.nil?
+      to_watcher(post("/v2/watch", body))
+    end
+
+    # List watchers, newest first.
+    def list_watchers(skip: nil, limit: nil)
+      d = get("/v2/watch#{list_query(skip, limit)}")
+      watchers = d["watchers"].is_a?(Array) ? d["watchers"] : []
+      WatcherList.new(
+        watchers: watchers.map { |w| to_watcher_summary(w) },
+        skip: num(d["skip"]),
+        limit: num(d["limit"], 20),
+        has_more: d["has_more"] == true
+      )
+    end
+
+    # Get one watcher by id (`wat_...`). Deleted watchers read as 404.
+    def get_watcher(watcher_id)
+      to_watcher(get("/v2/watch/#{encode_path(watcher_id)}"))
+    end
+
+    # Page through a watcher's check history, newest first.
+    def get_watcher_snapshots(watcher_id, limit: nil)
+      query = limit.nil? ? "" : "?limit=#{limit}"
+      d = get("/v2/watch/#{encode_path(watcher_id)}/snapshots#{query}")
+      snapshots = d["snapshots"].is_a?(Array) ? d["snapshots"] : []
+      WatcherSnapshotList.new(
+        watcher_id: str(d["watcher_id"]),
+        snapshots: snapshots.map { |s| to_watcher_snapshot(s) },
+        limit: num(d["limit"], 20)
+      )
+    end
+
+    # Update a watcher. At least one field is required. Set webhook_url to
+    # "" to clear the webhook; resuming a paused watcher re-checks the
+    # plan's watcher cap.
+    def update_watcher(watcher_id, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
+                        webhook_url: nil, notify_email: nil, status: nil)
+      body = {}
+      body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
+      body[:diff_mode] = diff_mode unless diff_mode.nil?
+      body[:track_fields] = track_fields unless track_fields.nil?
+      body[:webhook_url] = webhook_url unless webhook_url.nil?
+      body[:notify_email] = notify_email unless notify_email.nil?
+      body[:status] = status unless status.nil?
+      raise Enconvert::Error, "update_watcher: provide at least one field to update" if body.empty?
+
+      to_watcher(patch("/v2/watch/#{encode_path(watcher_id)}", body))
+    end
+
+    # Soft-delete a watcher (idempotent). Returns the tombstoned watcher
+    # with status "deleted".
+    def delete_watcher(watcher_id)
+      to_watcher(delete_request("/v2/watch/#{encode_path(watcher_id)}"))
+    end
+
+    private
+
+    # ------------------------------------------------------------------
+    # HTTP helpers
+    # ------------------------------------------------------------------
+
+    def json_request(method, path, body: nil, headers: {})
+      resp = @transport.request(method, path, headers: headers, body: body)
+      raise_for_status(resp)
+      JSON.parse(resp.body)
+    end
+
+    def post(path, body)
+      json_request(:post, path,
+                   body: body.nil? ? nil : JSON.generate(body),
+                   headers: { "Content-Type" => "application/json" })
+    end
+
+    def get(path)
+      json_request(:get, path)
+    end
+
+    def patch(path, body)
+      json_request(:patch, path, body: JSON.generate(body), headers: { "Content-Type" => "application/json" })
+    end
+
+    def delete_request(path)
+      json_request(:delete, path)
+    end
+
+    def encode_path(value)
+      ERB::Util.url_encode(value.to_s)
+    end
+
+    def list_query(skip, limit)
+      params = []
+      params << "skip=#{skip}" unless skip.nil?
+      params << "limit=#{limit}" unless limit.nil?
+      params.empty? ? "" : "?#{params.join('&')}"
+    end
+
+    # ------------------------------------------------------------------
+    # Request serializers
+    # ------------------------------------------------------------------
+
+    def serialize_perceive_options(o)
+      out = {}
+      out[:outputs] = o[:outputs] if o.key?(:outputs)
+      out[:extract] = o[:extract] if o.key?(:extract)
+      out[:schema] = o[:schema] if o.key?(:schema)
+      out[:wait_for] = o[:wait_for] if o.key?(:wait_for)
+      out[:wait_timeout_ms] = o[:wait_timeout_ms] if o.key?(:wait_timeout_ms)
+      out[:js_code] = o[:js_code] if o.key?(:js_code)
+      out[:viewport] = o[:viewport] if o.key?(:viewport)
+      out[:headers] = o[:headers] if o.key?(:headers)
+      out[:cookies] = o[:cookies] if o.key?(:cookies)
+      out[:auth] = o[:auth] if o.key?(:auth)
+      out[:proxy_url] = o[:proxy_url] if o.key?(:proxy_url)
+      out[:geolocation] = o[:geolocation] if o.key?(:geolocation)
+      out[:action_chain] = o[:action_chain] if o.key?(:action_chain)
+      out[:cache_mode] = o[:cache_mode] if o.key?(:cache_mode)
+      out[:pdf_options] = serialize_pdf_options(o[:pdf_options]) if o.key?(:pdf_options)
+      out[:block_resources] = o[:block_resources] if o.key?(:block_resources)
+      out[:respect_robots] = o[:respect_robots] if o.key?(:respect_robots)
+      out[:mobile] = o[:mobile] if o.key?(:mobile)
+      out
+    end
+
+    def serialize_css_field(f)
+      out = { name: f[:name], type: f[:type] }
+      out[:selector] = f[:selector] if f.key?(:selector)
+      out[:attribute] = f[:attribute] if f.key?(:attribute)
+      out[:pattern] = f[:pattern] if f.key?(:pattern)
+      out[:default] = f[:default] if f.key?(:default)
+      out[:transform] = f[:transform] if f.key?(:transform)
+      out[:fields] = f[:fields].map { |nf| serialize_css_field(nf) } if f.key?(:fields)
+      out
+    end
+
+    def serialize_css_schema(s)
+      out = { base_selector: s[:base_selector], fields: s[:fields].map { |f| serialize_css_field(f) } }
+      out[:name] = s[:name] if s.key?(:name)
+      out[:target_field] = s[:target_field] if s.key?(:target_field)
+      out
+    end
+
+    # ------------------------------------------------------------------
+    # Response mappers. Optional fields may be absent entirely
+    # (response_model_exclude_none) — every access is guarded.
+    # ------------------------------------------------------------------
+
+    def str(v, fallback = "")
+      v.is_a?(String) ? v : fallback
+    end
+
+    def opt_str(v)
+      v.is_a?(String) ? v : nil
+    end
+
+    def num(v, fallback = 0)
+      v.is_a?(Numeric) ? v : fallback
+    end
+
+    def opt_num(v)
+      v.is_a?(Numeric) ? v : nil
+    end
+
+    def str_arr(v)
+      v.is_a?(Array) ? v.select { |x| x.is_a?(String) } : []
+    end
+
+    def opt_obj(v)
+      v.is_a?(Hash) ? v : nil
+    end
+
+    def to_tokens(v)
+      d = v.is_a?(Hash) ? v : nil
+      V2Tokens.new(input: num(d && d["input"]), output: num(d && d["output"]))
+    end
+
+    def to_output_artifact(v)
+      d = v.is_a?(Hash) ? v : {}
+      V2OutputArtifact.new(
+        url: opt_str(d["url"]),
+        object_key: str(d["object_key"]),
+        size_bytes: num(d["size_bytes"]),
+        content_type: str(d["content_type"], "application/octet-stream"),
+        expires_in: num(d["expires_in"], 900)
+      )
+    end
+
+    def to_perceive_result(d)
+      raw_outputs = opt_obj(d["outputs"]) || {}
+      outputs = {}
+      raw_outputs.each { |name, artifact| outputs[name] = to_output_artifact(artifact) }
+      PerceiveResult.new(
+        operation_id: str(d["operation_id"]),
+        status: d["status"],
+        url: str(d["url"]),
+        url_final: opt_str(d["url_final"]),
+        content_hash: opt_str(d["content_hash"]),
+        render_quality: opt_num(d["render_quality"]),
+        cache_hit: d["cache_hit"] == true,
+        outputs: outputs,
+        structured: opt_obj(d["structured"]),
+        extraction_tier: opt_str(d["extraction_tier"]),
+        tokens: to_tokens(d["tokens"]),
+        cost_cents: num(d["cost_cents"]),
+        duration_ms: opt_num(d["duration_ms"]),
+        error: opt_str(d["error"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_perceive_batch_result(d)
+      items = d["items"].is_a?(Array) ? d["items"] : []
+      PerceiveBatchResult.new(
+        job_id: str(d["job_id"]),
+        status: d["status"],
+        output_mode: opt_str(d["output_mode"]) || "manifest",
+        total: num(d["total"]),
+        completed: num(d["completed"]),
+        failed: num(d["failed"]),
+        pending: num(d["pending"]),
+        zip: d["zip"].nil? ? nil : to_output_artifact(d["zip"]),
+        items: items.map { |i| to_perceive_result(i) },
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_discover_result(d)
+      DiscoverResult.new(
+        url: str(d["url"]),
+        mode: d["mode"],
+        total: num(d["total"]),
+        urls: str_arr(d["urls"]),
+        pages_crawled: num(d["pages_crawled"]),
+        truncated: d["truncated"] == true,
+        robots_respected: d["robots_respected"] == true,
+        sources: opt_obj(d["sources"]) || {},
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_lookup_item(d)
+      perceive = opt_obj(d["perceive"])
+      LookupItem.new(
+        title: opt_str(d["title"]),
+        url: opt_str(d["url"]),
+        snippet: opt_str(d["snippet"]),
+        position: opt_num(d["position"]),
+        source: opt_str(d["source"]),
+        date: opt_str(d["date"]),
+        image_url: opt_str(d["image_url"]),
+        thumbnail_url: opt_str(d["thumbnail_url"]),
+        extra: opt_obj(d["extra"]) || {},
+        perceive: perceive ? to_perceive_result(perceive) : nil
+      )
+    end
+
+    def to_lookup_result(d)
+      results = d["results"].is_a?(Array) ? d["results"] : []
+      LookupResult.new(
+        lookup_id: opt_num(d["lookup_id"]),
+        query: str(d["query"]),
+        category: d["category"],
+        country: opt_str(d["country"]),
+        locale: opt_str(d["locale"]),
+        time_filter: opt_str(d["time_filter"]),
+        total: num(d["total"]),
+        results: results.map { |r| to_lookup_item(r) },
+        perceive_top: num(d["perceive_top"]),
+        perceive_operation_ids: str_arr(d["perceive_operation_ids"]),
+        answer_box: opt_obj(d["answer_box"]),
+        knowledge_graph: opt_obj(d["knowledge_graph"]),
+        credits: opt_num(d["credits"]),
+        cost_cents: num(d["cost_cents"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_distill_item(d)
+      DistillItem.new(
+        url: str(d["url"]),
+        url_final: opt_str(d["url_final"]),
+        status: opt_str(d["status"]) || "completed",
+        data: opt_obj(d["data"]),
+        extraction_tier: opt_str(d["extraction_tier"]) || "none",
+        fields_from_css: num(d["fields_from_css"]),
+        fields_from_llm: num(d["fields_from_llm"]),
+        render_quality: opt_num(d["render_quality"]),
+        tokens: to_tokens(d["tokens"]),
+        cost_cents: num(d["cost_cents"]),
+        error: opt_str(d["error"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_distill_result(d)
+      results = d["results"].is_a?(Array) ? d["results"] : []
+      DistillResult.new(
+        operation_id: str(d["operation_id"]),
+        total: num(d["total"]),
+        completed: num(d["completed"]),
+        failed: num(d["failed"]),
+        results: results.map { |r| to_distill_item(r) },
+        total_cost_cents: num(d["total_cost_cents"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_ingest_job(d)
+      IngestJob.new(
+        job_id: str(d["job_id"]),
+        status: d["status"],
+        mode: d["mode"],
+        pages_discovered: num(d["pages_discovered"]),
+        pages_processed: num(d["pages_processed"]),
+        pages_failed: num(d["pages_failed"]),
+        total_chunks: num(d["total_chunks"]),
+        output_url: opt_str(d["output_url"]),
+        error_message: opt_str(d["error_message"]),
+        webhook_url: opt_str(d["webhook_url"]),
+        webhook_delivered: d["webhook_delivered"] == true,
+        created_at: opt_str(d["created_at"]),
+        completed_at: opt_str(d["completed_at"]),
+        warnings: str_arr(d["warnings"])
+      )
+    end
+
+    def to_ingest_job_summary(d)
+      IngestJobSummary.new(
+        job_id: str(d["job_id"]),
+        status: d["status"],
+        mode: d["mode"],
+        pages_discovered: num(d["pages_discovered"]),
+        pages_processed: num(d["pages_processed"]),
+        pages_failed: num(d["pages_failed"]),
+        total_chunks: num(d["total_chunks"]),
+        output_url: opt_str(d["output_url"]),
+        error_message: opt_str(d["error_message"]),
+        webhook_configured: d["webhook_configured"] == true,
+        webhook_delivered: d["webhook_delivered"] == true,
+        created_at: opt_str(d["created_at"]),
+        completed_at: opt_str(d["completed_at"])
+      )
+    end
+
+    def to_webhook_secret(d)
+      WebhookSecret.new(
+        secret: str(d["secret"]),
+        signature_header: str(d["signature_header"]),
+        timestamp_header: str(d["timestamp_header"]),
+        signature_scheme: str(d["signature_scheme"]),
+        replay_tolerance_seconds: num(d["replay_tolerance_seconds"]),
+        rotated: d["rotated"] == true
+      )
+    end
+
+    def to_watcher(d)
+      Watcher.new(
+        watcher_id: str(d["watcher_id"]),
+        url: str(d["url"]),
+        status: d["status"],
+        frequency_minutes: num(d["frequency_minutes"]),
+        diff_mode: d["diff_mode"],
+        track_fields: opt_obj(d["track_fields"]),
+        webhook_url: opt_str(d["webhook_url"]),
+        notify_email: d["notify_email"] != false,
+        consecutive_errors: num(d["consecutive_errors"]),
+        checks_count: num(d["checks_count"]),
+        last_check_at: opt_str(d["last_check_at"]),
+        next_check_at: opt_str(d["next_check_at"]),
+        last_change_at: opt_str(d["last_change_at"]),
+        created_at: opt_str(d["created_at"]),
+        updated_at: opt_str(d["updated_at"])
+      )
+    end
+
+    def to_watcher_summary(d)
+      WatcherSummary.new(
+        watcher_id: str(d["watcher_id"]),
+        url: str(d["url"]),
+        status: d["status"],
+        frequency_minutes: num(d["frequency_minutes"]),
+        checks_count: num(d["checks_count"]),
+        consecutive_errors: num(d["consecutive_errors"]),
+        last_check_at: opt_str(d["last_check_at"]),
+        next_check_at: opt_str(d["next_check_at"]),
+        last_change_at: opt_str(d["last_change_at"]),
+        created_at: opt_str(d["created_at"])
+      )
+    end
+
+    def to_watcher_snapshot(d)
+      changes = d["changes"].is_a?(Array) ? d["changes"].select { |c| c.is_a?(Hash) } : []
+      WatcherSnapshot.new(
+        checked_at: str(d["checked_at"]),
+        has_changes: d["has_changes"] == true,
+        similarity: opt_num(d["similarity"]),
+        render_quality: opt_num(d["render_quality"]),
+        change_count: num(d["change_count"]),
+        changes: changes
+      )
+    end
+  end
+end