wrangle 0.1.0 → 0.2.0

data/lib/wrangle/desktop_effect.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module Wrangle
+  # Verifies the intended target-specific postcondition after a delivered native action. A changed
+  # desktop revision is never proof by itself: unrelated animation, status text, or another control
+  # may have changed at the same time. When the intended target cannot be identified or its required
+  # state is outside partial coverage, the effect remains unverified.
+  class DesktopEffect
+    OPERATIONS = %w[PRESS SET_TEXT CLEAR TOGGLE EXPAND COLLAPSE].freeze
+    TOGGLE_ROLES = %w[checkbox switch radiobutton].freeze
+
+    class << self
+      def verify(proposal, before, after)
+        return "unverified" unless before.is_a?(Hash) && after.is_a?(Hash)
+
+        operation = proposal["operation"].to_s
+        candidate = proposal["candidate"]
+        return "unverified" unless OPERATIONS.include?(operation) && candidate.is_a?(Hash)
+
+        match, target = find_target(candidate, after)
+        return missing_target_effect(operation, before, after) if match == :missing
+        return "unverified" unless match == :found
+
+        case operation
+        when "PRESS" then verify_press(candidate, target, before, after)
+        when "SET_TEXT" then verify_set_text(candidate, target, proposal["text"], before, after)
+        when "CLEAR" then verify_clear(candidate, target, before, after)
+        when "TOGGLE" then verify_toggle(candidate, target, before, after)
+        when "EXPAND", "COLLAPSE" then verify_expansion(operation, candidate, target, before, after)
+        end
+      end
+
+      private
+
+      def find_target(candidate, observation)
+        elements = observation_elements(observation)
+        target_key = candidate["target_key"].to_s
+        unless target_key.empty?
+          keyed = elements.select do |element|
+            element["target_key"] == target_key && element["role"].to_s == candidate["role"].to_s
+          end
+          return [:found, keyed.first] if keyed.one?
+          return [:ambiguous, nil] if keyed.length > 1
+        end
+
+        visible = elements.select { |element| same_visible_identity?(candidate, element) }
+        return [:found, visible.first] if visible.one?
+        return [:missing, nil] if visible.empty?
+
+        [:ambiguous, nil]
+      end
+
+      def observation_elements(observation)
+        tree = observation["tree"]
+        return Array(observation["candidates"]) unless tree.is_a?(Hash)
+
+        elements = []
+        stack = [tree]
+        until stack.empty?
+          node = stack.pop
+          elements << normalize_tree_node(node)
+          stack.concat(Array(node["children"]).grep(Hash))
+        end
+        elements
+      end
+
+      def normalize_tree_node(node)
+        {
+          "role" => node["role"], "subrole" => node["subrole"], "label" => node["name"],
+          "value" => node["value"], "states" => node["states"], "operations" => node["operations"],
+          "children_count" => node["children_count"], "target_key" => node["target_key"]
+        }
+      end
+
+      def same_visible_identity?(before, after)
+        before["role"].to_s == after["role"].to_s &&
+          before["subrole"].to_s == after["subrole"].to_s &&
+          before["label"].to_s == after["label"].to_s
+      end
+
+      def missing_target_effect(operation, before, after)
+        return "unverified" unless operation == "PRESS" && complete?(after)
+
+        changed_revision?(before, after) ? "verified" : "unverified"
+      end
+
+      def verify_press(candidate, target, before, after)
+        if calculator_digit_press_effect?(candidate, before, after)
+          return changed_revision?(before, after) ? "verified" : "unverified"
+        end
+        return "unchanged" if press_signature(candidate) == press_signature(target)
+
+        changed_revision?(before, after) ? "verified" : "unverified"
+      end
+
+      def calculator_digit_press_effect?(candidate, before, after)
+        digit = candidate["label"].to_s
+        return false unless candidate["role"] == "button" && digit.match?(/\A\d\z/)
+        return false unless same_calculator_scope?(before, after) && complete?(before) && complete?(after)
+
+        previous = calculator_display(before)
+        current = calculator_display(after)
+        return false unless previous && current
+
+        expected = previous == "0" ? digit : "#{previous}#{digit}"
+        current == expected
+      end
+
+      def same_calculator_scope?(before, after)
+        before_scope = before["scope"]
+        after_scope = after["scope"]
+        before_scope.is_a?(Hash) && after_scope.is_a?(Hash) &&
+          before_scope["app"] == "Calculator" && after_scope["app"] == "Calculator" &&
+          !before_scope["id"].to_s.empty? && before_scope["id"] == after_scope["id"]
+      end
+
+      def calculator_display(observation)
+        values = observation_elements(observation).filter_map do |element|
+          next unless element["role"] == "statictext" && element["value"].is_a?(String)
+
+          normalized = element["value"].gsub(/[^0-9.-]/, "")
+          normalized unless normalized.empty?
+        end.uniq
+        values.one? ? values.first : nil
+      end
+
+      def press_signature(candidate)
+        {
+          "label" => candidate["label"].to_s, "value" => candidate["value"],
+          "states" => Array(candidate["states"]).map(&:to_s).reject { |state| state == "focused" }.uniq.sort,
+          "operations" => Array(candidate["operations"]).map(&:to_s).reject { |op| op == "DRILL" }.uniq.sort,
+          "children_count" => candidate["children_count"]
+        }
+      end
+
+      def verify_set_text(candidate, target, text, before, after)
+        return "unverified" unless text.is_a?(String) && same_visible_identity?(candidate, target)
+        return "unchanged" unless target["value"] == text
+        return "unchanged" if candidate["value"] == text
+
+        changed_revision?(before, after) ? "verified" : "unverified"
+      end
+
+      def verify_clear(candidate, target, before, after)
+        return "unverified" unless same_visible_identity?(candidate, target) && target["value"].is_a?(String)
+        return "unchanged" unless target["value"].strip.empty?
+        return "unchanged" if candidate["value"].is_a?(String) && candidate["value"].strip.empty?
+
+        changed_revision?(before, after) ? "verified" : "unverified"
+      end
+
+      def verify_toggle(candidate, target, before, after)
+        return "unverified" unless same_visible_identity?(candidate, target)
+
+        previous = toggle_value(candidate)
+        current = toggle_value(target)
+        return "unverified" if previous.nil? || current.nil?
+        return "unchanged" if previous == current
+
+        changed_revision?(before, after) ? "verified" : "unverified"
+      end
+
+      def toggle_value(candidate)
+        return unless TOGGLE_ROLES.include?(candidate["role"].to_s)
+
+        scalar = candidate["value"]
+        return scalar if [true, false].include?(scalar)
+        return scalar == 1 if [0, 1].include?(scalar)
+
+        states = Array(candidate["states"]).map(&:to_s)
+        states.include?("checked") || states.include?("selected")
+      end
+
+      def verify_expansion(operation, candidate, target, before, after)
+        return "unverified" unless same_visible_identity?(candidate, target) &&
+                                   candidate.key?("states") && target.key?("states")
+
+        previous = Array(candidate["states"]).map(&:to_s).include?("expanded")
+        current = Array(target["states"]).map(&:to_s).include?("expanded")
+        expected_previous = operation == "COLLAPSE"
+        return "unverified" unless previous == expected_previous
+        return "unchanged" if previous == current
+
+        changed_revision?(before, after) ? "verified" : "unverified"
+      end
+
+      def complete?(observation)
+        observation["complete"] == true && observation.dig("coverage", "truncated") != true
+      end
+
+      def changed_revision?(before, after)
+        previous = before["revision"]
+        current = after["revision"]
+        previous.is_a?(String) && current.is_a?(String) && !previous.empty? && previous != current
+      end
+    end
+  end
+end

data/lib/wrangle/desktop_observation.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+require_relative "errors"
+require_relative "observation"
+
+module Wrangle
+  # Normalizes a driver's hierarchical AX snapshot without throwing away its progressive structure.
+  class DesktopObservation
+    TEXT_ROLES = %w[textfield textarea searchfield combobox].freeze
+    TOGGLE_ROLES = %w[checkbox switch radiobutton].freeze
+    WINDOW_CONTROL = /\A(?:close|close window|close all|quit(?: .*)?|minimize|zoom|full ?screen)\z/i
+    VISIBLE_TEXT_CHARS = 500
+
+    attr_reader :state
+
+    class << self
+      # Provider and result evidence includes non-actionable text as well as controls. Returning only
+      # candidates made read-only tasks impossible to prove and allowed DONE without seeing the fact.
+      def evidence_items(observation)
+        items = []
+        seen = {}
+        collect_evidence(observation["tree"], items, seen) if observation["tree"].is_a?(Hash)
+        Array(observation["candidates"]).each do |candidate|
+          add_evidence(items, seen, candidate.slice("role", "label", "value", "states"))
+        end
+        items
+      end
+
+      # An action is addressable only by the semantic fields exposed to the provider. Opaque AX refs,
+      # paths, and target keys must never break a tie the provider cannot see.
+      def action_descriptor(candidate, operation)
+        {
+          "operation" => operation.to_s, "role" => candidate["role"].to_s,
+          "label" => bounded_visible(candidate["label"].to_s),
+          "value" => bounded_visible(scalar_value(candidate["value"])),
+          "states" => Array(candidate["states"]).map(&:to_s).uniq.sort
+        }.compact
+      end
+
+      def action_ambiguities(candidates)
+        entries = action_entries(candidates)
+        counts = entries.tally
+        seen = {}
+        entries.filter_map do |descriptor|
+          next unless counts[descriptor] > 1
+          next if seen[descriptor]
+
+          seen[descriptor] = true
+          descriptor.merge("matches" => counts[descriptor])
+        end
+      end
+
+      def action_unambiguous?(candidates, candidate, operation)
+        wanted = action_descriptor(candidate, operation)
+        Array(candidates).one? do |possible|
+          Array(possible["operations"]).map(&:to_s).uniq.include?(operation.to_s) &&
+            action_descriptor(possible, operation) == wanted
+        end
+      end
+
+      def reject_ambiguous_actions(candidates)
+        ambiguities = action_ambiguities(candidates)
+        ambiguous = ambiguities.to_h { |descriptor| [descriptor.except("matches"), true] }
+        filtered = Array(candidates).filter_map do |candidate|
+          operations = Array(candidate["operations"]).map(&:to_s).uniq.reject do |operation|
+            ambiguous[action_descriptor(candidate, operation)]
+          end
+          candidate.merge("operations" => operations) unless operations.empty?
+        end
+        [filtered, ambiguities]
+      end
+
+      private
+
+      def action_entries(candidates)
+        Array(candidates).flat_map do |candidate|
+          Array(candidate["operations"]).map(&:to_s).uniq.map do |operation|
+            action_descriptor(candidate, operation)
+          end
+        end
+      end
+
+      def bounded_visible(value)
+        return value unless value.is_a?(String) && value.length > VISIBLE_TEXT_CHARS
+
+        "#{value[0, VISIBLE_TEXT_CHARS]}…"
+      end
+
+      def collect_evidence(node, items, seen)
+        visible = { "role" => node["role"].to_s.downcase, "label" => node["name"].to_s,
+                    "value" => scalar_value(node["value"]), "states" => Array(node["states"]).map(&:to_s) }
+        add_evidence(items, seen, visible) unless visible["role"] == "window"
+        Array(node["children"]).each do |child|
+          collect_evidence(child, items, seen) if child.is_a?(Hash)
+        end
+      end
+
+      def add_evidence(items, seen, visible)
+        visible = visible.compact
+        label = visible["label"].to_s
+        value = visible["value"]
+        return if label.empty? && (value.nil? || value.to_s.empty?)
+        return if seen[visible]
+
+        seen[visible] = true
+        items << visible
+      end
+
+      def scalar_value(value)
+        value if value.is_a?(String) || value.is_a?(Numeric) || value == true || value == false
+      end
+    end
+
+    def initialize(scope:, snapshot:, window:, driver: "macos", read_only: false)
+      @scope = scope
+      @snapshot = snapshot
+      @window = window
+      @driver = driver
+      @read_only = read_only
+      validate!
+      @state = build
+    end
+
+    def revision = @state.fetch("revision")
+    def candidates = @state.fetch("candidates")
+
+    private
+
+    def validate!
+      unless @snapshot.is_a?(Hash) && @snapshot["snapshot_id"].is_a?(String) && @snapshot["tree"].is_a?(Hash)
+        raise DriverError, "Desktop observation requires a complete driver snapshot"
+      end
+      unless @window.is_a?(Hash) && @window["id"] == @scope.root &&
+             @window["process_instance"] == @scope.process_instance
+        raise ScopeLost, "The observed window is no longer the attached root surface"
+      end
+
+      observed_id = @snapshot.dig("window", "id")
+      raise ScopeLost, "The AX snapshot belongs to another window" if observed_id && observed_id != @scope.root
+    end
+
+    def build
+      observed_candidates = []
+      walk(@snapshot["tree"], observed_candidates)
+      candidates, ambiguous_actions = self.class.reject_ambiguous_actions(observed_candidates)
+      body = {
+        "schema" => "wrangle.observation.v1", "driver" => @driver,
+        "scope" => { "id" => @scope.id, "root" => @scope.root, "app" => @scope.app,
+                     "pid" => @scope.pid, "process_instance" => @scope.process_instance },
+        "window" => @window.slice("id", "bounds", "focused", "visible"),
+        "snapshot_id" => @snapshot["snapshot_id"], "complete" => @snapshot.fetch("complete", true),
+        "tree" => @snapshot["tree"], "candidates" => candidates,
+        "ambiguous_actions" => ambiguous_actions,
+        "coverage" => coverage(candidates, ambiguous_actions)
+      }
+      body["read_only"] = true if @read_only
+      body["revision"] = fingerprint(body)
+      body
+    end
+
+    def walk(node, candidates)
+      candidate = candidate(node)
+      candidates << candidate if candidate
+      Array(node["children"]).each { |child| walk(child, candidates) if child.is_a?(Hash) }
+    end
+
+    def candidate(node)
+      ref = node["ref_id"]
+      return nil unless ref.is_a?(String)
+
+      role = node["role"].to_s.downcase
+      operations = operations(role, node)
+      operations &= ["DRILL"] if Array(node["states"]).map(&:to_s).include?("disabled")
+      operations -= ["PRESS"] if window_control?(role, node)
+      return nil if operations.empty?
+
+      {
+        "ref" => ref, "role" => role, "subrole" => node["subrole"], "label" => node["name"].to_s,
+        "value" => scalar(node["value"]), "states" => Array(node["states"]).map(&:to_s),
+        "operations" => operations, "children_count" => node["children_count"],
+        "target_key" => node["target_key"]
+      }.compact
+    end
+
+    def window_control?(_role, node)
+      subrole = node["subrole"].to_s.delete_prefix("AX").downcase
+      %w[closebutton minimizebutton zoombutton fullscreenbutton].include?(subrole) ||
+        node["name"].to_s.match?(WINDOW_CONTROL)
+    end
+
+    def operations(role, node)
+      observed = node["operations"]
+      return native_operations(observed, node) if observed.is_a?(Array)
+
+      offered = []
+      offered << "DRILL" if node["children_count"].to_i.positive?
+      offered.push("SET_TEXT", "CLEAR") if TEXT_ROLES.include?(role)
+      offered << "TOGGLE" if TOGGLE_ROLES.include?(role)
+      offered << "PRESS" unless role.empty? || role == "window"
+      offered
+    end
+
+    def native_operations(observed, node)
+      allowed = %w[PRESS SET_TEXT CLEAR TOGGLE EXPAND COLLAPSE]
+      offered = observed.map(&:to_s) & allowed
+      offered.unshift("DRILL") if node["children_count"].to_i.positive?
+      offered.uniq
+    end
+
+    def scalar(value)
+      value if value.is_a?(String) || value.is_a?(Numeric) || value == true || value == false
+    end
+
+    def coverage(candidates, ambiguous_actions)
+      truncated = @snapshot["complete"] == false || truncated?(@snapshot["tree"])
+      provenance = Array(@snapshot["provenance"])
+      provenance = ["ax"] if provenance.empty?
+      { "provenance" => provenance, "truncated" => truncated,
+        "candidate_count" => candidates.length,
+        "ambiguous_action_count" => ambiguous_actions.sum { |action| action.fetch("matches") },
+        "ocr" => "not_used" }
+    end
+
+    def truncated?(node)
+      node["children_count"].to_i.positive? || Array(node["children"]).any? do |child|
+        child.is_a?(Hash) && truncated?(child)
+      end
+    end
+
+    def fingerprint(body)
+      # Non-actionable AX text can be intrinsically volatile (Finder's free-space status changes
+      # when the proposal log is written). Revisions bind scope, coverage, and every offered target,
+      # while the full tree remains available for inspection and progressive traversal.
+      semantic = without_refs(body.except("snapshot_id", "tree"))
+      Digest::SHA256.hexdigest(JSON.generate(Observation.canonical(semantic)))[0, 24]
+    end
+
+    def without_refs(value)
+      case value
+      when Hash
+        value.each_with_object({}) do |(key, child), stripped|
+          stripped[key] = without_refs(child) unless %w[ref ref_id].include?(key)
+        end
+      when Array then value.map { |child| without_refs(child) }
+      else value
+      end
+    end
+  end
+end

data/lib/wrangle/desktop_policy.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module Wrangle
+  # Deterministic permission policy. A provider may assess risk, but cannot make this gate weaker.
+  class DesktopPolicy
+    CREDENTIAL = /password|passcode|verification code|one[- ]time|security code|api key|secret|token/i
+    CONSEQUENCE = /\b(send|submit|delete|remove|trash|erase|buy|purchase|pay|confirm|save|allow|
+                       install|quit|sign\s+out)\b/ix
+    CONSEQUENTIAL_OPERATIONS = %w[TOGGLE].freeze
+    WINDOW_CONTROL = /\A(?:close|close window|close all|quit(?: .*)?|minimize|zoom|full ?screen)\z/i
+    WINDOW_SUBROLES = %w[closebutton minimizebutton zoombutton fullscreenbutton].freeze
+
+    Assessment = Data.define(:classification, :consequential, :permitted, :reason) do
+      def to_h
+        { "classification" => classification, "consequential" => consequential,
+          "permitted" => permitted, "reason" => reason }.compact
+      end
+    end
+
+    def assess(scope:, candidate:, operation:)
+      label = candidate["label"].to_s
+      if operation == "SET_TEXT" && label.match?(CREDENTIAL)
+        return Assessment.new(classification: "credential_handoff", consequential: true, permitted: false,
+                              reason: "Credentials must be entered through password-manager or SSO UI")
+      end
+      if operation == "PRESS" && window_control?(candidate, label)
+        return Assessment.new(classification: "scope_violation", consequential: true, permitted: false,
+                              reason: "Wrangle never closes or manages a handed-over root window")
+      end
+
+      consequential = CONSEQUENTIAL_OPERATIONS.include?(operation) ||
+                      (operation == "PRESS" && label.match?(CONSEQUENCE))
+      reason = "#{scope.app} action #{label.inspect} may have an external or persistent consequence" if consequential
+      Assessment.new(classification: consequential ? "consequential" : "standard",
+                     consequential:, permitted: true, reason:)
+    end
+
+    private
+
+    def window_control?(candidate, label)
+      subrole = candidate["subrole"].to_s.delete_prefix("AX").downcase
+      WINDOW_SUBROLES.include?(subrole) || label.match?(WINDOW_CONTROL)
+    end
+  end
+end

data/lib/wrangle/desktop_progressive_observation.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Wrangle
+  # Escalates a bounded AX skeleton to a deeper root-window observation before an autonomous action.
+  # The provider must re-decide from the completed state; target choices never cross this boundary.
+  module DesktopProgressiveObservation
+    private
+
+    def escalate_observation?
+      return false if @view
+
+      previous = current
+      started = monotonic
+      @observation = @driver.observe(@scope, skeleton: false)
+      @full_observation = true
+      @proposals.clear
+      fields = {
+        "scope_id" => @scope.id, "revision" => @observation["revision"],
+        "previous_revision" => previous["revision"], "complete" => @observation["complete"],
+        "candidates" => @observation["candidates"].length,
+        "ambiguous_actions" => @observation.dig("coverage", "ambiguous_action_count"),
+        "elapsed_ms" => elapsed_ms(started)
+      }
+      @log&.record("observe_escalated", fields)
+      true
+    end
+
+    def fresh_observation
+      return @driver.drill(@scope, **@view) if @view
+
+      @driver.observe(@scope, skeleton: !@full_observation)
+    end
+  end
+end

data/lib/wrangle/desktop_proposal.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+require_relative "desktop_observation"
+require_relative "errors"
+
+module Wrangle
+  # Validates and binds one typed action to one semantic desktop revision.
+  class DesktopProposal
+    TTL = 300
+    MAX_TEXT = 2000
+
+    def self.build(scope:, observation:, request:, policy:, created_at:)
+      index = request["ref"]
+      candidate = candidate(observation, index)
+      operation = request["operation"] || infer_operation(candidate)
+      unless candidate["operations"].include?(operation) && operation != "DRILL"
+        raise ArgumentError, "#{operation.inspect} was not offered for action ##{index.inspect}"
+      end
+      unless DesktopObservation.action_unambiguous?(observation["candidates"], candidate, operation)
+        raise ArgumentError, "#{operation.inspect} for action ##{index.inspect} matches multiple visible candidates"
+      end
+
+      validate_text!(operation, request["text"])
+      assessment = policy.assess(scope:, candidate:, operation:)
+      raise PolicyDenied, assessment.reason unless assessment.permitted
+
+      {
+        "id" => SecureRandom.hex(12), "scope_id" => scope.id, "revision" => observation["revision"],
+        "candidate" => candidate, "number" => index, "operation" => operation, "text" => request["text"],
+        "policy" => assessment.to_h, "created_at" => created_at
+      }
+    end
+
+    def self.compact(proposal)
+      text = proposal["text"]
+      {
+        "schema" => "wrangle.proposal.v1", "proposal_id" => proposal["id"],
+        "scope_id" => proposal["scope_id"], "revision" => proposal["revision"],
+        "action" => { "ref" => proposal["number"], "operation" => proposal["operation"],
+                      "role" => proposal.dig("candidate", "role"),
+                      "label" => proposal.dig("candidate", "label"),
+                      "text" => text && { "source" => "explicit", "characters" => text.length } }.compact,
+        "policy" => proposal["policy"]
+      }
+    end
+
+    def self.candidate(observation, index)
+      candidates = observation["candidates"]
+      unless index.is_a?(Integer) && index.positive? && index <= candidates.length
+        raise ArgumentError, "No action ##{index.inspect} in the current desktop observation"
+      end
+
+      candidates[index - 1]
+    end
+
+    def self.infer_operation(candidate)
+      actions = candidate["operations"] - ["DRILL"]
+      raise ArgumentError, "Specify one of #{actions.join(", ")}" unless actions.one?
+
+      actions.first
+    end
+    private_class_method :infer_operation
+
+    def self.validate_text!(operation, text)
+      raise ArgumentError, "Text is valid only for SET_TEXT" if operation != "SET_TEXT" && !text.nil?
+      return unless operation == "SET_TEXT"
+      return if text.is_a?(String) && !text.empty? && text.length <= MAX_TEXT && !text.include?("\0")
+
+      raise ArgumentError, "SET_TEXT requires an exact non-empty string of at most #{MAX_TEXT} characters"
+    end
+    private_class_method :validate_text!
+  end
+end