voice_control 0.1.0

data/lib/generators/voice_control/install/templates/voice_control.rb

@@ -0,0 +1,72 @@
+VoiceControl.configure do |config|
+  # Server-side Jev API key; accepts a string or a zero-argument callable.
+  config.api_key = ENV["JEV_API_KEY"]
+
+  # Jev model used to match natural-language commands to your vocabulary.
+  config.model = "jev-latest"
+
+  # Engine endpoints inherit this controller's authentication and callbacks.
+  config.parent_controller = "ApplicationController"
+
+  # Controls access to the widget and endpoints. Replace with your own access check.
+  # For example: -> { current_user&.admin? }. Access is denied until configured.
+  config.authorize = -> { false }
+
+  # Binds execution tickets to an account as well as its session; defaults to current_user.id when available.
+  # Override this for another authentication system.
+  # config.identity = -> { current_account.id }
+
+  # Filters browser-supplied context before matching; defaults to passing it through.
+  # Keep only the keys your commands need, especially if URLs contain private query parameters.
+  # config.context = ->(client_context) { client_context.slice("path") }
+
+  # Discovers labeled page controls for click, fill, select, check, scroll and submit commands.
+  # Sends control labels/IDs and dropdown labels to Jev; existing field values are excluded.
+  config.browser_actions = false
+
+  # Open/close shortcut; mod is Cmd on macOS and Ctrl elsewhere. Set nil to disable.
+  config.keyboard_shortcut = "mod+shift+u"
+
+  # Hold to speak, release to submit. Set nil to disable this shortcut.
+  config.push_to_talk_shortcut = "mod+shift+space"
+
+  # Speech recognition language as a BCP 47 tag, e.g. "uk-UA". Widget text and built-in page phrases stay English.
+  config.speech_language = "en-US"
+
+  # Corner for the launcher and panel: :bottom_right or :bottom_left.
+  config.widget_position = :bottom_right
+
+  # :small uses a 44px button and 20px icon; :normal uses 56px and 24px. Panel width is unchanged.
+  config.launcher_size = :normal
+
+  # Close and stop listening after this many milliseconds of inactivity; use a positive number.
+  config.idle_timeout = 120_000
+
+  # Maximum milliseconds per request, including reading its response; execution timeouts never retry automatically.
+  config.request_timeout = 30_000
+
+  # Matches below this confidence ask for disambiguation. Calibrate for your vocabulary.
+  config.confidence_threshold = 0.35
+
+  # Shows the latest transcript, matching details, Jev result and execution timing in the widget.
+  # Enable for trusted users while troubleshooting; copied details may contain private data.
+  config.debug = false
+
+  # Defaults to Rails.cache, with a MemoryStore fallback for development NullStore.
+  # Production needs a shared atomic cache for replay protection; see docs/deployment.md.
+  # config.execution_store = -> { Rails.cache }
+
+  # Optional matcher accepting transcript:, context:, commands:; nil uses Jev.
+  # Return command:, confidence:, candidates: without executing an action; see docs/configuration.md.
+  config.interpreter = nil
+
+  # Receives unexpected errors and command/controller details; the UI shows a generic message.
+  # The default logs only the error class and command key. Replace with your error tracker if needed.
+  # config.on_error = ->(error, details) { Rails.logger.error("VoiceControl #{error.class} command=#{details[:command]}") }
+
+  config.group "Navigation" do
+    config.command :home, description: "Open home", aliases: ["go home"], examples: ["open home"] do
+      execute { |_args, _context| VoiceControl::Result.navigate(main_app.root_path) }
+    end
+  end
+end

data/lib/voice_control/argument.rb

@@ -0,0 +1,59 @@
+require "bigdecimal"
+
+module VoiceControl
+  class Argument
+    TYPES = [:string, :integer, :decimal, :boolean, :enum].freeze
+    attr_reader :name, :type, :prompt
+
+    def initialize(name, type, required: true, default: nil, extract: nil, validate: nil, values: nil, prompt: nil)
+      raise ArgumentError, "Unknown argument type: #{type}" unless TYPES.include?(type)
+      raise ArgumentError, "Enum requires values" if type == :enum && (values.nil? || values.empty?)
+
+      @name, @type, @required = name.to_s, type, required
+      @default, @extractor, @validator, @values = default, extract, validate, values&.map(&:to_s)
+      @prompt = prompt || "What is the #{name.to_s.tr('_', ' ')}?#{@values ? " Choose: #{@values.join(', ')}." : ''}"
+    end
+
+    def initial_value(controller, transcript, context)
+      value = controller.instance_exec(transcript, context, &@extractor) if @extractor
+      if value.nil?
+        value = @default.respond_to?(:call) ? controller.instance_exec(transcript, context, &@default) : @default
+      end
+      value
+    end
+
+    def coerce(value, controller:)
+      return nil if value.nil? && !@required
+      raise InvalidInput, prompt if value.nil? || value.to_s.strip.empty? || value.to_s.length > 2_000
+
+      text = value.to_s.strip
+      result = case type
+      when :integer
+        raise ArgumentError unless text.match?(/\A[+-]?\d+\z/)
+        Integer(text, 10)
+      when :decimal
+        raise ArgumentError unless text.match?(/\A[+-]?\d+(?:\.\d+)?\z/)
+        BigDecimal(text).to_s("F")
+      when :boolean
+        case text.downcase
+        when "true", "yes", "on" then true
+        when "false", "no", "off" then false
+        else raise ArgumentError
+        end
+      when :enum
+        @values.find { |option| option.tr("_", " ").casecmp?(text.tr("_", " ")) } || (raise ArgumentError)
+      else
+        text
+      end
+      raise ArgumentError if @validator && !controller.instance_exec(result, &@validator)
+
+      result
+    rescue ArgumentError, TypeError
+      raise InvalidInput, prompt
+    end
+
+    def as_json(*)
+      { name: name, type: type, required: @required, values: @values, prompt: prompt }.compact
+    end
+  end
+end

data/lib/voice_control/browser_actions.rb

@@ -0,0 +1,210 @@
+module VoiceControl
+  class BrowserActions
+    MAX_TARGETS = 200
+    PREFIX = "voice_control_browser_".freeze
+    SCROLL_DIRECTIONS = %w[up down top bottom].freeze
+    HISTORY_DIRECTIONS = %w[back forward].freeze
+    attr_reader :commands
+
+    def initialize(page)
+      unless page.is_a?(Hash) && JSON.generate(page).bytesize <= 131_072 &&
+          page["page_id"].is_a?(String) && page["page_id"].match?(/\A[a-z0-9-]{1,64}\z/i) &&
+          page["elements"].is_a?(Array) && page["elements"].length <= MAX_TARGETS
+        raise InvalidInput, "Invalid browser controls. Reload the page and try again."
+      end
+
+      @page_id = page["page_id"]
+      @targets = page["elements"].map { |target| validate_target(target) }
+      raise InvalidInput, "Duplicate browser controls" unless @targets.map { |target| target["ref"] }.uniq.length == @targets.length
+
+      @selected_ref = page["selected_ref"]
+      if @selected_ref && !@targets.any? { |target| target["ref"] == @selected_ref && (target["actions"] & %w[fill select]).any? }
+        raise InvalidInput, "Select an editable field first."
+      end
+
+      @command_targets = {}
+      @commands = @targets.flat_map { |target| build_commands(target) } + build_scroll_commands + build_history_commands + [build_submit_command]
+    end
+
+    def snapshot(keys)
+      refs = keys.filter_map { |key| @command_targets[key] }
+      return if refs.empty? && keys.none? { |key|
+        key == "#{PREFIX}submit" || SCROLL_DIRECTIONS.any? { |direction| key == "#{PREFIX}scroll_#{direction}" } ||
+          HISTORY_DIRECTIONS.any? { |direction| key == "#{PREFIX}history_#{direction}" }
+      }
+
+      { "page_id" => @page_id, "elements" => @targets.select { |target| refs.include?(target["ref"]) },
+        "selected_ref" => (@selected_ref if refs.include?(@selected_ref)) }.compact
+    end
+
+    def exact_click_matches(transcript)
+      phrase = transcript.squish.downcase
+      @commands.select do |command|
+        command.key.start_with?("#{PREFIX}click_") &&
+          command.aliases.any? { |name| name.start_with?("click ") && name.squish.downcase == phrase }
+      end
+    end
+
+    class << self
+      def extract_value(text, label:)
+        text[/["“](.+?)["”]/, 1] ||
+          text[/\A(?:fill|enter|type|set)\s+(?:the\s+)?#{Regexp.escape(label)}(?:\s+field)?\s+(?:with|to)\s+(.+)\z/i, 1] ||
+          text[/\b(?:fill|set)\b.+?\b(?:with|to)\s+(.+)\z/i, 1] ||
+          text[/\b(?:enter|type)\s+(.+?)\s+(?:in|into)\s+.+\z/i, 1]
+      end
+
+      def selected_value(text)
+        text[/\A(?:enter|type)\s+(.+)\z/im, 1]&.sub(/\A"(.*)"\z/m, '\1')&.sub(/\A“(.*)”\z/m, '\1')
+      end
+
+      def select_value(text, label:)
+        extract_value(text, label: label) ||
+          text[/\A(?:select|choose)\s+(.+?)\s+(?:from|in|for)\s+(?:the\s+)?#{Regexp.escape(label)}\z/i, 1] ||
+          (text[/\A(?:select|choose)\s+(.+)\z/i, 1] unless text.match?(/\A(?:select|choose)\s+#{Regexp.escape(label)}\z/i))
+      end
+    end
+
+    private
+
+    def validate_target(target)
+      unless target.is_a?(Hash) && target["ref"].is_a?(String) && target["ref"].match?(/\Ae\d{1,9}\z/)
+        raise InvalidInput, "Invalid browser control reference"
+      end
+      clean = target.slice("ref", "label", "id", "name", "tag", "type")
+      unless %w[label id name tag type].all? { |key| clean[key].is_a?(String) && clean[key].length <= 160 } && clean["label"].present?
+        raise InvalidInput, "Invalid browser control label"
+      end
+      allowed = case clean["tag"]
+      when "button", "a" then %w[click]
+      when "textarea" then %w[fill focus clear]
+      when "h1", "h2", "h3", "h4", "h5", "h6" then %w[reveal]
+      when "select"
+        options = target["options"]
+        unless options.is_a?(Array) && options.length.between?(1, 100) && JSON.generate(options).bytesize <= 4096 &&
+            options.all? { |option| option.is_a?(Hash) && option["ref"].is_a?(String) && option["ref"].match?(/\Ao\d{1,6}\z/) && option["label"].is_a?(String) && option["label"].strip.length.between?(1, 160) }
+          raise InvalidInput, "Invalid dropdown options"
+        end
+        clean["options"] = options.map { |option| option.slice("ref", "label") }
+        if options.map { |option| option["ref"] }.uniq.length != options.length ||
+            options.map { |option| option["label"].tr("_", " ").downcase }.uniq.length != options.length
+          raise InvalidInput, "Dropdown options must have distinct labels."
+        end
+        %w[select focus]
+      when "input"
+        if clean["type"] == "checkbox"
+          %w[check uncheck]
+        elsif clean["type"] == "radio"
+          %w[choose]
+        elsif %w[button submit reset].include?(clean["type"])
+          %w[click]
+        elsif %w[text search email tel url number date time].include?(clean["type"])
+          %w[fill focus clear]
+        else
+          []
+        end
+      else
+        clean["type"] == "button" ? %w[click] : []
+      end
+      raise InvalidInput, "Unsupported browser control" if allowed.empty?
+
+      clean.merge("actions" => allowed)
+    end
+
+    def build_commands(target)
+      actions = target["actions"] + (target["ref"] == @selected_ref ? ["enter"] : [])
+      actions << "clear_selected" if target["ref"] == @selected_ref && target["actions"].include?("clear")
+      actions.map do |action|
+        selected = %w[enter clear_selected].include?(action)
+        fill = %w[fill enter].include?(action)
+        key = "#{PREFIX}#{action}_#{target['ref']}"
+        @command_targets[key] = target["ref"]
+        next build_checked_command(target, key, action) if %w[check uncheck choose].include?(action)
+        next build_dropdown_command(target, key, selected: selected) if action == "select" || (selected && target["tag"] == "select")
+        label = target["label"]
+        identity = [target["tag"], ("id: #{target['id']}" if target["id"].present?), ("name: #{target['name']}" if target["name"].present?)].compact.join(", ")
+        result_action = { "enter" => "fill", "clear_selected" => "clear" }.fetch(action, action)
+        result = { kind: "browser", action: result_action, target: target["ref"], page_id: @page_id }
+        result[:selected] = true if selected
+        aliases = (action == "fill" ? %w[fill enter type set] : [action]).map { |verb| "#{verb} #{label}" }
+        aliases << label if action == "click" && label.match?(/\A(?:edit|delete|remove|view|open|manage)\s/i)
+        aliases = ["clear this field", "clear selected field", "clear the selected field"] if action == "clear_selected"
+        aliases = ["show #{label}", "show the #{label} section", "scroll to #{label}", "go to #{label} section"] if action == "reveal"
+        description = if selected
+          "#{action == 'enter' ? 'Enter a value into' : 'Clear'} the selected field: #{label}"
+        elsif action == "reveal"
+          "Show #{label} section"
+        else
+          "#{action.capitalize} #{label} (#{identity})"
+        end
+        Command.new(key, description: description, group: "On this page",
+          aliases: action == "enter" ? %w[enter type] : aliases, examples: []) do
+          if fill
+            argument :value, :string, prompt: "What should I enter in #{label}?",
+              extract: ->(text, _context) { selected ? BrowserActions.selected_value(text) : BrowserActions.extract_value(text, label: label) }
+          end
+          execute { |args, _context| fill ? result.merge(value: args[:value]) : result }
+        end
+      end
+    end
+
+    def build_dropdown_command(target, key, selected:)
+      page_id = @page_id
+      label = target["label"]
+      options = target["options"]
+      values = options.map { |option| option["label"] }
+      aliases = selected ? %w[enter type] : ["set #{label}", "select #{label}", "choose #{label}"] + values.flat_map { |value| ["select #{value}", "choose #{value}"] }
+      description = selected ? "Enter a choice into the selected dropdown: #{label}" : "Select #{label}"
+      Command.new(key, description: description, group: "On this page", aliases: aliases) do
+        argument :value, :enum, values: values,
+          prompt: "Which #{label}? Choose: #{values.join(', ')}.",
+          extract: ->(text, _context) { selected ? BrowserActions.selected_value(text) : BrowserActions.select_value(text, label: label) }
+        execute do |args, _context|
+          result = { kind: "browser", action: "select", target: target["ref"], page_id: page_id,
+            option: options.find { |option| option["label"] == args[:value] }.fetch("ref") }
+          result[:selected] = true if selected
+          result
+        end
+      end
+    end
+
+    def build_checked_command(target, key, action)
+      label = target["label"]
+      subject = label.sub(/\A(?:enable|disable)\s+/i, "")
+      verbs = { "check" => ["check", "enable", "turn on"], "uncheck" => ["uncheck", "disable", "turn off"], "choose" => ["choose", "select", "check"] }.fetch(action)
+      aliases = verbs.product([label, subject].uniq).map { |verb, name| "#{verb} #{name}" }
+      result = { kind: "browser", action: action, target: target["ref"], page_id: @page_id }
+      Command.new(key, description: "#{action.capitalize} #{label}", group: "On this page", aliases: aliases) do
+        execute { |_args, _context| result }
+      end
+    end
+
+    def build_scroll_commands
+      page_id = @page_id
+      SCROLL_DIRECTIONS.map do |direction|
+        aliases = ["scroll #{direction}", "scroll to #{direction}", "scroll to the #{direction}"]
+        aliases += ["back to top", "go to top"] if direction == "top"
+        Command.new("#{PREFIX}scroll_#{direction}", description: "Scroll #{direction}", group: "On this page", aliases: aliases) do
+          execute { |_args, _context| { kind: "browser", action: "scroll", direction: direction, page_id: page_id } }
+        end
+      end
+    end
+
+    def build_history_commands
+      page_id = @page_id
+      HISTORY_DIRECTIONS.map do |direction|
+        Command.new("#{PREFIX}history_#{direction}", description: "Go #{direction}", group: "On this page",
+          aliases: ["go #{direction}", "browser #{direction}", direction]) do
+          execute { |_args, _context| { kind: "browser", action: "history", direction: direction, page_id: page_id } }
+        end
+      end
+    end
+
+    def build_submit_command
+      page_id = @page_id
+      Command.new("#{PREFIX}submit", description: "Submit active form", group: "On this page",
+        aliases: ["submit", "submit form", "submit this form", "submit active form", "submit this field"]) do
+        execute { |_args, _context| { kind: "browser", action: "submit", page_id: page_id } }
+      end
+    end
+  end
+end

data/lib/voice_control/command.rb

@@ -0,0 +1,51 @@
+module VoiceControl
+  class Command
+    attr_reader :key, :description, :group, :aliases, :examples, :arguments, :executor
+
+    def initialize(key, description:, group:, aliases: [], examples: [], pages: nil, visible: -> { true }, authorize: ->(_args, _context) { true }, &block)
+      @key, @description, @group = key, description, group
+      @aliases, @examples = aliases, examples
+      @visibility, @authorization = visible, authorize
+      @pages = pages.nil? ? nil : Array(pages)
+      if @pages && (@pages.empty? || !@pages.all? { |page| page.is_a?(Regexp) || local_path?(page) })
+        raise ArgumentError, "Command pages must be local paths or regular expressions"
+      end
+      @arguments = []
+      instance_eval(&block)
+      raise ArgumentError, "Command #{key} needs an execute block" unless executor
+    end
+
+    def argument(name, type, **options)
+      raise ArgumentError, "Duplicate argument: #{name}" if arguments.any? { |argument| argument.name == name.to_s }
+
+      arguments << Argument.new(name, type, **options)
+    end
+
+    def execute(&block)
+      @executor = block
+    end
+
+    def visible?(controller)
+      !!controller.instance_exec(&@visibility)
+    end
+
+    def available_on?(path)
+      !@pages || (local_path?(path) && @pages.any? { |page| page.is_a?(Regexp) ? page.match?(path) : page == path })
+    end
+
+    def allowed?(controller, args, context)
+      visible?(controller) && !!controller.instance_exec(args, context, &@authorization)
+    end
+
+    def as_json(*)
+      { key: key, description: description, group: group, aliases: aliases, examples: examples,
+        arguments: arguments.map(&:as_json) }
+    end
+
+    private
+
+    def local_path?(path)
+      path.is_a?(String) && path.length <= 2_048 && path.match?(%r{\A/(?!/)[^?#\s\\]*\z})
+    end
+  end
+end

data/lib/voice_control/configuration.rb

@@ -0,0 +1,71 @@
+module VoiceControl
+  class Configuration
+    attr_accessor :api_key, :model, :parent_controller, :authorize, :context,
+      :on_error, :interpreter, :keyboard_shortcut, :idle_timeout, :confidence_threshold,
+      :execution_store, :identity, :browser_actions, :debug, :push_to_talk_shortcut
+    attr_reader :commands, :widget_position, :launcher_size, :request_timeout, :speech_language
+
+    def initialize
+      @model = "jev-latest"
+      @parent_controller = "ApplicationController"
+      @authorize = -> { false }
+      @identity = -> { respond_to?(:current_user, true) ? current_user&.id : nil }
+      @context = ->(client_context) { client_context }
+      @on_error = ->(error, details) { Rails.logger.error("VoiceControl #{error.class} command=#{details[:command]}") }
+      @keyboard_shortcut = "mod+shift+u"
+      @idle_timeout = 120_000
+      @request_timeout = 30_000
+      @confidence_threshold = 0.35
+      development_store = ActiveSupport::Cache::MemoryStore.new if Rails.env.development?
+      @execution_store = -> { Rails.cache.is_a?(ActiveSupport::Cache::NullStore) && development_store ? development_store : Rails.cache }
+      @commands = {}
+      @browser_actions = false
+      @debug = false
+      @widget_position = :bottom_right
+      @launcher_size = :normal
+      @push_to_talk_shortcut = "mod+shift+space"
+      @speech_language = "en-US"
+    end
+
+    def speech_language=(value)
+      raise ArgumentError, "Speech language must be a BCP 47 tag such as en-US or uk-UA" unless value.is_a?(String) && value.match?(/\A[a-z]{2,3}(?:-[a-z0-9]{1,8})*\z/i)
+
+      @speech_language = value
+    end
+
+    def widget_position=(value)
+      raise ArgumentError, "Widget position must be bottom_right or bottom_left" unless %w[bottom_right bottom_left].include?(value.to_s)
+
+      @widget_position = value.to_sym
+    end
+
+    def request_timeout=(value)
+      raise ArgumentError, "Request timeout must be an integer from 1,000 to 300,000 milliseconds" unless value.is_a?(Integer) && value.between?(1_000, 300_000)
+
+      @request_timeout = value
+    end
+
+    def launcher_size=(value)
+      raise ArgumentError, "Launcher size must be normal or small" unless %w[normal small].include?(value.to_s)
+
+      @launcher_size = value.to_sym
+    end
+
+    def group(name, &block)
+      previous = @group
+      @group = name.to_s
+      block.call(self)
+    ensure
+      @group = previous
+    end
+
+    def command(key, **options, &block)
+      key = key.to_s
+      raise ArgumentError, "Duplicate command: #{key}" if commands.key?(key)
+      raise ArgumentError, "Invalid command key" unless key.match?(/\A[a-z][a-z0-9_]*\z/) && key != "none"
+      raise ArgumentError, "Reserved browser command key" if key.start_with?(BrowserActions::PREFIX)
+
+      commands[key] = Command.new(key, group: @group || "Commands", **options, &block)
+    end
+  end
+end