elelem 0.12.0 → 0.13.0

data/lib/elelem/plugins/provider.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+Elelem.configure do |config|
+  config.setup(:provider) do |session|
+    session.commands.register("provider", description: "Switch provider: /provider NAME [MODEL]", completions: lambda {
+      Elelem::Config.names
+    }) do |args|
+      name, model = args.to_s.split
+      next session.output.say("  -> available: #{Elelem::Config.names.join(", ")}") if name.nil?
+
+      # --model carries over only to the provider it was given for.
+      options = if model then { model: }
+                elsif name == session.provider_name then session.provider_options
+                else {}
+                end
+      session.provider = Elelem::Config.build_provider(name, **options)
+      session.provider_name = name
+      session.provider_options = options
+      session.output.say "  -> switched to #{name}"
+    end
+  end
+end

data/lib/elelem/plugins/read.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+Elelem.configure do |config|
+  config.setup(:read) do |agent|
+    agent.toolbox.add("read",
+                      description: "Read file",
+                      params: { path: { type: "string" } },
+                      required: ["path"],
+                      aliases: ["open"]) do |a|
+      path = Pathname.new(a["path"]).expand_path
+      next { error: "not found" } unless path.exist?
+
+      content = path.binread
+      next { error: "binary file (#{content.bytesize} bytes)" } if content.include?("\0")
+
+      content = content.force_encoding(Encoding::UTF_8).scrub
+
+      { content: Elelem::Builtins.truncate(content), path: a["path"] }
+    end
+
+    agent.toolbox.after("read") do |_, result|
+      if result[:error]
+        agent.output.say "  ! #{result[:error]}"
+      else
+        agent.output.display_file(result[:path], fallback: result[:content])
+      end
+    end
+  end
+end

data/lib/elelem/plugins/reload.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Elelem.configure do |config|
+  config.setup(:reload) do |agent|
+    agent.commands.register("reload", description: "Reload plugin code from disk") do
+      Elelem::Config.reload(agent)
+    end
+  end
+end

data/lib/elelem/plugins/terminal.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# The default output. A gem that registers its own output first keeps it.
+Elelem.configure do |config|
+  config.output { Elelem::Builtins::Terminal.new } unless config.factories.key?(:output)
+end

data/lib/elelem/plugins/tools.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+Elelem.configure do |config|
+  config.setup(:tools) do |agent|
+    describe = lambda do |tool|
+      parameters = tool.to_h.dig(:function, :parameters)
+      lines = ["## #{tool.name}", "", tool.description, "", "### Parameters", ""]
+      parameters[:properties].each do |param, spec|
+        req = parameters[:required].include?(param.to_s) ? ", required" : ""
+        desc = spec[:description] ? " - #{spec[:description]}" : ""
+        lines << "- `#{param}` (#{spec[:type]}#{req})#{desc}"
+      end
+      lines << "" << "*aliases: #{tool.aliases.join(", ")}*" if tool.aliases.any?
+      lines.join("\n")
+    end
+
+    completions = -> { agent.toolbox.tools.keys }
+    agent.commands.register("tools", description: "List available tools", completions: completions) do |arg|
+      tool = agent.toolbox.tools[arg] if arg && !arg.empty?
+      if tool
+        agent.output.say describe.call(tool), as: :markdown
+      elsif arg && !arg.empty?
+        agent.output.say "Unknown tool: #{arg}"
+      else
+        # values copies the list, so a plugin adding tools from a thread cannot break this loop.
+        rows = agent.toolbox.tools.values.map { |t| "| #{t.name} | #{t.description.lines.first.chomp} |" }
+        agent.output.say "| Tool | Description |\n|------|-------------|\n#{rows.join("\n")}", as: :markdown
+      end
+    end
+  end
+end

data/lib/elelem/plugins/write.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+Elelem.configure do |config|
+  config.setup(:write) do |agent|
+    agent.toolbox.add("write",
+                      description: "Write file",
+                      params: { path: { type: "string" }, content: { type: "string" } },
+                      required: %w[path content],
+                      aliases: ["write<|channel|>"]) do |a|
+      path = Pathname.new(a["path"]).expand_path
+      FileUtils.mkdir_p(path.dirname)
+      { bytes: path.write(a["content"]), path: a["path"] }
+    end
+
+    agent.toolbox.before("write") do |args|
+      path = Pathname.new(args["path"]).expand_path
+      next unless path.exist? && agent.input.interactive?
+
+      Tempfile.create(["elelem", File.extname(path)]) do |t|
+        t.write(args["content"])
+        t.flush
+        diff, = Open3.capture2("diff", "--color=always", "-u", path.to_s, t.path)
+        agent.output.print(diff)
+      end
+    rescue Errno::ENOENT
+      nil # no diff installed
+    end
+
+    agent.toolbox.after("write") do |_, result|
+      agent.output.display_file(result[:path], fallback: "  -> #{result[:path]}")
+      # The model sees the checks in the write's own result.
+      if agent.toolbox.tools.key?("verify")
+        result[:verify] = agent.toolbox.run("verify", { "path" => result[:path] }).to_h
+      end
+    end
+  end
+end

data/lib/elelem/plugins.rb

@@ -3,6 +3,7 @@
 module Elelem
   class Plugins
     LOAD_PATHS = [
+      File.expand_path("plugins", __dir__),
       "~/.agents/plugins",
       ".agents/plugins"
     ].freeze
@@ -15,10 +16,14 @@ module Elelem
     def load!(force: false)
       return if @loaded && !force
 
-      @load_paths.each { |path| load_dir(File.expand_path(path)) }
+      dirs.each { |dir| load_dir(dir) }
       @loaded = true
     end
 
+    def dirs
+      @load_paths.map { |path| File.expand_path(path) }
+    end
+
     private
 
     def load_dir(dir)
@@ -30,7 +35,7 @@ module Elelem
     def load_plugin(file)
       load(file)
     rescue StandardError, ScriptError => e
-      warn "elelem: failed to load plugin #{file}: #{e.message}"
+      $stderr.puts "elelem: failed to load plugin #{file}: #{e.message}"
     end
   end
 end

data/lib/elelem/registration.rb

@@ -30,8 +30,13 @@ module Elelem
       self
     end
 
-    def clear_setups!
-      @setups.clear
+    # Drops setups defined in files under dirs. Gem setups are kept, since
+    # require will not run them again.
+    def clear_setups_from!(dirs)
+      @setups.reject! do |_, block|
+        file = block.source_location&.first.to_s
+        dirs.any? { |dir| file.start_with?("#{dir}/") }
+      end
     end
   end
 end

data/lib/elelem/registry.rb

@@ -12,9 +12,9 @@ module Elelem
       self
     end
 
-    def build_provider(name)
+    def build_provider(name, **)
       @plugins.load!
-      @registration.providers.fetch(name.to_s) { require_gem(name) }.call
+      @registration.providers.fetch(name.to_s) { require_gem(name) }.call(**)
     end
 
     def apply(session)
@@ -23,7 +23,7 @@ module Elelem
     end
 
     def reload(session)
-      @registration.clear_setups!
+      @registration.clear_setups_from!(@plugins.dirs)
       session.toolbox.clear!
       session.commands.clear!
       @plugins.load!(force: true)
@@ -34,10 +34,13 @@ module Elelem
       @registration.providers.keys
     end
 
-    def build_session(provider, toolbox: Toolbox.new, input: nil, output: nil)
+    def build_session(provider, toolbox: Toolbox.new, input: nil, output: nil, provider_options: {},
+                      template: nil, system_prompt: nil)
       commands = Commands.new
-      agent = Agent.new(build_provider(provider), toolbox: toolbox, output: output || build_output)
-      session = Session.new(agent, input: input || build_input(commands: commands), commands: commands)
+      agent = Agent.new(build_provider(provider, **provider_options), toolbox: toolbox, output: output || build_output,
+                                                                      template: template, system_prompt: system_prompt)
+      session = Session.new(agent, input: input || build_input(commands: commands), commands: commands,
+                                   provider_name: provider, provider_options: provider_options)
       apply(session)
       session
     end
@@ -57,7 +60,9 @@ module Elelem
     def require_gem(name)
       require "elelem/#{name}"
       @registration.providers.fetch(name.to_s) { unregistered_provider(name) }
-    rescue LoadError
+    rescue LoadError => e
+      raise unless e.path == "elelem/#{name}"
+
       raise "unknown provider: #{name.inspect}. Available: #{names.join(", ")}. Run: gem install elelem-#{name}"
     end
 

data/lib/elelem/repl.rb

@@ -15,20 +15,20 @@ module Elelem
 
     private
 
+    # Only lines typed at the prompt go in history, not answers to a tool.
+    # A line is a command only when its first word names one, so a path such
+    # as /etc/hosts reaches the model. One failed turn does not end the session.
     def dispatch_line(line)
-      line.start_with?("/") ? command(line) : @session.turn(line)
+      @session.input.remember(line) if @session.input.respond_to?(:remember)
+      command(line) || @session.turn(line)
+    rescue StandardError, ScriptError => e
+      Elelem.logger.debug("repl: #{e.message}")
+      @session.output.say(e.message, as: :error)
     end
 
     def command(line)
       name, args = line.delete_prefix("/").split(" ", 2)
-      @session.commands.run(name, args) || @session.output.say(@session.commands.names.join(" "))
-    rescue StandardError, ScriptError => e
-      warn_and_say(e)
-    end
-
-    def warn_and_say(error)
-      Elelem.logger.warn("repl: #{error.message}")
-      @session.output.say(error.message, as: :error)
+      line.start_with?("/") && @session.commands.run(name, args)
     end
   end
 end

data/lib/elelem/result.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module Elelem
   class Result
     extend Forwardable
@@ -21,7 +19,5 @@ module Elelem
     end
 
     def ok? = !!@payload[:ok]
-    def error? = !!@payload[:error]
-    def error = @payload[:error]
   end
 end

data/lib/elelem/session.rb

@@ -5,13 +5,17 @@ module Elelem
     extend Forwardable
 
     attr_reader :agent, :input, :commands
+    # The provider the session was built with, and its options (such as --model).
+    attr_accessor :provider_name, :provider_options
 
     def_delegators :@agent, :provider, :provider=, :toolbox, :output, :conversation, :context, :turn
 
-    def initialize(agent, input: NullInput.new, commands: Commands.new)
+    def initialize(agent, input: NullInput.new, commands: Commands.new, provider_name: nil, provider_options: {})
       @agent = agent
       @input = input
       @commands = commands
+      @provider_name = provider_name&.to_s
+      @provider_options = provider_options
     end
   end
 end

data/lib/elelem/stub_provider.rb

@@ -1,15 +1,76 @@
 # frozen_string_literal: true
 
 module Elelem
+  # No model. Understands a tool name and its arguments, and the few plain
+  # requests the docs show. Reads only the first line, so piped text never
+  # becomes arguments.
   class StubProvider
-    def fetch(messages, tools = [], &block)
-      block.call(type: "thinking", text: "considering the request")
-      return block.call(type: "saying", text: "done.") if messages.last[:role] == "tool"
+    REQUESTS = [
+      [/\Ashow me (\S+?)[.?!]?\z/i, "read", ->(m) { { "path" => m[1] } }],
+      [%r{\Acreate (\S*[./]\S*) with (.+)\z}i, "write", ->(m) { { "path" => m[1], "content" => m[2] } }],
+      [/\Awhat files are here\??\z/i, "execute", ->(_) { { "command" => "ls" } }],
+      [/\Arun (?:the )?tests?\z/i, "execute",
+       ->(_) { { "command" => File.executable?("bin/test") ? "bin/test" : "rake" } }],
+      [/\A(?:run|execute) `?(.+?)`?\z/i, "execute", ->(m) { { "command" => m[1] } }]
+    ].freeze
 
-      block.call(type: "saying", text: "let me check.")
-      return unless (tool = tools.first)
+    HELP = <<~TEXT
+      I need a model to understand that. Without one, type a tool and its
+      arguments (/tools lists them), or: show me README.md, create notes.txt
+      with hello, what files are here, run the tests.
+      For a model: gem install elelem-llama, then elelem -p llama
+    TEXT
 
-      block.call(type: "doing", id: "stub-1", name: tool[:function][:name], arguments: {})
+    def fetch(messages, tools = [], &)
+      return yield(type: "saying", text: summary(messages)) if messages.last[:role] == "tool"
+
+      name, args = understand(messages.last[:content], tools)
+      name ? act(name, args, messages.length, &) : yield(type: "saying", text: HELP)
+    end
+
+    private
+
+    def act(name, args, turn)
+      yield(type: "thinking", text: "I'll call #{name}.")
+      yield(type: "doing", id: "stub-#{turn}", name: name, arguments: args)
+    end
+
+    def understand(content, tools)
+      text = content.to_s.lines.first.to_s.strip
+      request(text, tools.map { |t| t.dig(:function, :name) }) || direct(text, tools)
+    end
+
+    def request(text, names)
+      REQUESTS.each do |pattern, name, build|
+        match = pattern.match(text)
+        return [name, build.call(match)] if match && names.include?(name)
+      end
+      nil
+    end
+
+    def direct(text, tools)
+      name, rest = text.split(/\s+/, 2)
+      tool = tools.find { |t| t.dig(:function, :name) == name }
+      args = arguments(tool, rest.to_s) if tool
+      [name, args] if args && file_like?(name, args)
+    end
+
+    def arguments(tool, rest)
+      required = tool.dig(:function, :parameters, :required) || []
+      required.zip(rest.split(/\s+/, required.size)).to_h.compact.transform_keys(&:to_s)
+    end
+
+    # A model would not write to "a" when asked to "write a commit message".
+    def file_like?(name, args)
+      name != "write" || args["path"].to_s.match?(%r{[./]})
+    end
+
+    def summary(messages)
+      name = messages.rfind { |m| m[:tool_calls] }&.dig(:tool_calls, 0, :name)
+      result = JSON.parse(messages.last[:content].to_s).except("ok")
+      return "#{name} failed: #{result["error"]}." if result["error"]
+
+      result.empty? ? "#{name} is done." : result.map { |k, v| "#{k}: #{v}" }.join("\n")
     end
   end
 end

data/lib/elelem/system_prompt.rb

@@ -5,7 +5,7 @@ module Elelem
     DEFAULT = File.read(File.expand_path("prompts/default.erb", __dir__)).freeze
     USER_AGENTS_MD = Pathname.new(Dir.home).join(".agents/AGENTS.md").freeze
 
-    attr_accessor :template
+    attr_reader :template
 
     def initialize(template = nil)
       @template = template || DEFAULT

data/lib/elelem/tool.rb

@@ -4,31 +4,18 @@ module Elelem
   class Tool
     attr_reader :name, :description, :aliases
 
-    def initialize(name, description:, schema: Schema.new, aliases: [], &handler)
+    # params and required are JSON Schema; every call is checked against them.
+    def initialize(name, description:, params: {}, required: [], aliases: [], &handler)
       @name = name
       @description = description
-      @schema = schema
       @aliases = aliases.freeze
       @handler = handler
+      @schema = { type: "object", properties: params.freeze, required: required.freeze }.freeze
     end
 
-    def call(args)
-      @handler.call(args)
-    end
-
-    def validate(args)
-      @schema.validate(args)
-    end
-
-    def to_h
-      {
-        type: "function",
-        function: {
-          name: name,
-          description: description,
-          parameters: @schema.to_h
-        }
-      }
-    end
+    def call(args) = @handler.call(args)
+    def validate(args) = (@validator ||= JSONSchemer.schema(@schema)).validate(args || {}).map { |e| e["error"] }
+    def inspect = "#<#{self.class} #{name}>"
+    def to_h = { type: "function", function: { name: name, description: description, parameters: @schema } }
   end
 end

data/lib/elelem/toolbox.rb

@@ -2,7 +2,7 @@
 
 module Elelem
   class Toolbox
-    attr_reader :tools, :hooks, :aliases
+    attr_reader :tools
 
     def initialize
       @tools = {}
@@ -11,17 +11,12 @@ module Elelem
     end
 
     def clear!
-      @tools.clear
-      @aliases.clear
-      @hooks[:before].clear
-      @hooks[:after].clear
+      [@tools, @aliases, @hooks[:before], @hooks[:after]].each(&:clear)
     end
 
     def add(name, description:, params: {}, required: [], aliases: [], &handler)
-      schema = Schema.new(params: params, required: required)
-      tool = Tool.new(name, description: description, schema: schema, aliases: aliases, &handler)
-      @tools[name] = tool
-      tool.aliases.each { |a| @aliases[a] = name }
+      aliases.each { |a| @aliases[a] = name }
+      @tools[name] = Tool.new(name, description:, params:, required:, aliases:, &handler)
     end
 
     def before(tool_name = :*, &block)
@@ -32,15 +27,17 @@ module Elelem
       @hooks[:after][tool_name] << block
     end
 
+    # The tool by name, else by alias, so a real tool named like an alias wins.
     def tool_for(name)
-      tools.fetch(@aliases.fetch(name, name)) { NullTool.new(name, available: tools.keys) }
+      tools[name] || tools[@aliases[name]]
     end
 
     def run(name, args)
       tool = tool_for(name)
-      invalid(tool, args) || to_result(dispatch(tool, args))
+      invalid(name, tool, args) || to_result(dispatch(tool, args))
     rescue StandardError, ScriptError => e
-      failure(name, args, e)
+      Elelem.logger.debug("toolbox: #{e.message}\n#{e.backtrace.join("\n")}")
+      Result.failure(error: e.message, name: name, args: args)
     end
 
     def to_a
@@ -49,35 +46,25 @@ module Elelem
 
     private
 
-    def invalid(tool, args)
-      errors = tool.validate(args)
+    def invalid(name, tool, args)
+      errors = tool ? tool.validate(args) : ["unknown tool: #{name.inspect}. Available: #{tools.keys.join(", ")}."]
       Result.failure(error: errors.join(", ")) if errors.any?
     end
 
+    # A tool fails by returning error:, or ok: false (as execute does for a
+    # non-zero exit).
     def to_result(result)
-      result[:error] ? Result.failure(result) : Result.success(result)
-    end
-
-    def failure(name, args, error)
-      Elelem.logger.warn("toolbox: #{error.message}\n#{error.backtrace.join("\n")}")
-      Result.failure(error: error.message, name: name, args: args)
+      result[:error] || result[:ok] == false ? Result.failure(result) : Result.success(result)
     end
 
     def dispatch(tool, args)
-      run_before_hooks(tool, args)
-      result = tool.call(args)
-      run_after_hooks(tool, args, result)
-      result
-    end
-
-    def run_before_hooks(tool, args)
-      @hooks[:before][:*].each { |h| h.call(args, tool_name: tool.name) }
-      @hooks[:before][tool.name].each { |h| h.call(args) }
+      run_hooks(:before, tool, args)
+      tool.call(args).tap { |result| run_hooks(:after, tool, args, result) }
     end
 
-    def run_after_hooks(tool, args, result)
-      @hooks[:after][:*].each { |h| h.call(args, result, tool_name: tool.name) }
-      @hooks[:after][tool.name].each { |h| h.call(args, result) }
+    def run_hooks(type, tool, *)
+      @hooks[type][:*].each { |h| h.call(*, tool_name: tool.name) }
+      @hooks[type][tool.name].each { |h| h.call(*) }
     end
   end
 end

data/lib/elelem/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Elelem
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

data/lib/elelem.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "erb"
+require "fileutils"
 require "forwardable"
 require "json"
 require "json_schemer"
@@ -8,42 +9,47 @@ require "logger"
 require "open3"
 require "pathname"
 require "shellwords"
+require "stringio"
+require "tempfile"
 
-require_relative "elelem/agent"
-require_relative "elelem/slash_command"
-require_relative "elelem/commands"
-require_relative "elelem/conversation"
-require_relative "elelem/input"
-require_relative "elelem/null_input"
+require_relative "elelem/version"
 require_relative "elelem/output"
 require_relative "elelem/null_output"
+require_relative "elelem/input"
+require_relative "elelem/null_input"
+require_relative "elelem/builtins"
+require_relative "elelem/builtins/terminal"
+require_relative "elelem/system_prompt"
+require_relative "elelem/conversation"
+require_relative "elelem/result"
+require_relative "elelem/tool"
+require_relative "elelem/toolbox"
+require_relative "elelem/commands"
+require_relative "elelem/agent"
+require_relative "elelem/session"
+require_relative "elelem/repl"
+require_relative "elelem/stub_provider"
 require_relative "elelem/plugins"
 require_relative "elelem/registration"
 require_relative "elelem/registry"
-require_relative "elelem/repl"
 require_relative "elelem/config"
-require_relative "elelem/result"
-require_relative "elelem/schema"
-require_relative "elelem/session"
-require_relative "elelem/stub_provider"
-require_relative "elelem/system_prompt"
-require_relative "elelem/tool"
-require_relative "elelem/transcript"
-require_relative "elelem/null_tool"
-require_relative "elelem/toolbox"
-require_relative "elelem/version"
 
 module Elelem
-  def self.start(provider: "stub", toolbox: Toolbox.new)
-    Repl.new(build(provider: provider, toolbox: toolbox)).run
+  def self.start(provider: "stub", toolbox: Toolbox.new, provider_options: {}, template: nil, system_prompt: nil)
+    Repl.new(build(provider: provider, toolbox: toolbox, provider_options: provider_options, template: template,
+                   system_prompt: system_prompt)).run
   end
 
-  def self.ask(prompt, provider: "stub", toolbox: Toolbox.new, output: NullOutput.new)
-    build(provider: provider, toolbox: toolbox, input: NullInput.new, output: output).turn(prompt)
+  def self.ask(prompt, provider: "stub", toolbox: Toolbox.new, output: NullOutput.new, provider_options: {},
+               template: nil, system_prompt: nil)
+    build(provider: provider, toolbox: toolbox, input: NullInput.new, output: output,
+          provider_options: provider_options, template: template, system_prompt: system_prompt).turn(prompt)
   end
 
-  def self.build(provider: "stub", toolbox: Toolbox.new, input: nil, output: nil)
-    Config.build_session(provider, toolbox: toolbox, input: input, output: output)
+  def self.build(provider: "stub", toolbox: Toolbox.new, input: nil, output: nil, provider_options: {},
+                 template: nil, system_prompt: nil)
+    Config.build_session(provider, toolbox: toolbox, input: input, output: output,
+                                   provider_options: provider_options, template: template, system_prompt: system_prompt)
   end
 
   def self.logger