samagotchi 0.102.0 → 0.103.0

data/lib/samagotchi/working_set.rb

@@ -47,18 +47,45 @@ module Samagotchi
     # collapsed); +state+ live|stale|forgotten|dropped; +tokens+ its content
     # outputs as sent; +ops+ read|edit for a file, read|write for a memory,
     # read for a URL, in first-seen order; +count+ the runs that touched
-    # it; +first_id+ the first one's chi id (ToolIds).
-    Item = Data.define(:kind, :key, :state, :tokens, :ops, :count, :first_id)
+    # it; +first_id+ the first one's chi id (ToolIds); +pinned+ whether
+    # the user pinned it (WorkingSetPin: the model's forget_outputs refuses
+    # its outputs), whatever the strategy; +has_drop+ whether any of its
+    # content outputs holds the user's applied drop (an undrop has
+    # something to take back: a forgotten item too, its other outputs
+    # stubbed by the model's layers).
+    Item = Data.define(:kind, :key, :state, :tokens, :ops, :count, :first_id, :pinned, :has_drop)
+
+    # An item the user added (/working-set add) that the next turn's note
+    # asks the model to read: its identity (+kind+, +key+, as an Item's)
+    # and how the note names it (+label+: a file relative to the session's
+    # folder, "memory NAME", a URL).
+    Add = Data.define(:kind, :key, :label)
 
     # A projection and the conversation's size (entries) it was built
     # from: the web keeps the newer of an event's and a GET's by it.
-    Snapshot = Data.define(:items, :at) do
-      def empty? = items.empty?
+    # +adds+: the pending note's adds (#pending_adds), not items yet.
+    Snapshot = Data.define(:items, :at, :adds) do
+      def initialize(items:, at:, adds: [])
+        super
+      end
+
+      def empty? = items.empty? && adds.empty?
 
-      # The JSON the event and the route carry: {items: [...], at:}.
-      def to_h = { items: items.map(&:to_h), at: at }
+      # The JSON the event and the route carry: {items: [...], at:}, and
+      # adds: [{kind, key, label}] while any is pending.
+      def to_h
+        hash = { items: items.map(&:to_h), at: at }
+        hash[:adds] = adds.map(&:to_h) unless adds.empty?
+        hash
+      end
     end
 
+    # The turn-start note (WorkingSetSteer#record!): a tail system message
+    # of this kind, its ops ([{"op", "kind", "key", "label"}]) in NOTE_OPS.
+    NOTE_KIND = "working_set"
+    NOTE_OPS = :working_set_ops
+    ADD = "add"
+
     FILE_OPS = { "read" => "read", "edit" => "edit", "write" => "edit" }.freeze
     CONTENT = %w[read memory_read web_fetch].freeze
     MEMORY_READ = Tools::MemoryRead::NAME
@@ -83,21 +110,52 @@ module Samagotchi
     # @param strategy [Symbol, Array<Symbol>] :none or the layers, as
     #   LLMContextView takes it (the session's current one)
     # @param stat [#call] File.stat, or a stand-in (specs)
+    # @param pins [Array<WorkingSetPin>] the session's pins
     # @return [Snapshot]
-    def build(conversation, cwd:, strategy:, stat: File.method(:stat))
+    def build(conversation, cwd:, strategy:, stat: File.method(:stat), pins: [])
       conversation = Array(conversation)
       view = LLMContextView.new(strategy: strategy)
+      pinned = pins.to_set { |pin| [pin.kind, pin.key] }
       items = all_touches(conversation, cwd, view).group_by { |touch| [touch.kind, touch.key] }.map do |(kind, key), group|
-        item(kind, key, group, conversation, view, stat)
+        item(kind, key, group, conversation, view, stat, pinned.include?([kind, key]))
+      end
+      Snapshot.new(items: items, at: conversation.size, adds: pending_adds(conversation))
+    end
+
+    # The index of the note the next turn will send (one after the last
+    # turn's entries: only system messages follow it), or nil.
+    def pending_note_index(messages)
+      (messages.size - 1).downto(0) do |index|
+        message = messages[index]
+        return nil unless message.is_a?(Hash) && message[:role].to_s == "system"
+        return index if message[:kind].to_s == NOTE_KIND
+      end
+      nil
+    end
+
+    # The ops the pending note lists ([{"op", "kind", "key", "label"}]).
+    def pending_ops(messages)
+      messages = Array(messages)
+      index = pending_note_index(messages)
+      index ? Array(messages[index][NOTE_OPS]).select { |op| op.is_a?(Hash) } : []
+    end
+
+    # The items the user added since the last turn, which the next turn's
+    # note asks the model to read.
+    # @return [Array<Add>]
+    def pending_adds(messages)
+      pending_ops(messages).select { |op| op["op"] == ADD }.map do |op|
+        Add.new(kind: op["kind"].to_s, key: op["key"].to_s, label: op["label"].to_s)
       end
-      Snapshot.new(items: items, at: conversation.size)
     end
 
     # One content output (read, memory_read, web_fetch) of an item: its
     # chi +id+ (ToolIds), the +index+ of the tool_response entry holding
     # it, whether the id is +derived+ (a legacy entry's: nothing can name
     # it) and whether the model is still +sent+ it under the strategy.
-    Output = Data.define(:id, :index, :derived, :sent)
+    # +shared_with+: the other names a memory_read of several names holds in the
+    # same output (an op on the output acts on them too), else empty.
+    Output = Data.define(:id, :index, :derived, :sent, :shared_with)
 
     # The content outputs of the item +kind+/+key+ (#build's identity: a
     # file's absolute path, a memory's name, a URL), in conversation
@@ -113,10 +171,23 @@ module Samagotchi
 
         ref = touch.run.ref
         Output.new(id: ref.id, index: touch.run.index, derived: ref.derived?,
-                   sent: still_sent?(touch, conversation, view))
+                   sent: still_sent?(touch, conversation, view), shared_with: shared_with(touch, key))
       end
     end
 
+    # The ids of the content outputs of the items +pins+ name (the ones
+    # the model's forget_outputs refuses), by the same pairing as #build.
+    # @param pins [Array<WorkingSetPin>]
+    # @return [Set<String>]
+    def pinned_ids(conversation, pins:, cwd:)
+      return Set.new if pins.empty?
+
+      conversation = Array(conversation)
+      pinned = pins.to_set { |pin| [pin.kind, pin.key] }
+      touches = all_touches(conversation, cwd, LLMContextView.new(strategy: LLMContextView::NONE))
+      touches.select { |touch| touch.content && pinned.include?([touch.kind, touch.key]) }.to_set { |touch| touch.run.ref.id }
+    end
+
     # Every paired run's touches, with what +view+ sends of its output.
     def all_touches(conversation, cwd, view)
       stored = LLMContextStale.paired_runs(conversation)
@@ -159,6 +230,9 @@ module Samagotchi
       ToolResponse.interrupted_text?(text) || text.sub(ToolResponse::LEAD, "").lstrip.start_with?("Error")
     end
 
+    # The other names +touch+'s output holds (a memory_read of several).
+    def shared_with(touch, key) = touch.share > 1 ? found_names(touch.run) - [key] : []
+
     # A memory_read's names (Tools::MemoryRead.parse_names: none for the
     # index listing), less the ones its output says weren't found.
     def found_names(run)
@@ -168,7 +242,7 @@ module Samagotchi
       names - missing
     end
 
-    def item(kind, key, group, conversation, view, stat)
+    def item(kind, key, group, conversation, view, stat, pinned)
       first = group.first
       contents = group.select(&:content)
       tokens = contents.sum { |touch| touch.sent.length / TokenUsage::CHARS_PER_TOKEN / touch.share }.ceil
@@ -180,7 +254,7 @@ module Samagotchi
                 LIVE
               end
       Item.new(kind: kind, key: key, state: state, tokens: tokens, ops: group.map(&:op).uniq, count: group.size,
-               first_id: first.run.ref.id)
+               first_id: first.run.ref.id, pinned: pinned, has_drop: contents.any? { |touch| dropped?(touch, conversation) })
     end
 
     # Whether the model is still sent +touch+'s output: whole (with the
@@ -233,6 +307,6 @@ module Samagotchi
 
     def field(hash, key) = hash.key?(key) ? hash[key] : hash[key.to_s]
 
-    private_class_method :all_touches, :touches, :failed?, :found_names, :item, :still_sent?, :dropped?, :with_id, :changed?, :stamp_of, :field
+    private_class_method :all_touches, :touches, :failed?, :found_names, :shared_with, :item, :still_sent?, :dropped?, :with_id, :changed?, :stamp_of, :field
   end
 end

data/lib/samagotchi/working_set_command.rb

@@ -2,22 +2,35 @@
 
 require_relative "working_set"
 require_relative "working_set_steer"
+require_relative "tools/memory"
 
 module Samagotchi
   # /working-set: with no arguments the session's working set (WorkingSet,
   # as the web's list shows it: each item, what was done to it, its state
-  # and tokens, then the changes the next turn starts with); `drop KEY…`
-  # and `undrop KEY…` steer it (WorkingSetSteer). KEY: a path (absolute or
-  # the session's folder's), memory:NAME, a URL.
+  # and tokens, then the changes the next turn starts with); `drop KEY…`,
+  # `undrop KEY…`, `pin KEY…`, `unpin KEY…`, `add KEY…` and `unadd KEY…`
+  # steer it (WorkingSetSteer). An add asks the model, in the next turn's
+  # note, to read the item (a file that exists, a memory, an http(s) URL).
+  # KEY: a path (absolute or the session's folder's), memory:NAME, a URL;
+  # one with spaces quoted ("my dir/a.rb", WorkingSetSteer.words), as the
+  # listing shows it.
+  # A pin is saved on the session (Session#working_set_pins); the turn-start
+  # note names it only where the model has forget_outputs (the strategy's
+  # forget layer), or to cancel a pin or an unpin it named.
   #
   # An op runs between turns (mid_turn :queue: a line sent while a turn
   # runs waits for its end), on a copy of the session's messages that
   # replaces them, so the next turn's first request is the first with it.
   class WorkingSetCommand
     NAME = "/working-set"
-    USAGE = "usage: #{NAME} [drop KEY… | undrop KEY…] (KEY: a path, memory:NAME or a URL)".freeze
-    VERBS = [WorkingSetSteer::DROP, WorkingSetSteer::UNDROP].freeze
-    OP_WORDS = { WorkingSetSteer::DROP => "dropped", WorkingSetSteer::UNDROP => "undropped" }.freeze
+    USAGE = "usage: #{NAME} [drop KEY… | undrop KEY… | pin KEY… | unpin KEY… | add KEY… | unadd KEY…] " \
+            "(KEY: a path, memory:NAME or a URL; \"quote\" one with spaces)".freeze
+    VERBS = [WorkingSetSteer::DROP, WorkingSetSteer::UNDROP, WorkingSetSteer::PIN, WorkingSetSteer::UNPIN,
+             WorkingSetSteer::ADD, WorkingSetSteer::UNADD].freeze
+    OP_WORDS = { WorkingSetSteer::DROP => "dropped", WorkingSetSteer::UNDROP => "undropped",
+                 WorkingSetSteer::PIN => "pinned", WorkingSetSteer::UNPIN => "unpinned",
+                 WorkingSetSteer::ADD => "added", WorkingSetSteer::UNADD => "unadded" }.freeze
+    PIN_OPS = [WorkingSetSteer::PIN, WorkingSetSteer::UNPIN].freeze
 
     # @param engine [Engine] #session, #working_set_strategy,
     #   #clone_messages, #replace_messages
@@ -28,39 +41,39 @@ module Samagotchi
     end
 
     # @param args [String] what follows /working-set
-    # @return [Array(String, Boolean, Symbol)] the reply, whether the
-    #   conversation changed, and :ok or :error (every key refused)
+    # @return [Array(String, Array<Symbol>, Symbol)] the reply, what
+    #   changed (:messages, :working_set_pins; none when nothing did), and
+    #   :ok or :error (every key refused)
     # @raise [ArgumentError] a line that isn't one (its message and USAGE)
     def run(args)
-      verb, *words = args.to_s.split
-      return [listing, false, :ok] if verb.nil?
+      verb, *words = WorkingSetSteer.words(args)
+      return [listing, [], :ok] if verb.nil?
       raise ArgumentError, USAGE unless VERBS.include?(verb) && !words.empty?
 
       session = @engine.session or raise ArgumentError, "no session yet"
       cwd = session.working_directory
       strategy = @engine.working_set_strategy
       messages = @engine.clone_messages(session.messages)
+      pins = session.working_set_pins
       lines = []
       done = 0
+      noted = false
       words.each do |word|
         target = WorkingSetSteer.target(word, cwd: cwd)
-        result = if verb == WorkingSetSteer::DROP
-                   WorkingSetSteer.drop(messages, target, strategy: strategy, cwd: cwd)
-                 else
-                   sent = !WorkingSetSteer.pending?(messages, WorkingSetSteer::DROP, target)
-                   WorkingSetSteer.undrop(messages, target, strategy: strategy, cwd: cwd, sent: sent)
-                 end
-        messages = WorkingSetSteer.record!(messages, result)
-        lines << done_line(result)
+        result = op(verb, messages, pins, target, strategy, cwd)
+        [result.undone, result].compact.select { |each| noted?(each, messages, strategy) }.each do |each|
+          messages = WorkingSetSteer.record!(messages, each)
+          noted = true
+        end
+        pins = WorkingSetSteer.pinning(pins, result) if PIN_OPS.include?(result.op)
+        lines << done_line(result.undone) if result.undone
+        lines << done_line(result, strategy)
         done += 1
       rescue WorkingSetSteer::Refused => e
-        lines << "#{verb} #{word}: #{e.message}"
+        lines << "#{verb} #{WorkingSetSteer.quote(word)}: #{e.message}"
       end
-      if done.positive?
-        @engine.replace_messages(messages)
-        @save.call(session)
-      end
-      [lines.join("\n"), done.positive?, done.positive? ? :ok : :error]
+      changed = done.positive? ? save(session, noted && messages, pins) : []
+      [lines.join("\n"), changed, done.positive? ? :ok : :error]
     end
 
     # The set under the strategy the next turn runs, and the pending ops.
@@ -69,10 +82,11 @@ module Samagotchi
       return "no session yet" unless session
 
       messages = Array(session.messages)
-      items = WorkingSet.build(messages, cwd: session.working_directory, strategy: @engine.working_set_strategy).items
+      items = WorkingSet.build(messages, cwd: session.working_directory, strategy: @engine.working_set_strategy,
+                               pins: session.working_set_pins).items
       return "working set: empty (the files, memories and URLs the turns read show here)\n#{USAGE}" if items.empty?
 
-      rows = items.map { |item| [name(item, session.working_directory), words(item)] }
+      rows = items.map { |item| [WorkingSetSteer.quote(name(item, session.working_directory)), words(item)] }
       width = rows.map { |row, _| row.length }.max
       lines = ["working set (#{items.size} item#{"s" unless items.size == 1}):"]
       lines.concat(rows.map { |row, text| "  #{row.ljust(width)}  #{text}" })
@@ -86,6 +100,65 @@ module Samagotchi
 
     private
 
+    def op(verb, messages, pins, target, strategy, cwd)
+      case verb
+      when WorkingSetSteer::DROP then WorkingSetSteer.drop(messages, target, strategy: strategy, cwd: cwd, pins: pins)
+      when WorkingSetSteer::UNDROP
+        sent = !WorkingSetSteer.pending?(messages, WorkingSetSteer::DROP, target)
+        WorkingSetSteer.undrop(messages, target, strategy: strategy, cwd: cwd, sent: sent)
+      when WorkingSetSteer::PIN then WorkingSetSteer.pin(messages, pins, target, strategy: strategy, cwd: cwd)
+      when WorkingSetSteer::UNPIN then WorkingSetSteer.unpin(pins, target)
+      when WorkingSetSteer::ADD
+        WorkingSetSteer.add(messages, target, strategy: strategy, cwd: cwd, found: method(:missing))
+      else WorkingSetSteer.unadd(messages, target)
+      end
+    end
+
+    # Why an added +target+ can't be read (a Refused reason), nil when it
+    # can: a file that is there, a memory the project or system scope
+    # holds (memory_read's look). A URL is fetched as it is.
+    def missing(target)
+      case target.kind
+      when WorkingSet::FILE
+        return :directory if File.directory?(target.key)
+
+        File.file?(target.key) ? nil : :no_file
+      when WorkingSet::MEMORY
+        return :bad_memory if target.key.empty? || Tools::MemoryRead.invalid_name?(target.key)
+
+        found = %w[project system].any? { |scope| File.file?(Tools::MemoryRead.memory_path(target.key, scope)) }
+        found ? nil : :no_memory
+      end
+    end
+
+    # Whether the turn-start note lists +done+: a drop or an undrop always;
+    # a pin or an unpin where the model has forget_outputs, or when it
+    # cancels one the note lists.
+    def noted?(done, messages, strategy)
+      return true unless PIN_OPS.include?(done.op)
+
+      forget?(strategy) || WorkingSetSteer.pending?(messages, WorkingSetSteer::INVERSE.fetch(done.op), done.target)
+    end
+
+    def forget?(strategy) = Array(strategy).include?(:forget)
+
+    # Puts the ops' messages (nil: unchanged) and pins in place and saves
+    # the session.
+    # @return [Array<Symbol>] what changed
+    def save(session, messages, pins)
+      changed = []
+      if messages
+        @engine.replace_messages(messages)
+        changed << :messages
+      end
+      unless pins == session.working_set_pins
+        session.working_set_pins = pins
+        changed << :working_set_pins
+      end
+      @save.call(session) unless changed.empty?
+      changed
+    end
+
     def name(item, cwd)
       case item.kind
       when WorkingSet::FILE then WorkingSetSteer.target(item.key, cwd: cwd).label
@@ -98,17 +171,47 @@ module Samagotchi
       parts = [item.ops.join(", ")]
       parts << "×#{item.count}" if item.count > 1
       parts << item.state unless item.state == WorkingSet::LIVE
+      # A drop on some of its reads (the rest forgotten or still sent): undrop takes it back.
+      parts << "part dropped" if item.has_drop && item.state != WorkingSet::DROPPED
+      parts << "pinned" if item.pinned
       parts << "~#{item.tokens} tokens" if item.tokens.positive?
       parts.join(" · ")
     end
 
-    def done_line(done)
-      what = "#{OP_WORDS[done.op]} #{done.target.label} (#{done.ids.join(", ")})"
-      if done.op == WorkingSetSteer::DROP
+    def done_line(done, strategy = nil)
+      ids = done.ids.empty? ? "" : " (#{done.ids.join(", ")})"
+      "#{op_line(done, "#{OP_WORDS[done.op]} #{done.target.label}#{ids}", strategy)}#{with_line(done)}"
+    end
+
+    # A memory_read of several names is one output: the op acted on the
+    # others' read too. Says which names went with it.
+    def with_line(done)
+      return "" if done.shared_with.empty?
+
+      names = done.shared_with.map { |name| "memory #{name}" }.join(", ")
+      "; the same memory_read holds #{names}: #{OP_WORDS[done.op]} with it (that read only)"
+    end
+
+    def op_line(done, what, strategy)
+      case done.op
+      when WorkingSetSteer::PIN
+        off = forget?(strategy) ? "" : " (the strategy has no forget layer now: the pin holds when it has)"
+        "#{what}: forget_outputs refuses its outputs, its later reads too#{off}"
+      when WorkingSetSteer::UNPIN then "#{what}: the model may forget it again"
+      when WorkingSetSteer::ADD
+        "#{what}: the next turn's note asks the model to read it first (one tool call)"
+      when WorkingSetSteer::UNADD then "#{what}: its add was never sent; nothing changes"
+      when WorkingSetSteer::DROP
         frees = done.freed_chars.positive? ? "frees #{WorkingSetSteer.k(done.freed_chars)}" : "frees nothing, the stubs are as long"
         "#{what}: the next turn's first request sends the stubs (#{frees}; the server reads the " \
           "#{WorkingSetSteer.k(done.tail_chars)} after them again)"
-      elsif done.free
+      else
+        undrop_line(what, done)
+      end
+    end
+
+    def undrop_line(what, done)
+      if done.free
         "#{what}: its drop was never sent; nothing changes"
       else
         "#{what}: the next request sends it whole again (the server reads the #{WorkingSetSteer.k(done.tail_chars)} " \

data/lib/samagotchi/working_set_pin.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Samagotchi
+  # One item of the working set the user pinned (/working-set pin): its
+  # identity as WorkingSet names an item, +kind+ file|memory|url and +key+
+  # (a file's absolute path, a memory's name, a URL). The model's
+  # forget_outputs refuses its outputs, the ones it has and the ones it
+  # reads later (WorkingSetSteer). Saved as the session's
+  # working_set_pins ([{"kind", "key"}]); a respawn, a continue and a fork
+  # keep them (SessionSetup).
+  WorkingSetPin = Data.define(:kind, :key) do
+    def initialize(kind:, key:)
+      super(kind: kind.to_s, key: key.to_s)
+    end
+
+    # The session file's form.
+    def to_file = { "kind" => kind, "key" => key }
+
+    # +value+'s pins, in order, no repeats: pins, or a file's Hashes
+    # (string or symbol keys); anything else (a hand-edited entry without
+    # a kind or key) is left out.
+    # @return [Array<WorkingSetPin>] frozen
+    def self.list(value)
+      Array(value).filter_map do |entry|
+        next entry if entry.is_a?(WorkingSetPin)
+        next unless entry.is_a?(Hash)
+
+        kind = entry["kind"] || entry[:kind]
+        key = entry["key"] || entry[:key]
+        new(kind: kind, key: key) unless kind.to_s.empty? || key.to_s.empty?
+      end.uniq.freeze
+    end
+  end
+end