samagotchi 0.31.0 → 0.33.0

data/lib/samagotchi/context_fetch.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "context_sources"
+require_relative "context_providers"
+require_relative "log"
+require_relative "process_group"
+
+module Samagotchi
+  # Runs an attached context source's command once and records the result
+  # in its snapshot (ContextSources): the worker's ContextPoller and
+  # `chi context refresh` both come here.
+  #
+  # The command runs with `sh -c` in its own process group (a timeout, the
+  # output cap or the caller's cancel kills the group: no orphans), in
+  # +cwd+, with stdin closed, SAMAGOTCHI_CONTEXT_NAME and (once there is
+  # one) SAMAGOTCHI_CONTEXT_PREVIOUS, the last snapshot's path. stdout is
+  # the text (ContextSources.parse_output); stderr goes to the debug log,
+  # its last line into the error. A non-blocking flock on <name>.lock keeps
+  # two workers (or a worker and the CLI) from running one source at once.
+  module ContextFetch
+    TIMEOUT_SECONDS = 60
+    STDERR_TAIL_BYTES = 4096
+    READ_CHUNK = 64 * 1024
+    # How long a group gets after TERM before KILL.
+    KILL_GRACE_SECONDS = 1.0
+    OVER_CAP = "it printed more than 1 MiB"
+
+    # +status+: :new (a new revision), :same (the same text), :error (the
+    # fetch failed: +error+), :busy (another process holds the lock),
+    # :fresh (tried within +fresh_within+, by someone else meanwhile),
+    # :cancelled (the caller stopped it: nothing recorded, since a worker
+    # leaving isn't the source failing), :gone (the source was removed or
+    # replaced while it ran: nothing recorded for it).
+    Outcome = Data.define(:status, :snapshot, :error)
+    # What the command did: +output+ its stdout, +error+ nil or why it
+    # failed, +cancelled+ the caller stopped it.
+    Run = Data.define(:output, :error, :stderr, :cancelled)
+
+    module_function
+
+    # @param attached [ContextSources::Attached] a command source
+    # @param fresh_within [Numeric, nil] skip it when it was tried this
+    #   many seconds ago (checked under the lock)
+    # @param cancelled [#call] true: kill the command (the worker stops)
+    # @return [Outcome]
+    def fetch(attached, cwd:, fresh_within: nil, timeout: TIMEOUT_SECONDS, cancelled: -> { false })
+      location = attached.location
+      name = attached.name
+      File.open(location.lock_path(name), File::RDWR | File::CREAT, 0o644) do |lock|
+        return Outcome.new(status: :busy, snapshot: nil, error: nil) unless lock.flock(File::LOCK_EX | File::LOCK_NB)
+
+        before = location.snapshot(name)
+        age = before.age
+        return Outcome.new(status: :fresh, snapshot: before, error: nil) if fresh_within && age && age < fresh_within
+
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        run = begin
+          run_command(ContextProviders.command_for(attached.source), cwd: cwd, env: env_for(location, name),
+                                                                     timeout: timeout, cancelled: cancelled)
+        rescue ContextProviders::Invalid => e
+          Run.new(output: nil, error: e.message, stderr: "", cancelled: false)
+        end
+        outcome = if run.cancelled
+                    Outcome.new(status: :cancelled, snapshot: before, error: nil)
+                  elsif !current?(attached)
+                    Outcome.new(status: :gone, snapshot: nil, error: nil)
+                  else
+                    record_current(attached, run, before)
+                  end
+        Log.info(:context, "fetched", name: name, scope: location.scope, status: outcome.status,
+                                      ms: ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round,
+                                      error: outcome.error)
+        outcome
+      end
+    end
+
+    # Whether the source is still the one this fetch ran: `chi context rm`
+    # (or rm and add with another command) doesn't take the lock.
+    def current?(attached) = attached.location.source(attached.name) == attached.source
+
+    # #record, then the check again: a removal between the first check and
+    # the write leaves no snapshot behind (the lock is still ours, so the
+    # file is ours too).
+    def record_current(attached, run, before)
+      outcome = record(attached.location, attached.name, run, before)
+      return outcome if current?(attached)
+
+      FileUtils.rm_f(attached.location.snapshot_path(attached.name))
+      Outcome.new(status: :gone, snapshot: nil, error: nil)
+    end
+
+    def record(location, name, run, before)
+      if run.error
+        snapshot = location.record_error(name, run.error)
+        return Outcome.new(status: :error, snapshot: snapshot, error: snapshot.error)
+      end
+
+      begin
+        fetched = ContextSources.parse_output(run.output)
+      rescue ContextSources::Invalid => e
+        snapshot = location.record_error(name, e.message)
+        return Outcome.new(status: :error, snapshot: snapshot, error: snapshot.error)
+      end
+      snapshot = location.record_text(name, fetched)
+      Outcome.new(status: snapshot.revision == before.revision ? :same : :new, snapshot: snapshot, error: nil)
+    end
+
+    def env_for(location, name)
+      previous = location.snapshot_path(name)
+      { "SAMAGOTCHI_CONTEXT_NAME" => name,
+        "SAMAGOTCHI_CONTEXT_PREVIOUS" => File.exist?(previous) ? previous : nil }
+    end
+
+    # @return [Run]
+    def run_command(cmd, cwd:, env:, timeout:, cancelled:)
+      return Run.new(output: nil, error: "its folder #{cwd} is gone", stderr: "", cancelled: false) unless cwd && Dir.exist?(cwd)
+
+      out_r, out_w = IO.pipe
+      err_r, err_w = IO.pipe
+      pid = ProcessGroup.spawn(env, "sh", "-c", cmd, chdir: cwd, in: File::NULL, out: out_w, err: err_w)
+      out_w.close
+      err_w.close
+      out_reader = ProcessGroup::PipeReader.new(out_r, cap: ContextSources::TEXT_MAX_BYTES, chunk: READ_CHUNK)
+      err_reader = ProcessGroup::PipeReader.new(err_r, cap: STDERR_TAIL_BYTES, keep: :tail, chunk: READ_CHUNK)
+
+      error, status = wait(pid, timeout: timeout, over: -> { out_reader.over? }, cancelled: cancelled)
+      reaped = true
+      # Whatever the command left running in its group goes with it.
+      ProcessGroup.signal(pid, "KILL")
+      [out_reader, err_reader].each { |reader| reader.wait(KILL_GRACE_SECONDS) || reader.kill }
+      stderr_text = err_reader.text(Encoding::UTF_8).scrub
+      Log.debug(:context, "stderr", text: stderr_text[-1000..] || stderr_text) unless stderr_text.strip.empty?
+      return Run.new(output: nil, error: nil, stderr: stderr_text, cancelled: true) if error == :cancelled
+
+      # Checked again once the reader is done: a command that exits before
+      # wait's next tick never gets an over from it.
+      error = OVER_CAP if out_reader.over?
+      error ||= exit_error(status, stderr_text)
+      Run.new(output: out_reader.text, error: error, stderr: stderr_text, cancelled: false)
+    rescue SystemCallError => e
+      Run.new(output: nil, error: "couldn't run it: #{e.message}", stderr: "", cancelled: false)
+    ensure
+      # Cut short (Ctrl-C in chi context refresh, an exception, a killed
+      # poller thread): the group doesn't get the terminal's SIGINT
+      # (pgroup), so stop and reap it here.
+      abandon(pid, [out_reader, err_reader]) if pid && !reaped
+      [out_r, out_w, err_r, err_w].each { |io| io&.close unless io&.closed? }
+    end
+
+    # @return [Array(String|Symbol|nil, Process::Status|nil)] why it was
+    #   stopped (:cancelled for the caller), and its status
+    def wait(pid, timeout:, over:, cancelled:)
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+      loop do
+        _, status = Process.wait2(pid, Process::WNOHANG)
+        return [nil, status] if status
+
+        reason = if over.call then OVER_CAP
+                 elsif cancelled.call then :cancelled
+                 elsif Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline then "timed out after #{timeout.round} s"
+                 end
+        return [reason, stop_group(pid)] if reason
+
+        sleep(0.05)
+      end
+    end
+
+    def abandon(pid, readers)
+      stop_group(pid)
+      readers.each { |reader| reader&.kill }
+    rescue StandardError => e
+      Log.warn(:context, "stop_failed", pid: pid, error: e.class.name)
+    end
+
+    # TERM, then KILL after a grace; @return [Process::Status, nil]
+    def stop_group(pid)
+      status = nil
+      reaped = -> { status = Process.wait2(pid, Process::WNOHANG)&.last }
+      return status if ProcessGroup.stop(pid, grace: KILL_GRACE_SECONDS, poll: 0.05, stopped: reaped)
+
+      Process.wait2(pid).last
+    rescue Errno::ECHILD
+      nil
+    end
+
+    def exit_error(status, stderr)
+      return nil if status.nil? || status.success?
+
+      last = stderr.lines.map(&:strip).reject(&:empty?).last
+      code = status.exitstatus ? "exit #{status.exitstatus}" : "killed by signal #{status.termsig}"
+      last ? "#{code}: #{last}" : code
+    end
+  end
+end

data/lib/samagotchi/context_note.rb

@@ -11,16 +11,22 @@ module Samagotchi
     KIND = "note"
     # The message keys a note carries besides role and content; everything
     # that copies messages field by field must keep them.
-    KEYS = %i[kind note_id source from_session from_cwd].freeze
+    # context_source: the attached context source a note is about
+    # (ContextAbsorber), so the UIs can tie it to its chip.
+    KEYS = %i[kind note_id source from_session from_cwd context_source].freeze
 
     module_function
 
-    # Takes SessionInbox.read_note's keys.
+    # Takes SessionInbox.read_note's keys. +turn_start+ / +turn_id+: the
+    # note starts a turn chi runs for it (a context wake), as a wake turn's
+    # delegate report does (Steer.turn_prompt?).
     # @return [Hash] the conversation message
-    def message(note_id:, text:, source:, created_at: nil, from_session: nil, from_cwd: nil, **)
+    def message(note_id:, text:, source:, created_at: nil, from_session: nil, from_cwd: nil, context_source: nil,
+                turn_start: nil, turn_id: nil, **)
       { role: "system", kind: KIND, note_id: note_id, source: source, from_session: from_session,
-        from_cwd: from_cwd, content: frame(text, source: source, created_at: created_at,
-                                                 from_session: from_session, from_cwd: from_cwd) }.compact
+        from_cwd: from_cwd, context_source: context_source, turn_start: (true if turn_start == true), turn_id: turn_id,
+        content: frame(text, source: source, created_at: created_at, from_session: from_session,
+                             from_cwd: from_cwd) }.compact
     end
 
     def frame(text, source:, created_at: nil, from_session: nil, from_cwd: nil)

data/lib/samagotchi/context_poller.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative "config"
+require_relative "context_fetch"
+require_relative "context_sources"
+require_relative "log"
+
+module Samagotchi
+  # A session worker's thread that keeps its attached context fresh: every
+  # TICK_SECONDS it runs each command source (the session's and its
+  # project's, not muted here) whose last try is older than its interval
+  # (--every, else context.every_seconds), and calls +on_change+ after a
+  # new text or a failure so the loop absorbs it at once. The first pass,
+  # right after the worker starts, runs every source not tried in the last
+  # MIN_EVERY_SECONDS: a woken worker's first absorb says what changed while
+  # it was away.
+  #
+  # Polling isn't activity: it never holds the worker up (WorkerIdleExit),
+  # and it stops with it (#stop kills a running command's group). Project
+  # sources are shared by the project's workers; ContextFetch's lock and a
+  # second freshness check under it make one fetch per interval.
+  class ContextPoller
+    TICK_SECONDS = 5
+
+    # @param project_root [String, nil] Session#project_root
+    # @param cwd [String] the session's folder (a session source runs there)
+    # @param on_change [#call] after a fetch brought a new text or failed
+    def initialize(session_id:, state_dir:, project_root:, cwd:, on_change:, tick: TICK_SECONDS)
+      @session_id = session_id
+      @state_dir = state_dir
+      @project_root = project_root
+      @cwd = cwd
+      @on_change = on_change
+      @tick = tick
+      @own = ContextSources.session_location(session_id, state_dir: state_dir)
+      @queue = Thread::Queue.new
+      @stopping = false
+    end
+
+    def start
+      @thread ||= Thread.new do
+        Thread.current.name = "context-poller"
+        run
+      end
+      self
+    end
+
+    # Stops the loop and kills a command it is running; returns once it has.
+    def stop
+      @stopping = true
+      @queue << :stop
+      @thread&.join((ContextFetch::KILL_GRACE_SECONDS * 3) + @tick)
+      @thread = nil
+    end
+
+    # One pass (the thread's loop body; specs call it).
+    # @param first [Boolean] the worker's first pass
+    def poll(first: false)
+      ContextSources.attached(@session_id, project_root: @project_root, state_dir: @state_dir).each do |attached|
+        break if @stopping
+        next if attached.source.push? || @own.muted?(attached.name)
+
+        interval = first ? ContextSources::MIN_EVERY_SECONDS : interval_of(attached.source)
+        age = attached.snapshot.age
+        next if age && age < interval
+
+        outcome = ContextFetch.fetch(attached, cwd: cwd_for(attached), fresh_within: interval, cancelled: -> { @stopping })
+        @on_change.call if %i[new error].include?(outcome.status)
+      rescue StandardError => e
+        Log.exception(:context, "poll_failed", e, name: attached.name)
+      end
+    end
+
+    private
+
+    def run
+      first = true
+      until @stopping
+        poll(first: first)
+        first = false
+        @queue.pop(timeout: @tick)
+      end
+    rescue StandardError => e
+      Log.exception(:context, "poller_crashed", e)
+    end
+
+    def interval_of(source)
+      seconds = source.every_seconds || Config.get("context.every_seconds").to_i
+      [seconds, ContextSources::MIN_EVERY_SECONDS].max
+    end
+
+    # A session's source runs in the session's folder, a project's in the
+    # project root (the session's folder when the root is no checkout).
+    def cwd_for(attached)
+      return @cwd if attached.location.session?
+
+      ContextSources.project_cwd(@project_root, @cwd)
+    end
+  end
+end

data/lib/samagotchi/context_providers.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require "rbconfig"
+require "shellwords"
+
+module Samagotchi
+  autoload :ContextSources, File.expand_path("context_sources", __dir__)
+  autoload :Log, File.expand_path("log", __dir__)
+  module MemoryBundle
+    autoload :Provenance, File.expand_path("memory_bundle/provenance", __dir__)
+  end
+
+  # Declarative context providers (attached context, §3.9): a bundle's
+  # manifest maps a URL pattern to a source, so any process (the CLI, the
+  # web server, a worker) can attach a URL without loading plugins:
+  #
+  #   context_providers:
+  #     - match: '\Ahttps://github\.com/([^/]+)/([^/]+)/pull/(\d+)'
+  #       name: 'pr-\3'
+  #       cmd: '{ruby} {bundle_dir}/scripts/pr_context.rb {url}'
+  #       why: 'GitHub PR'
+  #       every_seconds: 300
+  #
+  # The installer records them in the bundle's provenance; .resolve reads
+  # them from there. +name+ takes the match's groups (\1…); +cmd+ takes
+  # {url}, the matched URL (shell-quoted), when the source is attached, and
+  # keeps {bundle_dir}, the installed bundle's folder, and {ruby}, the Ruby
+  # chi runs on, for ContextFetch to fill in when it runs: a bundle upgrade
+  # moves nothing, and a `ruby` on the PATH may be another one (macOS's 2.6,
+  # a launchd PATH).
+  module ContextProviders
+    # A provider as the manifest declares it.
+    Provider = Data.define(:match, :name, :cmd, :why, :every_seconds) do
+      def to_h = { "match" => match, "name" => name, "cmd" => cmd, "why" => why, "every_seconds" => every_seconds }.compact
+
+      def regexp = Regexp.new(match)
+    end
+
+    # What a URL resolved to: the provider's bundle, and the source's
+    # fields (+cmd+ still holding {bundle_dir}).
+    Resolved = Data.define(:bundle, :name, :cmd, :why, :hint, :every_seconds)
+
+    class Invalid < ArgumentError; end
+
+    BUNDLE_DIR = "{bundle_dir}"
+    RUBY = "{ruby}"
+    URL = "{url}"
+    MIN_EVERY_SECONDS = 30
+
+    module_function
+
+    # The manifest's context_providers: list.
+    # @return [Array<Provider>] [] when absent
+    # @raise [Invalid]
+    def parse_list(raw)
+      return [] if raw.nil?
+      raise Invalid, "context_providers: must be a list of mappings" unless raw.is_a?(Array)
+
+      raw.map { |item| parse(item) }
+    end
+
+    # @raise [Invalid]
+    def parse(item)
+      raise Invalid, "context_providers: each item must be a mapping, not #{item.inspect}" unless item.is_a?(Hash)
+
+      data = item.transform_keys(&:to_s)
+      %w[match name cmd].each do |key|
+        value = data[key]
+        raise Invalid, "context_providers: an item has no #{key}:" unless value.is_a?(String) && !value.strip.empty?
+      end
+      begin
+        Regexp.new(data["match"])
+      rescue RegexpError => e
+        raise Invalid, "context_providers: match #{data["match"].inspect} isn't a regular expression (#{e.message})"
+      end
+      Provider.new(match: data["match"], name: data["name"], cmd: data["cmd"], why: data["why"]&.to_s,
+                   every_seconds: every(data["every_seconds"]))
+    end
+
+    # The source the installed bundles' providers make of +url+: the first
+    # bundle (by name) whose provider matches. A bundle whose providers
+    # don't parse is skipped (logged).
+    # @return [Resolved, nil] nil when none matches
+    # @raise [Invalid] the provider makes a name that isn't a source name
+    def resolve(url)
+      url = url.to_s.strip
+      MemoryBundle::Provenance.each_installed do |bundle, data|
+        next if data[:error] || !data[:context_providers].is_a?(Array)
+
+        providers(bundle, data).each do |provider|
+          match = provider.regexp.match(url) or next
+          return resolved(bundle, provider, match)
+        end
+      end
+      nil
+    end
+
+    # Whether an installed bundle declares a provider (the web's "+ URL").
+    def any?
+      MemoryBundle::Provenance.each_installed.any? do |_bundle, data|
+        !data[:error] && data[:context_providers].is_a?(Array) && !data[:context_providers].empty?
+      end
+    end
+
+    def providers(bundle, data)
+      parse_list(data[:context_providers])
+    rescue Invalid => e
+      Log.warn(:context, "providers_unreadable", bundle: bundle, msg: e.message)
+      []
+    end
+
+    def resolved(bundle, provider, match)
+      url = match[0]
+      name = provider.name.gsub(/\\(\d)/) { match[Regexp.last_match(1).to_i].to_s }
+      begin
+        ContextSources.check_name!(name)
+      rescue ContextSources::Invalid
+        raise Invalid, "bundle #{bundle} makes #{name.inspect} of it, which isn't a source name"
+      end
+      Resolved.new(bundle: bundle, name: name, cmd: provider.cmd.gsub(URL) { Shellwords.escape(url) },
+                   why: provider.why, hint: url, every_seconds: provider.every_seconds)
+    end
+
+    # +source+'s command as it runs: {bundle_dir} is its provider's
+    # installed folder, once that bundle is there and its scripts are as
+    # installed (they run outside the guardrails gate, as a plugin loads:
+    # checked the same way).
+    # @return [String]
+    # @raise [Invalid] the bundle is gone or a script changed
+    def command_for(source)
+      return source.cmd unless source.provider
+
+      provenance = MemoryBundle::Provenance.new(name: source.provider)
+      data = provenance.read
+      raise Invalid, "bundle #{source.provider} isn't installed" unless data
+
+      scripts = data[:scripts].is_a?(Hash) ? data[:scripts] : {}
+      scripts.each do |file, meta|
+        path = File.join(provenance.scripts_dir, file.to_s)
+        recorded = MemoryBundle::Provenance.recorded_sha(meta.is_a?(Hash) ? meta[:sha256] : nil)
+        next if File.file?(path) && MemoryBundle::Provenance.file_sha(path) == recorded
+
+        raise Invalid, "bundle #{source.provider}'s scripts/#{file} differs from the installed one (reinstall the bundle)"
+      end
+      source.cmd.gsub(BUNDLE_DIR) { Shellwords.escape(provenance.bundle_dir) }.gsub(RUBY) { Shellwords.escape(RbConfig.ruby) }
+    end
+
+    def every(value)
+      return nil if value.nil?
+
+      seconds = Integer(value.to_s, 10, exception: false)
+      unless seconds && seconds >= MIN_EVERY_SECONDS
+        raise Invalid, "context_providers: every_seconds is #{value.inspect}; it takes seconds, at least #{MIN_EVERY_SECONDS}"
+      end
+
+      seconds
+    end
+  end
+end