pikuri-os 0.1.0

data/lib/pikuri/os/local_search.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Pikuri
+  module Os
+    # Backend wrapper over GNOME's +localsearch+ CLI (the desktop file
+    # index, 2024 rename of tracker3 / tracker-miners). Not a
+    # {Pikuri::Tool} — the shared, stateless seam behind
+    # +fileindex_search+ / +fileindex_read+, so localsearch-specific
+    # knowledge (argv, parsing, json-ld extraction, binary probe) lives in
+    # one place.
+    #
+    # +localsearch search+ returns at most *one* match snippet per file
+    # from a possibly-stale index. {.search} returns exactly those
+    # one-per-file hits and the tools tell the LLM to read the file for the
+    # rest — enumerating every per-file hit would need a stemmer (the index
+    # is stemmed, "mouse" matches "mice"), deliberately out of scope.
+    #
+    # PDFs and office docs come for free: the index extracted their text at
+    # index time, so {.read_text} returns it via +info -c+ with no
+    # extractor dependency here.
+    module LocalSearch
+      # @return [String] the localsearch CLI.
+      BINARY = 'localsearch'
+
+      # @return [String] short human name for this backend, used by the
+      #   fileindex tools when prefixing an +"Error: ..."+ observation.
+      def self.label = 'localsearch'
+
+      # Uniform accessor mirroring {Recoll.limitations}. A constant here
+      # (unlike recoll's runtime method) because this backend's blind spots
+      # are the same on every host.
+      #
+      # @return [String]
+      def self.limitations = LIMITATIONS
+
+      # Plain-English note of this backend's *structural* blind spots,
+      # appended to the fileindex tools' LLM description so the model knows
+      # when to fall back to a filesystem text/regex search. Lives on the
+      # backend (a different index ships its own). All three are confirmed
+      # against GNOME localsearch (measured; see +pikuri-os/DESIGN.md+).
+      #
+      # @return [String]
+      LIMITATIONS = <<~LIMITS.chomp
+        Not covered by this index (fall back to a text/regex search over the files for these):
+        - Source code and other developer files are not content-indexed — only their names are. Search their text directly.
+        - Nothing inside a git checkout is indexed (any folder containing a .git). Search those files directly.
+        - No folder scoping: it ranks across your whole home at once, so a common word confined to one project may stay buried below the results shown. Prefer distinctive words, or search that folder's text directly.
+      LIMITS
+
+      # Raised when a localsearch invocation exits non-zero — a
+      # recoverable failure the calling tool renders as +"Error: ..."+.
+      class CommandError < StandardError; end
+
+      # @return [Regexp] one ANSI SGR escape (color/bold), stripped from
+      #   snippets before they reach the LLM.
+      ANSI_SGR = /\e\[[0-9;]*m/
+
+      # Run +localsearch search -d --limit <limit> <query>+ and parse the
+      # results. +-d+ adds the (ignored) URN and, on the line after each
+      # path, a short snippet with the matched word highlighted; we strip
+      # the ANSI and keep the text.
+      #
+      # @param query [String]
+      # @param limit [Integer]
+      # @return [Array<Hash>] +[{ path: String, snippet: String|nil }, …]+,
+      #   de-duplicated by path
+      # @raise [CommandError] if localsearch exits non-zero
+      def self.search(query:, limit:)
+        result = Pikuri::Subprocess.spawn(BINARY, 'search', '-d', '--limit', limit.to_s, query,
+                                          chdir: '/').wait
+        unless result.status.success?
+          stderr = result.output.strip
+          stderr = "exited #{result.status.exitstatus}" if stderr.empty?
+          raise CommandError, stderr
+        end
+
+        parse_search_output(result.output)
+      end
+
+      # Parse +localsearch search -d+ output into hits. Each result is a
+      # +file://<uri> (urn:…)+ line followed by an indented snippet line;
+      # the path is percent-decoded and the snippet ANSI-stripped.
+      #
+      # @param raw [String]
+      # @return [Array<Hash>] +[{ path:, snippet: }, …]+ de-duped by path
+      def self.parse_search_output(raw)
+        hits = []
+        current = nil
+        raw.each_line do |line|
+          encoded = line[%r{\Afile://(\S+)}, 1]
+          if encoded
+            current = { path: decode_file_uri(encoded), snippet: nil }
+            hits << current
+          elsif current && current[:snippet].nil? && !line.strip.empty?
+            current[:snippet] = line.gsub(ANSI_SGR, '').strip
+          end
+        end
+        hits.uniq { |hit| hit[:path] }
+      end
+
+      # Full stored plain text of an indexed file, via +localsearch info
+      # -c -o json-ld <path>+. Works for PDFs/office docs (extracted at
+      # index time). +nil+ when the path isn't in the index or has no
+      # stored text — the caller turns that into an LLM-facing message.
+      #
+      # @param path [String]
+      # @return [String, nil]
+      def self.read_text(path)
+        result = Pikuri::Subprocess.spawn(BINARY, 'info', '-c', '-o', 'json-ld', path,
+                                          chdir: '/').wait
+        return nil unless result.status.success?
+
+        text = extract_plain_text(result.output)
+        text && !text.strip.empty? ? text : nil
+      end
+
+      # Find +nie:plainTextContent+ anywhere in the json-ld document (it
+      # nests under +@graph+, a named graph). Lenient: if the raw bytes
+      # don't parse as JSON (a stray diagnostic prefix), retry from the
+      # first +{+.
+      #
+      # @param raw [String]
+      # @return [String, nil]
+      def self.extract_plain_text(raw)
+        data = parse_json_lenient(raw)
+        data && find_plain_text(data)
+      end
+
+      # Verify localsearch is reachable; raise loudly otherwise.
+      #
+      # @raise [RuntimeError] if the binary is missing or unusable
+      def self.check_binaries!
+        result = Pikuri::Subprocess.spawn(BINARY, '--version', chdir: '/').wait
+        return if result.status.success?
+
+        raise install_hint
+      rescue Errno::ENOENT
+        raise install_hint
+      end
+
+      # Non-raising presence probe — lets a caller wire the file-index
+      # tools only when localsearch is installed and degrade gracefully
+      # otherwise, rather than failing the whole agent.
+      #
+      # @return [Boolean]
+      def self.available?
+        Pikuri::Subprocess.spawn(BINARY, '--version', chdir: '/').wait.status.success?
+      rescue Errno::ENOENT
+        false
+      end
+
+      # @param encoded [String] the tail of a +file://+ URI
+      # @return [String] percent-decoded path (+%XX+ only, UTF-8 tagged)
+      def self.decode_file_uri(encoded)
+        encoded.gsub(/%([0-9A-Fa-f]{2})/) { ::Regexp.last_match(1).hex.chr }
+               .force_encoding('UTF-8')
+      end
+      private_class_method :decode_file_uri
+
+      # @return [Object, nil]
+      def self.parse_json_lenient(raw)
+        JSON.parse(raw)
+      rescue JSON::ParserError
+        brace = raw.index('{')
+        return nil unless brace
+
+        begin
+          JSON.parse(raw[brace..])
+        rescue JSON::ParserError
+          nil
+        end
+      end
+      private_class_method :parse_json_lenient
+
+      # @param node [Object]
+      # @return [String, nil] first String under a +nie:plainTextContent+
+      #   key, depth-first
+      def self.find_plain_text(node)
+        case node
+        when Hash
+          node.each do |key, value|
+            return value if key == 'nie:plainTextContent' && value.is_a?(String)
+
+            found = find_plain_text(value)
+            return found if found
+          end
+          nil
+        when Array
+          node.each do |element|
+            found = find_plain_text(element)
+            return found if found
+          end
+          nil
+        end
+      end
+      private_class_method :find_plain_text
+
+      # @return [String]
+      def self.install_hint
+        "fileindex_search / fileindex_read require '#{BINARY}' on PATH — GNOME's " \
+          "desktop file indexer (package 'localsearch', formerly 'tracker-miners'), " \
+          'which ships with the GNOME desktop on Ubuntu 26.04+. Install it, or wire ' \
+          'these tools only on hosts that have it.'
+      end
+      private_class_method :install_hint
+    end
+  end
+end

data/lib/pikuri/os/machine_memory.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Pikuri
+  module Os
+    # Durable, cross-conversation memory for the OS helper — a single
+    # user-owned +MACHINE.md+, the +CLAUDE.md+-for-this-host. This class is
+    # the *read* half (resolve the XDG path, load content, size-warn,
+    # render the prompt section); the *write* half is the ordinary
+    # (confirmed) +write+/+edit+ tools plus a prompt line pointing the agent
+    # at {.default_path}. Narrative in book/os-assistant.md.
+    #
+    # A markdown file, not mem0: this agent reads untrusted local content
+    # (logs, downloads), so auto-extracted memory could be poisoned. A file
+    # makes that risk *visible + confirmed + reversible* — every write is a
+    # plain-text diff through the confirmer, the whole memory is one
+    # readable file — and needs no vector store or embedder, fitting the
+    # "small enough to audit" posture.
+    #
+    # {#content} reads fresh on every call and {#prompt_section} is re-pulled
+    # on every {Pikuri::Agent#clear_conversation}, so a user's or agent's
+    # edit surfaces on the next "/clear" rather than freezing at boot. Read
+    # a handful of times (boot + clears), never per turn.
+    class MachineMemory
+      LOGGER = Pikuri.logger_for('MachineMemory')
+
+      # @return [String] the memory filename.
+      FILENAME = 'MACHINE.md'
+
+      # @return [Integer] soft size ceiling. On overshoot we log a warning to
+      #   trim — mirrors the Claude CLI nudging an oversized CLAUDE.md.
+      #   No hard cap, no eviction: the file is prepended to every prompt,
+      #   so a visible nudge is enough.
+      OVERSIZE_BYTES = 10 * 1024
+
+      # @return [Pathname] the resolved memory file path.
+      attr_reader :path
+
+      # @param path [String, Pathname] memory file location; defaults to
+      #   {.default_path}. Overridable for tests.
+      def initialize(path: self.class.default_path)
+        @path = Pathname.new(path)
+      end
+
+      # Host-scoped path under pikuri's XDG config root
+      # ({Pikuri::Paths.config} — +$XDG_CONFIG_HOME/pikuri+ or
+      # +~/.config/pikuri+). Not project-local: for this agent the machine
+      # *and the user's home* are the subject, so the notes span the whole
+      # host.
+      #
+      # @return [String]
+      def self.default_path
+        Pikuri::Paths.config.join(FILENAME).to_s
+      end
+
+      # Raw file content, or +''+ when the file doesn't exist. Read
+      # fresh on each call (not memoized) so a re-pull on conversation
+      # clear picks up edits; logs the size-guard warning on each read.
+      #
+      # @return [String]
+      def content
+        read_content
+      end
+
+      # @return [Boolean] whether the content exceeds {OVERSIZE_BYTES}.
+      def oversized?
+        content.bytesize > OVERSIZE_BYTES
+      end
+
+      # The system-prompt section: the notes wrapped in a labeled block
+      # naming the file (so the agent knows where its memory lives and can
+      # update it), or +nil+ when there's nothing recorded yet — keeping
+      # the prompt lean until there's something to say.
+      #
+      # @return [String, nil]
+      def prompt_section
+        text = content.strip
+        return nil if text.empty?
+
+        <<~SECTION.chomp
+          # Notes about this machine
+
+          Durable, user-owned notes about this computer (kept in #{@path}). Treat
+          them as trusted context; edit the file to keep the notes current as you
+          learn about this computer's setup.
+
+          #{text}
+        SECTION
+      end
+
+      private
+
+      # @return [String]
+      def read_content
+        return '' unless @path.file?
+
+        text = @path.read
+        if text.bytesize > OVERSIZE_BYTES
+          LOGGER.warn(
+            "#{@path} is #{text.bytesize / 1024} KB (over the #{OVERSIZE_BYTES / 1024} KB " \
+            'guideline); it is prepended to every prompt — consider trimming it.'
+          )
+        end
+        text
+      end
+    end
+  end
+end

data/lib/pikuri/os/recoll.rb

@@ -0,0 +1,307 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Pikuri
+  module Os
+    # Backend wrapper over the +recoll+ full-text index, via its headless
+    # +recollq+ CLI (Xapian-backed). The DE-independent sibling of
+    # {LocalSearch}: same seam shape ({.search} / {.read_text} /
+    # {.available?} / {.check_binaries!}), interchangeable behind the
+    # +fileindex_search+ / +fileindex_read+ tools, and {Extension} wires
+    # whichever is +available?+ first.
+    #
+    # Recall on prose ties +localsearch+, so recoll is *not* a better
+    # engine — it earns its place where +localsearch+ is structurally blind
+    # (source trees, +.git+ checkouts, non-GNOME desktops). Its blind spot
+    # is the inverse: recoll indexes exactly the *folders* the user
+    # configured (+topdirs+), source code included — everything outside them
+    # is invisible. The measured survey behind the pick-one-per-host order
+    # is in +pikuri-os/DESIGN.md+.
+    #
+    # {.search} asks +recollq -F 'url abstract'+ for two base64-encoded
+    # fields per line (recoll's own recommended machine shape) so paths and
+    # snippets with spaces/brackets/newlines survive with no escaping
+    # guesswork. {.read_text} uses +recollq -d+, which dumps a document's
+    # stored/extracted text (PDFs/office docs included) after a header line.
+    module Recoll
+      # @return [String] the headless recoll query CLI.
+      BINARY = 'recollq'
+
+      # @return [String] the recoll indexer/config CLI (used only to locate
+      #   the config; never invoked to *index* from here).
+      INDEXER = 'recollindex'
+
+      # @return [String] recoll config filename inside the config dir.
+      CONFIG_FILE = 'recoll.conf'
+
+      # recoll's compiled-in default when +topdirs+ is unset: the whole
+      # home directory (the sample config ships +topdirs = ~+ too).
+      #
+      # @return [String]
+      DEFAULT_TOPDIRS = '~'
+
+      # @return [String] the Xapian index directory name inside the config
+      #   dir; its presence is what {.available?} treats as "usable".
+      INDEX_DIR = 'xapiandb'
+
+      # Raised when a recollq invocation exits non-zero — a recoverable
+      # failure the calling tool renders as +"Error: ..."+.
+      class CommandError < StandardError; end
+
+      # @return [String] short human name for this backend, used by the
+      #   fileindex tools when prefixing an +"Error: ..."+ observation.
+      def self.label = 'recoll'
+
+      # Plain-English note of what this backend will *not* surface, appended
+      # to the fileindex tools' LLM description so the model knows when to
+      # fall back to a filesystem text/regex search. A *method*, not a
+      # constant like {LocalSearch::LIMITATIONS}, because the blind spot is
+      # host-specific: the user's +topdirs+, only readable at runtime
+      # ({.indexed_dirs}). Falls back to a generic caveat when the config
+      # can't be read.
+      #
+      # @return [String]
+      def self.limitations
+        dirs = indexed_dirs
+        return generic_limitation if dirs.empty?
+
+        listed = dirs.map { |dir| "- #{dir}" }.join("\n")
+        <<~LIMITS.chomp
+          Only files under these folders are indexed (anything elsewhere on disk is not in this index — search those files' text directly instead):
+          #{listed}
+        LIMITS
+      end
+
+      # Folders recoll is configured to index, read from +topdirs+ in the
+      # active config file. Resolution mirrors recoll itself: the config dir
+      # is +$RECOLL_CONFDIR+ or +~/.recoll+; +topdirs+ is a space-separated
+      # list (quoted entries may contain spaces, a trailing +\+ continues to
+      # the next line) living in the global section, i.e. above the first
+      # +[section]+ header. Entries are +~+- and +$VAR+-expanded to absolute
+      # paths. Defaults to the home dir ({DEFAULT_TOPDIRS}) when the config
+      # is absent or names no +topdirs+, matching recoll's own default.
+      #
+      # @return [Array<String>] absolute folder paths, or +[]+ if the config
+      #   dir exists but the file is unreadable
+      def self.indexed_dirs
+        path = File.join(confdir, CONFIG_FILE)
+        raw = File.exist?(path) ? File.read(path) : nil
+        entries = raw ? parse_topdirs(raw) : [DEFAULT_TOPDIRS]
+        entries = [DEFAULT_TOPDIRS] if entries.empty?
+        entries.map { |entry| expand_entry(entry) }
+      rescue SystemCallError
+        []
+      end
+
+      # The recoll configuration directory recoll would use: +$RECOLL_CONFDIR+
+      # if set and non-empty, else +~/.recoll+.
+      #
+      # @return [String]
+      def self.confdir
+        env = ENV['RECOLL_CONFDIR']
+        env && !env.empty? ? File.expand_path(env) : File.expand_path('~/.recoll')
+      end
+
+      # Run +recollq -F 'url abstract' -n <limit>+ and parse the results.
+      #
+      # @param query [String] recoll query-language string
+      # @param limit [Integer]
+      # @return [Array<Hash>] +[{ path: String, snippet: String|nil }, …]+,
+      #   de-duplicated by path
+      # @raise [CommandError] if recollq exits non-zero
+      def self.search(query:, limit:)
+        result = Pikuri::Subprocess.spawn(BINARY, '-F', 'url abstract', '-n', limit.to_s, query,
+                                          chdir: '/').wait
+        unless result.status.success?
+          stderr = result.output.strip
+          stderr = "exited #{result.status.exitstatus}" if stderr.empty?
+          raise CommandError, stderr
+        end
+
+        parse_search_output(result.output)
+      end
+
+      # Parse +recollq -F 'url abstract'+ output into hits. Each data line is
+      # two base64 tokens (url, abstract); recollq's +"Recoll query: …"+ /
+      # +"N results"+ banner lines are skipped because their first token
+      # isn't valid base64 for a +file://+ URL.
+      #
+      # @param raw [String]
+      # @return [Array<Hash>] +[{ path:, snippet: }, …]+ de-duped by path
+      def self.parse_search_output(raw)
+        hits = []
+        raw.each_line do |line|
+          url_b64, abstract_b64 = line.split(/\s+/, 2)
+          url = decode_base64(url_b64)
+          next unless url&.start_with?('file://')
+
+          snippet = decode_base64(abstract_b64.to_s.strip)
+          snippet = snippet&.strip
+          hits << { path: url.delete_prefix('file://'), snippet: (snippet unless snippet.to_s.empty?) }
+        end
+        hits.uniq { |hit| hit[:path] }
+      end
+
+      # Full stored/extracted text of an indexed file, via +recollq -d+
+      # scoped to the file's own name. Works for PDFs/office docs (extracted
+      # at index time). +nil+ when the path isn't in the index or has no
+      # stored text — the caller turns that into an LLM-facing message.
+      #
+      # @param path [String]
+      # @return [String, nil]
+      def self.read_text(path)
+        abs = File.expand_path(path)
+        result = Pikuri::Subprocess.spawn(BINARY, '-d', '-n', '50', "filename:#{File.basename(abs)}",
+                                          chdir: '/').wait
+        return nil unless result.status.success?
+
+        text = extract_dumped_text(result.output, abs)
+        text && !text.strip.empty? ? text : nil
+      end
+
+      # Pull the dumped body of the result whose header line matches +abs+
+      # out of +recollq -d+ output. Each result is a tab-delimited header
+      # line (+<mime>\t[<url>]\t[<title>]\t<n>\tbytes+) followed by its text,
+      # up to the next header line.
+      #
+      # @param raw [String]
+      # @param abs [String] absolute path to match against the header URL
+      # @return [String, nil]
+      def self.extract_dumped_text(raw, abs)
+        want = "file://#{abs}"
+        body = []
+        capturing = false
+        raw.each_line do |line|
+          url = line[%r{\t\[(file://[^\]]*)\]\t}, 1]
+          if url
+            break if capturing
+
+            capturing = (url == want)
+          elsif capturing
+            body << line
+          end
+        end
+        capturing || !body.empty? ? body.join.chomp : nil
+      end
+
+      # Verify recollq is reachable; raise loudly otherwise. Presence is
+      # judged by whether the binary *ran* (no +Errno::ENOENT+), not by exit
+      # status: +recollq -h+ exits non-zero by design, so a zero-exit probe
+      # would false-negative on a perfectly good install.
+      #
+      # @raise [RuntimeError] if the binary is missing
+      def self.check_binaries!
+        Pikuri::Subprocess.spawn(BINARY, '-h', chdir: '/').wait
+      rescue Errno::ENOENT
+        raise install_hint
+      end
+
+      # Non-raising *usability* probe — for callers that select a backend
+      # among several and want the one that can actually serve searches.
+      # Stricter than {.check_binaries!}: it requires both the +recollq+
+      # binary *and* a built index ({INDEX_DIR} under {.confdir}), because
+      # an installed-but-never-indexed recoll errors ("Xapian index open
+      # error") on every query — so it must not win backend selection over
+      # a working alternative. See {.check_binaries!} for why exit status is
+      # ignored for the binary half.
+      #
+      # @return [Boolean]
+      def self.available?
+        return false unless File.directory?(File.join(confdir, INDEX_DIR))
+
+        Pikuri::Subprocess.spawn(BINARY, '-h', chdir: '/').wait
+        true
+      rescue Errno::ENOENT
+        false
+      end
+
+      # Join backslash-continued lines, then read the +topdirs+ value from
+      # the global section (everything before the first +[section]+ header).
+      # The last assignment wins (ConfSimple semantics).
+      #
+      # @param raw [String]
+      # @return [Array<String>] raw (unexpanded) topdirs entries
+      def self.parse_topdirs(raw)
+        value = nil
+        joined_lines(raw).each do |line|
+          stripped = line.strip
+          break if stripped.start_with?('[')       # first [section] ends the global scope
+          next if stripped.empty? || stripped.start_with?('#')
+
+          key, _, rest = stripped.partition('=')
+          value = rest if key.strip == 'topdirs'
+        end
+        value ? tokenize(value) : []
+      end
+
+      # @param raw [String]
+      # @return [Array<String>] logical lines with trailing-backslash
+      #   continuations folded into one
+      def self.joined_lines(raw)
+        lines = []
+        buffer = +''
+        raw.each_line(chomp: true) do |line|
+          if line.end_with?('\\')
+            buffer << line[0..-2] << ' '
+          else
+            lines << (buffer + line)
+            buffer = +''
+          end
+        end
+        lines << buffer unless buffer.empty?
+        lines
+      end
+      private_class_method :joined_lines
+
+      # Split a topdirs value into entries, honouring +"double quotes"+
+      # around paths that contain spaces.
+      #
+      # @param value [String]
+      # @return [Array<String>]
+      def self.tokenize(value)
+        value.scan(/"([^"]*)"|(\S+)/).map { |quoted, bare| quoted || bare }
+      end
+      private_class_method :tokenize
+
+      # Expand +$VAR+ / +${VAR}+ then +~+ to an absolute path; a relative
+      # entry resolves against the home dir (recoll's own base).
+      #
+      # @param entry [String]
+      # @return [String]
+      def self.expand_entry(entry)
+        env_expanded = entry.gsub(/\$\{(\w+)\}|\$(\w+)/) { ENV[::Regexp.last_match(1) || ::Regexp.last_match(2)] || '' }
+        File.expand_path(env_expanded, Dir.home)
+      end
+      private_class_method :expand_entry
+
+      # @param token [String, nil]
+      # @return [String, nil] decoded bytes tagged UTF-8, or +nil+ if +token+
+      #   is blank or not valid strict base64
+      def self.decode_base64(token)
+        return nil if token.nil? || token.empty?
+
+        Base64.strict_decode64(token).force_encoding('UTF-8')
+      rescue ArgumentError
+        nil
+      end
+      private_class_method :decode_base64
+
+      # @return [String]
+      def self.generic_limitation
+        'Only folders configured in recoll (topdirs) are indexed; files ' \
+          "outside them are not found — search those files' text directly instead."
+      end
+      private_class_method :generic_limitation
+
+      # @return [String]
+      def self.install_hint
+        "fileindex_search / fileindex_read require '#{BINARY}' on PATH — the " \
+          "recoll full-text indexer's headless query CLI (package 'recoll'). " \
+          'Install it and build an index (recollindex), or wire these tools ' \
+          'only on hosts that have it.'
+      end
+      private_class_method :install_hint
+    end
+  end
+end