gemchat 0.1.0

data/lib/gemchat/paths.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gemchat
+  # Where things live on disk.
+  #
+  # Its own file so that anything loaded on its own — notably the Bundler plugin
+  # shim, which runs in a fresh child during `bundle install` — can resolve a path
+  # without requiring the whole gem, and without a circular require back through
+  # lib/gemchat.rb.
+  module Paths
+    module_function
+
+    # Index location: the ri cache and index.sqlite.
+    #
+    # GEMCHAT_HOME is taken verbatim. An override that silently grew a suffix
+    # would make it impossible to predict where the data went.
+    #
+    # Deliberately not memoised. Memoising a value derived from the environment
+    # means the first reader wins for the life of the process, so setting
+    # GEMCHAT_HOME later does nothing — which once made the test suite write to
+    # the developer's real ~/.gemchat. The join is cheap enough to repeat.
+    def root
+      File.expand_path(ENV["GEMCHAT_HOME"] || File.join(Dir.home, ".gemchat"))
+    end
+
+    # Settings and trust records, kept separate from the index so wiping the
+    # cache does not also discard the decisions about which projects to trust.
+    def config_home
+      ENV["GEMCHAT_CONFIG_HOME"] ||
+        File.join(ENV["XDG_CONFIG_HOME"] || File.join(Dir.home, ".config"), "gemchat")
+    end
+  end
+end

data/lib/gemchat/plugin_index.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Gemchat
+  # Reads Bundler's own record of which plugins are installed and which hooks
+  # they registered.
+  #
+  # This exists because "is the hook actually armed?" cannot be answered by
+  # looking at the Gemfile. A Gemfile line only says the plugin was *requested*;
+  # Bundler activates it on the next install, and only writes this index when it
+  # does. The difference is the whole point — §15.1a lists four ways a plugin
+  # looks installed and does nothing, and `hooks:` being empty here is the
+  # signature of the worst one: a plugin that ran once, during the install that
+  # installed it, and never again.
+  module PluginIndex
+    FILENAME = "index"
+
+    module_function
+
+    # Honours BUNDLE_APP_CONFIG the way Bundler does, so this reads the same
+    # index the hook does.
+    def dir
+      config = ENV["BUNDLE_APP_CONFIG"]
+      config ||= File.join(Dir.pwd, ".bundle")
+      File.join(config, "plugin")
+    end
+
+    def path
+      File.join(dir, FILENAME)
+    end
+
+    def exist?
+      File.file?(path)
+    end
+
+    def data
+      return nil unless exist?
+
+      parsed = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true)
+      parsed.is_a?(Hash) ? parsed : nil
+    rescue
+      nil
+    end
+
+    def installed_plugins
+      data&.dig("plugin_paths")&.keys || []
+    end
+
+    def registered_hooks
+      data&.dig("hooks") || {}
+    end
+
+    def hook?(name, event = "after-install-all")
+      Array(registered_hooks[event]).include?(name)
+    end
+
+    # Whether the hook will actually run, as opposed to being merely declared.
+    def armed?(name = "gemchat", event: "after-install-all")
+      hook?(name, event)
+    end
+  end
+end

data/lib/gemchat/prose.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require_relative "markdown"
+
+module Gemchat
+  # The prose half of a gem: its README, its guides, and the reference material
+  # it ships under `doc/`.
+  #
+  # ri answers "what does this method take and return". Prose answers "how is
+  # this library actually used", which is the question an agent most often has
+  # and the one ri is worst at. Each source is stored under its own
+  # `source_type`, so a gem's README and its reference manual never overwrite one
+  # another and search output can say what kind of document a hit came from.
+  #
+  # The `readme` / `guides` / `doc` split is not incidental: `gemchat_app` already
+  # uses exactly this vocabulary, gives `guides` a ranking prior of 1.15, and
+  # groups `readme`+`guides` only at query time to enforce prose diversity. See
+  # §20.1 of the plan.
+  class Prose
+    FILENAME = "README"
+    PROSE_EXTENSIONS = %w[.md .markdown .mkd .rdoc .txt .textile].freeze
+    GUIDE_DIRS = %w[guides guide].freeze
+    DOC_DIRS = %w[doc docs].freeze
+
+    # Generated output. A gem installed with documentation has ri under
+    # `doc/<name>-<version>/ri`, and older gemspecs committed rdoc's HTML tree.
+    # Both duplicate what the ri half already covers, so both are dropped — but
+    # only from `doc/`, and only by shape. Hand-written files are kept.
+    GENERATED_DIRS = %w[ri].freeze
+    GENERATED_EXTENSIONS = %w[.html .htm .js .css].freeze
+
+    # Languages seen in bundled documentation, keyed by the marker that appears
+    # in a filename or directory name. Skipped by default: they cost embedding
+    # time and can never match an English query, while polluting BM25 with tokens
+    # no English query contains. `en` is deliberately absent.
+    FOREIGN_MARKERS = %w[
+      ja zh ko fr de es pt it ru pl tr nl sv da no fi cs hu el he ar hi th vi
+      id uk ro bg sr hr sk sl et lv lt ca gl eu af sw fa ur bn ta ms tl
+    ].freeze
+
+    Document = Struct.new(:source_type, :label, :path, :text)
+
+    class << self
+      def chunks(gem_name, gem_version, gem_dir, all_languages: false)
+        documents(gem_dir, all_languages: all_languages)
+          .flat_map { |doc| chunks_for(gem_name, gem_version, doc) }
+      end
+
+      def documents(gem_dir, all_languages: false)
+        paths(gem_dir, all_languages: all_languages).map { |path|
+          Document.new(source_type_for(gem_dir, path), label_for(gem_dir, path),
+            path, read(path))
+        }
+      end
+
+      def paths(gem_dir, all_languages: false)
+        return [] unless File.directory?(gem_dir)
+
+        all = [readme_in(gem_dir)] +
+          within(gem_dir, GUIDE_DIRS) +
+          within(gem_dir, DOC_DIRS)
+
+        all.compact.uniq.select { |path| indexable?(path, all_languages) }
+      end
+
+      def source_type_for(gem_dir, path)
+        relative = path.sub("#{gem_dir}/", "")
+        return "readme" unless relative.include?("/")
+
+        first = relative.split("/").first
+        GUIDE_DIRS.include?(first) ? "guides" : "doc"
+      end
+
+      # The directory the document lives in, relative to the gem root. Keeps
+      # `doc/rexml/parsers.md` and `doc/csv/io.rdoc` distinguishable in a result
+      # title, which "doc" alone would not.
+      def label_for(gem_dir, path)
+        relative = path.sub("#{gem_dir}/", "")
+        return "README" unless relative.include?("/")
+
+        File.dirname(relative)
+      end
+
+      private
+
+      def within(gem_dir, dirs)
+        dirs.flat_map { |d| Dir.glob(File.join(gem_dir, d, "**", "*")) }
+      end
+
+      def readme_in(gem_dir)
+        Prose::PROSE_EXTENSIONS.each do |ext|
+          candidate = File.join(gem_dir, "#{FILENAME}#{ext}")
+          return candidate if File.file?(candidate)
+        end
+        bare = File.join(gem_dir, FILENAME)
+        bare if File.file?(bare)
+      end
+
+      def indexable?(path, all_languages)
+        return false unless File.file?(path)
+        return false if foreign?(path) && !all_languages
+        # A README is taken whatever its extension; everything under a guide or
+        # doc directory has to look like prose, or the tree fills up with
+        # example sources and translation catalogues.
+        return true if File.basename(path).start_with?(FILENAME.downcase)
+        return false unless PROSE_EXTENSIONS.include?(extension(path))
+
+        !generated?(path)
+      end
+
+      def extension(path)
+        File.extname(path).downcase
+      end
+
+      def generated?(path)
+        parts = path.split("/")
+        return true if parts.any? { |p| GENERATED_DIRS.include?(p) }
+
+        GENERATED_EXTENSIONS.include?(extension(path))
+      end
+
+      # Matches a language marker only when it stands alone as a path segment or
+      # as a dot- or underscore-delimited token in the basename, so a directory
+      # called `text` or `example` is not mistaken for one.
+      def foreign?(path)
+        parts = path.split("/")
+        base = File.basename(parts.pop, ".*").downcase
+        tokens = base.split(/[._-]/).map { |p| p.sub(/\.(rd|md|txt|textile)$/, "") }
+        return true if (parts + tokens).any? { |p| FOREIGN_MARKERS.include?(p.downcase) }
+        # `irb.rd.ja` and `doc.ja.md` put the marker in a trailing extension.
+        extra = File.basename(path).downcase.split(".").map { |p| p.sub(/^rd$/, "") }
+        extra.any? { |p| FOREIGN_MARKERS.include?(p) }
+      end
+
+      # scrub on the success path too, not only when the read raises: a file
+      # with a few invalid bytes reads fine and then explodes in gsub, which
+      # costs a gem its prose AND aborts the run part-way through the corpus.
+      def read(path)
+        File.read(path, mode: "r:UTF-8").scrub
+      rescue
+        File.binread(path).force_encoding("UTF-8").scrub
+      end
+
+      def chunks_for(gem_name, gem_version, doc)
+        Markdown.sections(clean(doc.text)).flat_map { |section|
+          body = tidy(section.body)
+          next [] if body.empty?
+
+          # Split once, then label, so the part count is known without
+          # re-splitting the body for every part's title.
+          parts = split(body, max: Chunker::MAX_BODY_CHARS)
+          total = parts.size
+
+          parts.each_with_index.map { |part, i|
+            Chunker::Chunk.new(
+              gem_name: gem_name,
+              gem_version: gem_version,
+              source_type: doc.source_type,
+              title: title(gem_name, doc, section, total: total, index: i),
+              body: part,
+              class_name: nil,
+              method_name: nil,
+              method_type: nil,
+              signature: nil
+            )
+          }
+        }
+      end
+
+      # A `· 2/5` suffix only appears once a section is actually split, so the
+      # common case keeps a clean title and a search returning three parts of one
+      # section is not indistinguishable from three unrelated hits.
+      def title(gem_name, doc, section, total:, index:)
+        base = [gem_name, doc.label, section.title].compact.reject(&:empty?).join(" › ")
+
+        (total > 1) ? "#{base} · #{index + 1}/#{total}" : base
+      end
+
+      # Split on blank lines so a chunk boundary is a paragraph boundary, never
+      # mid-sentence. Paragraphs longer than the cap are hard-cut as a last
+      # resort, because dropping them would lose content silently.
+      def split(body, max:)
+        return [body] if body.length <= max
+
+        out = []
+        current = +""
+        body.split(/\n{2,}/).each do |paragraph|
+          if current.empty?
+            current << paragraph.dup
+          elsif current.length + paragraph.length + 2 <= max
+            current << "\n\n" << paragraph
+          else
+            out << current
+            current = +""
+            current << paragraph.dup
+          end
+        end
+        out << current unless current.empty?
+        out.flat_map { |chunk| (chunk.length <= max) ? [chunk] : chunk.scan(/.{1,#{max}}/m) }
+      end
+
+      def clean(text)
+        text.gsub(/\[!\[[^\]]*\]\([^)]*\)\]\([^)]*\)/, "")
+      end
+
+      def tidy(body)
+        body.gsub(/<!--.*?-->/m, "").strip
+      end
+    end
+  end
+end

data/lib/gemchat/ri.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Gemchat
+  # Generates and reads ri documentation for an installed gem.
+  #
+  # ri is the primary corpus: it carries the API surface (class, method,
+  # signature, description) that an agent needs to call a gem correctly. Only
+  # lib/ and ext/ are documented, and --all is deliberately omitted, so private
+  # and internal machinery stays out of the index.
+  class Ri
+    SCHEMA_VERSION = "1"
+
+    class GenerationError < StandardError; end
+
+    def self.documented_dirs(gem_dir)
+      %w[lib ext].select { |dir| File.directory?(File.join(gem_dir, dir)) }
+    end
+
+    # Reads an existing ri store, as installed by `gem install` or `gem rdoc`.
+    # Returns nil when absent. Bundler does not generate ri by default, so this
+    # is a fast path rather than the common case.
+    def self.existing_store_path(name, version)
+      roots = gem_roots
+      roots.each do |root|
+        candidate = File.join(root, "doc", "#{name}-#{version}", "ri")
+        return candidate if File.directory?(candidate)
+      end
+      nil
+    end
+
+    # Bundler's install root is usually absent from Gem.path. Resolved
+    # defensively: a bare lockfile with no active bundle is normal when gemchat
+    # runs outside `bundle exec`.
+    def self.gem_roots
+      roots = Gem.path.dup
+      if defined?(Bundler)
+        begin
+          bundle_path = Bundler.bundle_path.to_s
+          if File.directory?(bundle_path)
+            roots << bundle_path
+            roots << File.join(bundle_path, "ruby", RbConfig::CONFIG["ruby_version"])
+          end
+        rescue StandardError, LoadError
+          nil
+        end
+      end
+      roots.compact.uniq
+    end
+
+    def self.locate(name, version)
+      existing_store_path(name, version) || generated_store_path(name, version)
+    end
+
+    def self.generated_store_path(name, version)
+      File.join(Gemchat.root, "ri", "#{name}-#{version}")
+    end
+
+    # Generates an ri store into the private cache. RDoc creates the output
+    # directory itself and refuses to write into one that already exists, so
+    # the parent is created and the target is left alone.
+    def self.generate(name, version, gem_dir:, output_dir:)
+      dirs = documented_dirs(gem_dir)
+      return nil if dirs.empty?
+
+      FileUtils.mkdir_p(File.dirname(output_dir))
+      return output_dir if File.directory?(output_dir)
+
+      args = ["--ri", "--op", output_dir, "--quiet", "--force-update", *dirs]
+      Dir.chdir(gem_dir) { RDoc::RDoc.new.document(args) }
+      output_dir
+    rescue SystemStackError
+      # RDoc recurses on symlink cycles and deeply nested modules.
+      # SystemStackError < Exception, so a bare rescue misses it, and a
+      # truncated store must never be read.
+      FileUtils.rm_rf(output_dir)
+      raise GenerationError, "ri generation for #{name} overflowed the stack"
+    rescue => e
+      FileUtils.rm_rf(output_dir)
+      raise GenerationError, "ri generation for #{name} failed: #{e.message}"
+    end
+
+    # Reads an RDoc store into plain Ruby data:
+    #   [{ class_name:, description:, methods: [{ name:, type:, signature:, description: }] }]
+    def self.read(ri_path, name)
+      return [] unless File.directory?(ri_path)
+
+      driver = RDoc::RI::Driver.new(RDoc::RI::Driver.process_args(["--doc-dir", ri_path]))
+      store = driver.stores.find { |s| s.path == ri_path || s.path.to_s.include?(name) }
+      return [] unless store
+
+      store.cache[:modules].filter_map { |class_name| read_class(store, class_name) }
+    end
+
+    def self.read_class(store, class_name)
+      klass = store.load_class(class_name)
+      methods = klass.method_list.filter_map { |m| read_method(store, class_name, m) }
+      {
+        class_name: class_name,
+        description: markup_to_text(klass.comment),
+        methods: methods
+      }
+    rescue
+      nil
+    end
+    private_class_method :read_class
+
+    def self.read_method(store, class_name, method)
+      full = store.load_method(class_name, "#{(method.type == "class") ? "::" : "#"}#{method.name}")
+      {
+        name: method.name,
+        type: method.type,
+        signature: full.arglists || full.call_seq || method.name,
+        description: markup_to_text(full.comment)
+      }
+    rescue
+      nil
+    end
+    private_class_method :read_method
+
+    def self.markup_to_text(comment)
+      return "" unless comment
+
+      doc = comment.respond_to?(:parse) ? comment.parse : comment
+      doc.accept(RDoc::Markup::ToMarkdown.new).strip
+    rescue
+      ""
+    end
+    private_class_method :markup_to_text
+  end
+end

data/lib/gemchat/stopwords.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gemchat
+  # The stopwords the lexical arm drops before building an FTS5 query.
+  #
+  # FTS5 has no `stopwords=` option, so without this the words below sit in the
+  # index and match. On an OR'd query that is the whole problem: `the`, `of` and
+  # `how` appear in nearly every chunk, so they match everything and rank
+  # nothing. Measured on a ~22k-chunk corpus, the OR'd arm located the right
+  # chunk for 1 of 5 known-answer queries; see §11.3 of the plan.
+  #
+  # Vendored from PostgreSQL's `src/backend/snowball/stopwords/english.stop`,
+  # the same 127 words that `websearch_to_tsquery('english', ...)` uses in
+  # gemchat_app. Pinned by a count in the tests because a truncated paste is the
+  # most likely way this rots silently, and it would fail quietly -- the arm
+  # would just get slightly worse.
+  #
+  # Do NOT substitute gemchat_app's own list. That one
+  # (`app/services/agentic/gem_similarity_service.rb`) is tuned for SEO title
+  # work and drops `gem`, `ruby`, `library`, `tool`, `management` and `system`,
+  # all of which are signal in a documentation index.
+  #
+  # Not the same algorithm, either: this list is paired with FTS5's `porter`
+  # (Porter-1980) in the schema, whereas gemchat_app stems with Snowball via
+  # PostgreSQL. Local and hosted retrieval will never stem identically.
+  module Stopwords
+    ENGLISH = %w[
+      i me my myself we our ours ourselves
+      you your yours yourself yourselves he him his himself she her hers herself
+      it its itself they them their theirs themselves
+      what which who whom this that these those
+      am is are was were be been being have has had having
+      do does did doing a an the and but if or because as until while
+      of at by for with about against between into through during
+      before after above below to from up down
+      in out on off over under again further then once here there
+      when where why how all any both each few more most other some such
+      no nor not only own same so than too very s t
+      can will just don should now
+    ].to_set.freeze
+
+    # True when every alphanumeric token of `term` is a stopword.
+    #
+    # The tokenisation matters and is not a detail. `unicode61` splits on
+    # non-alphanumerics, so `how-to` is two tokens -- `how` and `to` -- both
+    # stopwords, and `don't` is `don` and `t`, which are *both* in the upstream
+    # list precisely because Snowball wrote it that way. A plain `include?` on the
+    # whole term would keep both, which is where the noise survives.
+    #
+    # `[[:alnum:]]` rather than `[a-z0-9]` because unicode61 keeps accented and
+    # non-Latin letters inside a token; splitting on ASCII would tear `café` into
+    # `caf`, which happens to be harmless here (a fragment is not a stopword, so
+    # the term is kept) but is the wrong rule to encode.
+    #
+    # A term with no alphanumeric tokens at all is *not* treated as a stopword.
+    # Returning false keeps it in the query, so a degenerate term cannot empty
+    # the list and turn a real question into a nil query.
+    def self.only_stopwords?(term)
+      tokens = term.to_s.downcase.scan(/[[:alnum:]]+/)
+      return false if tokens.empty?
+
+      tokens.all? { |token| ENGLISH.include?(token) }
+    end
+  end
+end