gemchat 0.1.0

data/lib/gemchat/indexer.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "bundler"
+
+module Gemchat
+  # Indexes the gems in a lockfile. ri generation is the main path, because
+  # Bundler does not produce ri on install (§8.3); reading a pre-existing store
+  # is a fast path for `gem install` users and for anyone who ran `gem rdoc`.
+  class Indexer
+    # `status` is what the store did: :indexed, :updated or :unchanged. It is
+    # deliberately separate from `ri_source`, which describes where the ri came
+    # from. Cached ri plus an unchanged digest is a normal no-op, and conflating
+    # the two makes it look like work happened.
+    Result = Struct.new(:name, :version, :status, :chunks, :ri_source, :error) do
+      def ok?
+        error.nil?
+      end
+
+      def changed?
+        status == :indexed || status == :updated
+      end
+    end
+
+    def initialize(store:, lockfile_path: "Gemfile.lock", include_transitive: false,
+      all_languages: false)
+      @store = store
+      @lockfile_path = lockfile_path
+      @include_transitive = include_transitive
+      @all_languages = all_languages
+    end
+
+    def lockfile_specs
+      parser = ::Bundler::LockfileParser.new(File.read(@lockfile_path))
+      specs = parser.specs.reject { |s| s.name == "ruby-core-stdlib" }
+      unless @include_transitive
+        direct = parser.dependencies.keys - ["ruby-core-stdlib"]
+        specs = specs.select { |s| direct.include?(s.name) }
+      end
+      specs.map { |s| [s.name, s.version.to_s] }
+    end
+
+    def index_gem(name, version, prefer_existing: true)
+      gem_dir = gem_dir_for(name, version)
+      return Result.new(name:, version:, status: :missing, chunks: 0, ri_source: "missing") unless gem_dir
+
+      ri_source = nil
+      ri_path = prefer_existing ? Ri.existing_store_path(name, version) : nil
+      ri_source = "existing" if ri_path
+
+      if ri_path.nil?
+        ri_path = Ri.generated_store_path(name, version)
+        if File.directory?(ri_path)
+          ri_source ||= "generated"
+        else
+          begin
+            ri_path = Ri.generate(name, version, gem_dir:, output_dir: ri_path)
+            ri_source = "generated"
+          rescue Ri::GenerationError => e
+            return Result.new(name:, version:, status: :failed, chunks: 0, ri_source: "failed", error: e.message)
+          end
+        end
+      end
+
+      chunks = Chunker.from_ri(name, version, Ri.read(ri_path, name))
+      attach_sources(chunks, Symbols.for(gem_dir))
+      outcome = @store.replace_gem(name, version, "ri", chunks)
+
+      # The prose half, indexed separately and labelled by source_type. Stored
+      # under its own source_type so a gem that gains or loses a README does not
+      # force the much larger ri row to be rewritten.
+      prose = Prose.chunks(name, version, gem_dir, all_languages: @all_languages)
+      prose_outcome = write_prose(name, version, prose)
+
+      Result.new(name:, version:, status: outcome.status,
+        chunks: outcome.chunks + prose_outcome.sum(&:chunks), ri_source: ri_source)
+    rescue => e
+      Result.new(name:, version:, status: :error, chunks: 0, ri_source: "error", error: "#{e.class}: #{e.message}")
+    end
+
+    # Each prose source is written under its own source_type so a gem that gains
+    # or loses a README never rewrites its much larger ri row.
+    def write_prose(name, version, chunks)
+      chunks.group_by(&:source_type).map do |source_type, group|
+        @store.replace_gem(name, version, source_type, group)
+      end
+    end
+
+    # Stamps each ri chunk with the file and line its definition came from, using
+    # the Prism pass's table.
+    #
+    # Matching is on (class_name, method_name, method_type) because that is all an
+    # ri chunk knows and all a definition carries. It is deliberately not
+    # fuzzy: a chunk that cannot be matched keeps a nil location, and an invented
+    # path is worse than no path for a citation.
+    def attach_sources(chunks, table)
+      return chunks if table.empty?
+
+      chunks.each do |chunk|
+        definition = table[[chunk.class_name.to_s, chunk.method_name.to_s, chunk.method_type.to_s]]
+        next if definition.nil?
+
+        chunk.source_path = definition.source_path
+        chunk.source_line = definition.source_line
+      end
+    end
+
+    def gem_dir_for(name, version)
+      spec = Gem::Specification.find_by_name(name, version)
+      spec&.gem_dir
+    rescue Gem::MissingSpecError
+      nil
+    end
+  end
+end

data/lib/gemchat/init.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Gemchat
+  # `gemchat init` — the once-per-project setup, and the only command that may
+  # modify a Gemfile.
+  #
+  # Separate from `index` on purpose. A command called `index` that edits your
+  # Gemfile is a surprise, and in CI or a pre-commit hook it would either block
+  # on a prompt — hanging the build — or silently rewrite a file it has no
+  # business touching. So `index` stays non-interactive and read-only, and this
+  # one is explicit about being the mutating one.
+  #
+  # Idempotent: running it twice changes nothing and says so. That matters
+  # because the two lines it adds are easy to get wrong by hand, and a
+  # half-configured plugin fails *silently* — the hook appears installed and
+  # never fires.
+  class Init
+    GEM_LINE = 'gem "gemchat", group: :development'
+    PLUGIN_LINE = 'plugin "gemchat"'
+    GEMFILE = "Gemfile"
+    COMMENT = "# gemchat: local documentation index"
+
+    Result = Struct.new(:added, :already_present, :indexed) do
+      def changed? = added.any?
+    end
+
+    def initialize(dir: Dir.pwd, stdout: $stdout, stderr: $stderr, assume_yes: false)
+      @dir = dir
+      @stdout = stdout
+      @stderr = stderr
+      @assume_yes = assume_yes
+    end
+
+    def gemfile_path
+      File.join(@dir, GEMFILE)
+    end
+
+    def run
+      unless File.file?(gemfile_path)
+        raise Error, "no #{GEMFILE} in #{@dir} — run `bundle init` first"
+      end
+
+      unless Lockfile.exist?(@dir)
+        raise Error, "no Gemfile.lock in #{@dir} — run `bundle install` first, " \
+          "so gemchat has a bundle to index"
+      end
+
+      result = Result.new(added: [], already_present: [], indexed: false)
+
+      pending = [GEM_LINE, PLUGIN_LINE].reject do |line|
+        present = declared?(line)
+        result.already_present << line if present
+        present
+      end
+
+      unless pending.empty?
+        confirm!(pending)
+        append(pending)
+        result.added.concat(pending)
+      end
+
+      # Opting in by running init is deliberate, so trust is granted here. It
+      # also writes .gemchat.yml, which is what lets the hook skip the next
+      # bundle install for free.
+      Trust.grant!(@dir, via: "gemchat init")
+      result.indexed = true
+      result
+    end
+
+    # Matches the *directive* as well as the name, so a user's own
+    # `gem "gemchat", path: "..."` counts as declaring the dependency while a
+    # `plugin "gemchat"` counts for the plugin. Keying on the name alone
+    # reported both as present the moment either was written, and the second
+    # line was silently never added -- the plugin then never fires, which is the
+    # exact silent failure this command exists to prevent.
+    def declared?(line)
+      return false unless File.file?(gemfile_path)
+
+      directive = line[/\A(gem|plugin)\b/, 1]
+      gem_name = line[/"([^"]+)"/, 1]
+      return false unless directive && gem_name
+
+      File.readlines(gemfile_path).any? do |candidate|
+        candidate.match?(/^\s*#{Regexp.escape(directive)}\s+["']#{Regexp.escape(gem_name)}["']/)
+      end
+    end
+
+    def append(lines)
+      body = File.read(gemfile_path)
+      body += "\n" unless body.empty? || body.end_with?("\n")
+      File.write(gemfile_path, "#{body}\n#{COMMENT}\n#{lines.join("\n")}\n")
+    end
+
+    def confirm!(lines)
+      return if @assume_yes || !@stdout.tty?
+
+      lines.each { |line| @stdout.puts("add to #{GEMFILE}: #{line}") }
+      @stdout.print "  ok? [y/N] "
+      answer = $stdin.gets.to_s.strip.downcase
+      return if ["y", "yes"].include?(answer)
+
+      raise Error, "aborted; #{GEMFILE} unchanged"
+    end
+  end
+end

data/lib/gemchat/lockfile.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Gemchat
+  # The lockfile digest, which is the whole basis of the staleness check.
+  #
+  # Every command that changes a bundle — `install`, `add`, `update` — rewrites
+  # Gemfile.lock, so comparing one sha256 covers all three. That is why the
+  # plugin never has to work out which command ran.
+  module Lockfile
+    FILENAME = "Gemfile.lock"
+
+    module_function
+
+    def path(dir = Dir.pwd)
+      File.join(dir, FILENAME)
+    end
+
+    def exist?(dir = Dir.pwd)
+      File.file?(path(dir))
+    end
+
+    # nil when there is no lockfile, which callers must treat as "unknown"
+    # rather than "unchanged" — otherwise a project without a lockfile would
+    # silently report an up-to-date index forever.
+    def digest(dir = Dir.pwd)
+      file = path(dir)
+      return nil unless File.file?(file)
+
+      Digest::SHA256.file(file).hexdigest
+    end
+  end
+end

data/lib/gemchat/manifest.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Gemchat
+  # `.gemchat.yml` — the per-project record of what the index was built from.
+  #
+  # This exists so the staleness check can answer "is anything to do?" without
+  # opening SQLite or reading ri. A plugin hook fires on every bundle install in
+  # a project, and the overwhelming majority of those change nothing; the digest
+  # is a 64-byte file read that keeps them free.
+  #
+  # It is also checked-in code-adjacent state, so it is covered by the same
+  # trust gate as the plugin itself (§17): a manifest arriving in a repository
+  # you cloned is not evidence of anything.
+  class Manifest
+    FILENAME = ".gemchat.yml"
+
+    attr_reader :dir, :lockfile_sha256, :gems, :hosted
+
+    def initialize(dir = Dir.pwd)
+      @dir = dir
+      @lockfile_sha256 = nil
+      @gems = {}
+      @hosted = false
+      load
+    end
+
+    def self.path(dir = Dir.pwd)
+      File.join(dir, FILENAME)
+    end
+
+    def exist?
+      File.file?(self.class.path(dir))
+    end
+
+    def load
+      file = self.class.path(dir)
+      return self unless File.file?(file)
+
+      data = YAML.safe_load_file(file, permitted_classes: [Symbol], aliases: true)
+      return self unless data.is_a?(Hash)
+
+      @lockfile_sha256 = data["lockfile_sha256"]
+      @gems = data["gems"].is_a?(Hash) ? data["gems"] : {}
+      @hosted = data["hosted"] == true
+      self
+    rescue
+      # A corrupt manifest must not be fatal. Treating it as absent means the
+      # next run re-indexes, which is slower but correct; refusing to start
+      # would leave the project permanently un-indexed.
+      @lockfile_sha256 = nil
+      @gems = {}
+      @hosted = false
+      self
+    end
+
+    # True only when the lockfile is byte-identical to the one the index was
+    # built from. A missing manifest, a missing lockfile, or a corrupt manifest
+    # all mean "do the work".
+    def up_to_date?(current_digest)
+      return false if current_digest.nil?
+      return false if lockfile_sha256.nil?
+
+      lockfile_sha256 == current_digest
+    end
+
+    def record(lockfile_digest, gem_versions = {})
+      @lockfile_sha256 = lockfile_digest
+      @gems = gem_versions
+      self
+    end
+
+    def write
+      File.write(self.class.path(dir), to_yaml)
+      self
+    end
+
+    def to_yaml
+      data = {"lockfile_sha256" => lockfile_sha256, "gems" => gems}
+      # Only written when true. A `hosted: false` in every project would churn
+      # every .gemchat.yml in the world to record a default nobody set.
+      data["hosted"] = true if @hosted
+      data.to_yaml
+    end
+
+    # The project-level default set by `gemchat init --hosted`. Separate from
+    # GEMCHAT_HOST and from a key being present: those mean the tier is
+    # *available*, this means this project asked for it.
+    def hosted=(value)
+      @hosted = value == true
+    end
+
+    def hosted? = @hosted
+
+    def version_for(name)
+      gems.dig(name, "version")
+    end
+  end
+end

data/lib/gemchat/markdown.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Gemchat
+  # Splits a Markdown or RDoc document into heading-scoped sections.
+  #
+  # Heading scope is the right unit for prose, the same way a method is the right
+  # unit for ri. A README's value is concentrated in its sections — "how to
+  # configure the thread pool" lives under a heading, not smeared across the
+  # file — and a chunk that carries its heading path can be found by the words
+  # that heading uses.
+  #
+  # Handles all three heading styles found in gem READMEs: ATX (`## Title`),
+  # setext (a title underlined with `===`), and RDoc (`== Title`).
+  module Markdown
+    Section = Struct.new(:path, :body) do
+      # Breadcrumb, e.g. "Configuration > Rack handlers".
+      def title
+        path.join(" > ")
+      end
+    end
+
+    ATX = /\A(\#{1,6})\s+(.+?)\s*\#*\s*\z/
+    RDOC = /\A(={1,6})\s+(.+?)\s*\z/
+    SETEXT = /\A(={3,}|-{3,})\s*\z/
+    FENCE = /\A\s*(```+|~~~+)/
+
+    # A section under construction: the heading path plus the lines seen so far.
+    Pending = Struct.new(:path, :lines)
+
+    module_function
+
+    def sections(text)
+      finished = []
+      pending = nil
+      stack = []
+      previous = nil
+      fenced = false
+
+      text.to_s.each_line do |line|
+        if line.match?(FENCE)
+          # Inside a fence a `#` is code, not a heading. Splitting a section at a
+          # comment inside an example is both wrong and confusing: the example
+          # and the prose introducing it become separate chunks.
+          fenced = !fenced
+          pending&.lines&.<< line
+          previous = line
+          next
+        end
+
+        if fenced
+          pending&.lines&.<< line
+          previous = line
+          next
+        end
+
+        # A setext underline applies to the line *before* it, so that line has to
+        # be pulled back out of the body it was already collected into.
+        if setext?(line, previous)
+          pending&.lines&.pop
+          finished, pending, stack = flush(finished, pending, stack,
+            [line.start_with?("=") ? 1 : 2, previous.strip])
+        elsif (heading = heading_for(line))
+          finished, pending, stack = flush(finished, pending, stack, heading)
+        elsif pending
+          pending.lines << line
+        end
+
+        previous = line
+      end
+
+      finished << pending if pending
+      finished.filter_map { |p| to_section(p) }
+    end
+
+    def heading_for(line)
+      if (m = line.match(ATX))
+        [m[1].length, m[2].strip]
+      elsif (m = line.match(RDOC))
+        [m[1].length, m[2].strip]
+      end
+    end
+
+    def setext?(line, previous)
+      line.match?(SETEXT) && previous && !previous.strip.empty? &&
+        !previous.start_with?("#", "=")
+    end
+
+    def flush(finished, pending, stack, heading)
+      finished << pending if pending
+      level, title = heading
+      stack.pop while stack.any? && stack.last[0] >= level
+      stack.push([level, title])
+      [finished, Pending.new(stack.map(&:last), []), stack]
+    end
+
+    def to_section(pending)
+      body = pending.lines.join.strip
+      return nil if pending.path.empty? && body.empty?
+
+      Section.new(pending.path, body)
+    end
+  end
+end

data/lib/gemchat/models.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "net/http"
+require "uri"
+
+module Gemchat
+  # The embedder model: where it comes from, how it is fetched, and how it is
+  # verified.
+  #
+  # The plan originally specified `embeddinggemma-300M-Q8_0` as the default. That
+  # model is **gated: manual** on HuggingFace, so fetching it needs a license
+  # click and a personal token. An unattended `gemchat embed` cannot do that, and
+  # a default that fails for every user is not a default. `nomic-embed-text-v1.5`
+  # is ungated, Apache-2.0, 768-dimensional (matching gemchat_app's vectors, so
+  # the two agree on what a good retrieval looks like), and its Q8_0 build is
+  # 146MB rather than 300MB.
+  module Models
+    REGISTRY = {
+      "nomic-embed-text-v1.5" => {
+        file: "nomic-embed-text-v1.5.Q8_0.gguf",
+        url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q8_0.gguf",
+        # Measured, not assumed: the sha256 of the bytes this build actually
+        # serves. A pinned digest is what makes "checksum verification" in the
+        # plan mean something.
+        sha256: "3e24342164b3d94991ba9692fdc0dd08e3fd7362e0aacc396a9a5c54a544c3b7",
+        bytes: 146_146_432,
+        dimensions: 768,
+        license: "Apache-2.0",
+        # Nomic models are trained with task prefixes. Without them the query and
+        # document embeddings sit in slightly different regions and retrieval
+        # degrades: measured on this corpus, the correct hit's similarity rose
+        # from 0.515 to 0.580 once both prefixes were applied.
+        #
+        # Modelled per model rather than hardcoded, because a different default
+        # would have its own convention or none.
+        query_prefix: "search_query: ",
+        document_prefix: "search_document: "
+      }
+    }.freeze
+
+    DEFAULT = "nomic-embed-text-v1.5"
+
+    class Error < Gemchat::Error; end
+
+    module_function
+
+    def default
+      ENV["GEMCHAT_EMBED_MODEL"] || DEFAULT
+    end
+
+    def spec(name = default)
+      REGISTRY.fetch(name) { raise Error, "unknown embedding model: #{name.inspect}" }
+    end
+
+    def dir
+      File.join(Gemchat.root, "models")
+    end
+
+    def path(name = default)
+      File.join(dir, spec(name)[:file])
+    end
+
+    # Identifies vectors in the store. Versioned, because swapping the file
+    # behind the same name would silently mix incompatible vectors. The prefixes
+    # are part of the identity for the same reason: changing them changes what
+    # the vectors mean, so an existing index must be rebuilt, not extended.
+    def identity(name = default)
+      "#{name}@q8_0+prefixed"
+    end
+
+    def prefix(kind, name = default)
+      info = spec(name)
+      (kind.to_sym == :query) ? info[:query_prefix].to_s : info[:document_prefix].to_s
+    end
+
+    # Size only, not the digest: SHA-256 over 146MB costs real time and this is
+    # called on every `status` and every `search`. `download` verifies the digest
+    # before the file is ever moved into place, so a wrong file cannot be
+    # installed by this path -- only by something writing there directly.
+    def installed?(name = default)
+      File.file?(path(name)) && size_of(name) == spec(name)[:bytes]
+    end
+
+    def size_of(name = default)
+      File.size(path(name))
+    rescue Errno::ENOENT
+      nil
+    end
+
+    # Explains what to do when the model is absent. Returns true when it is
+    # there. This never blocks a download: `gemchat embed` downloads on demand,
+    # and the only thing standing between a missing model and a fetched one is
+    # this line, so anything that treats a false return as "give up" silently
+    # disables embedding forever.
+    def present(name = default)
+      return true if installed?(name)
+
+      warn <<~MSG
+        #{name} is not downloaded yet (#{megabytes(name)}MB).
+
+          gemchat embed     # downloads it, then embeds the index
+          gemchat status    # what is present
+
+        Until then, `gemchat search` uses BM25 only, which works offline with no
+        model at all. It finds exact identifiers well; it will not match a
+        question whose words never appear in the documentation.
+      MSG
+      false
+    end
+
+    def megabytes(name = default)
+      (spec(name)[:bytes] / 1_000_000.0).round
+    end
+
+    # Streams to a .part file and renames on success, so an interrupted download
+    # can never leave a truncated file that later looks installed. Resumes with a
+    # Range request when a partial file is already there.
+    def download(name = default, stdout: $stdout, force: false)
+      info = spec(name)
+      target = path(name)
+      partial = "#{target}.part"
+
+      FileUtils.rm_f(target) if force && File.exist?(target)
+      return target if installed?(name)
+
+      FileUtils.mkdir_p(dir)
+      resume_from = File.exist?(partial) ? File.size(partial) : 0
+      if resume_from.positive? && resume_from >= info[:bytes]
+        FileUtils.rm_f(partial)
+        resume_from = 0
+      end
+
+      uri = URI.parse(info[:url])
+      downloaded = resume_from
+      stream(uri, info, partial, resume_from) do |_chunk, total|
+        downloaded = total
+        report_progress(stdout, downloaded, info[:bytes])
+      end
+
+      unless File.size(partial) == info[:bytes]
+        raise Error, "incomplete download: got #{File.size(partial)} of #{info[:bytes]} bytes (kept #{File.basename(partial)} to resume)"
+      end
+
+      actual = Digest::SHA256.file(partial).hexdigest
+      unless actual == info[:sha256]
+        FileUtils.rm_f(partial)
+        raise Error, "checksum mismatch for #{info[:file]}\n  expected #{info[:sha256]}\n  actual   #{actual}"
+      end
+
+      FileUtils.mv(partial, target)
+      stdout.puts("  verified sha256 #{actual[0, 16]}…")
+      target
+    end
+
+    # Streams the body into `partial`, following redirects.
+    #
+    # Following them is not optional. HuggingFace answers a model URL with a 302
+    # to a CDN host, and Net::HTTP does not follow redirects, so a request that
+    # only accepts Net::HTTPSuccess fails on the very first response -- reported
+    # as "HTTP 302", which reads like a broken link rather than a normal,
+    # expected part of the fetch.
+    #
+    # The Authorization header is dropped once a redirect crosses to another
+    # host: the CDN does not need it, and resending a token to a host the user
+    # did not name is not something to do by accident.
+    def stream(uri, info, partial, resume_from, max_hops = 5)
+      File.open(partial, resume_from.positive? ? "ab" : "wb") do |file|
+        total = resume_from
+
+        loop do
+          request = Net::HTTP::Get.new(uri)
+          request["Range"] = "bytes=#{resume_from}-" if resume_from.positive?
+          request["Authorization"] = "Bearer #{ENV["HF_TOKEN"]}" if token?
+
+          redirect = nil
+
+          # A connection per hop, never one connection reused across a redirect.
+          # The redirect target is a different host, and replaying a request for
+          # a second host down the first host's TLS session is what turns a
+          # perfectly good signed CDN URL into a 403.
+          Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+            http.request(request) do |res|
+              if res.is_a?(Net::HTTPRedirection) && res["location"]
+                redirect = URI.join(uri, res["location"])
+                next
+              end
+
+              unless res.is_a?(Net::HTTPSuccess)
+                raise Error, "download failed: HTTP #{res.code} for #{info[:file]}"
+              end
+
+              # A server that ignores Range answers 200 with the whole file, so
+              # appending to an existing partial would corrupt it. Rewind instead.
+              if resume_from.positive? && res.code == "200"
+                file.truncate(0)
+                file.rewind
+                total = 0
+              end
+
+              res.read_body do |chunk|
+                file.write(chunk)
+                total += chunk.bytesize
+                yield(chunk, total)
+              end
+            end
+          end
+
+          break if redirect.nil?
+
+          max_hops -= 1
+          if max_hops <= 0
+            raise Error, "too many redirects fetching #{info[:file]}"
+          end
+
+          uri = redirect
+        end
+      end
+    end
+
+    # Only the origin gets the token. The CDN URL is already signed, and
+    # resending a bearer token to a host the user never named is not something
+    # to do by accident.
+    def token?
+      ENV["HF_TOKEN"] && !ENV["HF_TOKEN"].empty?
+    end
+
+    def report_progress(io, done, total)
+      return unless io.tty?
+
+      pct = (done * 100.0 / total).round
+      io.print("\r  #{megabytes_of(done)}/#{megabytes_of(total)}MB #{pct}%")
+      io.flush
+    end
+
+    def megabytes_of(bytes)
+      (bytes / 1_000_000.0).round(1)
+    end
+  end
+end