gemchat 0.1.0

data/lib/gemchat/symbols.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require "prism"
+require "pathname"
+
+module Gemchat
+  # The sole source of file:line and signature metadata, per §11 of the plan.
+  # ri tells us a class and a method; it does not tell us which file the method is
+  # defined in, or on which line. That is the information an agent cannot recover
+  # from the result itself, and it is what a citation needs.
+  #
+  # Ported from the app's `RubySourceDefinitionService`, minus Rails: no
+  # `Rails.logger`, no ActiveSupport's `blank?`/`presence`, no `filter_map` on
+  # an Array that is sometimes a Hash. The visitor logic is unchanged, including
+  # its known imprecision -- a method defined inside `extend SomeModule` is
+  # reported under the lexical owner rather than the mixin. §4.4 argues that
+  # reporting the lexical owner is arguably *more* precise, since it says the
+  # method comes from a mixin.
+  class Symbols
+    # The same caps the app uses. A generated or vendored tree can contain tens of
+    # thousands of .rb files, and this pass runs on every indexed gem.
+    MAX_FILES = 2000
+    MAX_FILE_BYTES = 1_000_000
+
+    Definition = Struct.new(
+      :class_name, :method_name, :method_type, :signature, :source_path, :source_line
+    ) do
+      # Looked up against an ri chunk, which knows class/method/method_type but
+      # not a file. A missing owner on either side means the chunk is a class-level
+      # or top-level definition and there is nothing to match.
+      def key
+        [class_name.to_s, method_name.to_s, method_type.to_s]
+      end
+    end
+
+    # Definitions for every .rb under `dir`, keyed by Definition#key.
+    def self.for(dir)
+      new(dir).index
+    end
+
+    def initialize(dir)
+      @dir = dir.to_s
+    end
+
+    def index
+      return {} unless File.directory?(@dir)
+
+      files = Dir.glob(File.join(@dir, "**", "*.rb")).sort.first(MAX_FILES)
+      defs = files.flat_map { |path| parse_file(path) }
+
+      # First definition wins. A gem that reopens a class in several files has
+      # several definitions for one key, and the first in sorted order is the
+      # most predictable choice. Duplicates are counted so the caller can report
+      # an ambiguous match rather than pretending there was only one.
+      defs.group_by(&:key).transform_values(&:first).tap { |out| @collisions = defs.size - out.size }
+    end
+
+    def collisions
+      @collisions.to_i
+    end
+
+    private
+
+    def parse_file(path)
+      return [] if File.size(path) > MAX_FILE_BYTES
+
+      source = File.read(path, encoding: Encoding::UTF_8)
+      result = Prism.parse(source)
+      relative = Pathname.new(path).relative_path_from(Pathname.new(@dir)).to_s
+      Visitor.new(relative).tap { |v| v.visit(result.value) }.definitions
+    rescue SystemCallError, ArgumentError, Prism::ParseError => e
+      # One unparseable or unreadable file must not cost the whole gem its source
+      # locations. A gem with a syntax error in one file still has thousands of
+      # parseable ones.
+      warn "  symbols: skipping #{File.basename(path)} (#{e.class})" if ENV["GEMCHAT_DEBUG"]
+      []
+    end
+
+    # Tracks class/module nesting and `class << self` scope.
+    class Visitor < Prism::Visitor
+      attr_reader :definitions
+
+      def initialize(source_path)
+        @source_path = source_path
+        @definitions = []
+        @namespace = []
+        @singleton_depth = 0
+      end
+
+      def visit_module_node(node)
+        with_namespace(constant_name(node.constant_path)) { visit_child_nodes(node) }
+      end
+
+      def visit_class_node(node)
+        with_namespace(constant_name(node.constant_path)) do
+          # `class << self` opens a singleton scope for the rest of the body.
+          if node.body.is_a?(Prism::SingletonClassNode)
+            @singleton_depth += 1
+            begin
+              visit_child_nodes(node.body)
+            ensure
+              @singleton_depth -= 1
+            end
+          else
+            visit_child_nodes(node)
+          end
+        end
+      end
+
+      def visit_singleton_class_node(node)
+        @singleton_depth += 1
+        begin
+          visit_child_nodes(node)
+        ensure
+          @singleton_depth -= 1
+        end
+      end
+
+      def visit_def_node(node)
+        @definitions << Definition.new(
+          current_owner, node.name.to_s,
+          class_method?(node) ? "class" : "instance",
+          signature_for(node), @source_path, node.location.start_line
+        )
+        visit_child_nodes(node)
+      end
+
+      private
+
+      def with_namespace(name)
+        @namespace.push(name)
+        yield
+      ensure
+        @namespace.pop
+      end
+
+      def current_owner
+        full = @namespace.reject { |n| n.to_s.strip.empty? }.join("::")
+        full.empty? ? nil : full
+      end
+
+      def class_method?(node)
+        return true if @singleton_depth.positive?
+
+        receiver = node.receiver
+        receiver.is_a?(Prism::SelfNode) || receiver.is_a?(Prism::ConstantReadNode)
+      end
+
+      def signature_for(node)
+        params = node.parameters&.location&.slice || ""
+        "#{node.name}(#{params})"
+      end
+
+      def constant_name(node)
+        node&.slice || ""
+      end
+    end
+  end
+end

data/lib/gemchat/trust.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require "fileutils"
+require_relative "paths"
+
+module Gemchat
+  # Which projects may run the plugin's index hook on their own.
+  #
+  # A Bundler plugin is checked-in code that executes for anyone who clones the
+  # repository. Running `gemchat index` is much less dangerous than arbitrary
+  # code, but it still means one machine's bundle install indexes a path derived
+  # from someone else's Gemfile, and it writes to the invoking user's home
+  # directory. So the hook asks first.
+  #
+  # The gate is deliberately low-friction for a project's own author: the first
+  # *explicit* gemchat command run inside a project grants trust for that
+  # project. Someone who just added `plugin "gemchat"` to their own Gemfile gets
+  # auto-indexing with no ceremony. Someone who cloned that repository does not,
+  # because they have not run a gemchat command.
+  #
+  # Records live in GEMCHAT_CONFIG_HOME, not the data root, so wiping the index
+  # does not also discard these decisions.
+  class Trust
+    FILENAME = "trusted.json"
+
+    class << self
+      def path
+        File.join(Paths.config_home, FILENAME)
+      end
+
+      def all
+        return {} unless File.file?(path)
+
+        data = JSON.parse(File.read(path))
+        data.is_a?(Hash) ? data : {}
+      rescue SystemCallError, JSON::ParserError
+        # Unreadable or corrupt trust file means "nothing is trusted", which fails
+        # closed. Deliberately narrow: a bare rescue also swallowed a NoMethodError
+        # from a missing require and reported every project as untrusted, which
+        # looks exactly like a working gate and is very hard to see.
+        {}
+      end
+
+      def trusted?(dir = Dir.pwd)
+        all.key?(expand(dir))
+      end
+
+      def grant!(dir = Dir.pwd, via: "explicit command")
+        records = all
+        records[expand(dir)] = {"granted_at" => Time.now.utc.iso8601, "via" => via}
+        write(records)
+        records[expand(dir)]
+      end
+
+      def revoke!(dir = Dir.pwd)
+        records = all
+        removed = records.delete(expand(dir))
+        write(records) if removed
+        !removed.nil?
+      end
+
+      def write(records)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, JSON.pretty_generate(records))
+      end
+
+      # Project identity is the absolute, symlink-resolved path. Resolving
+      # matters: trusting /tmp/x and then arriving at /private/tmp/x on macOS
+      # must be the same project, or trust silently never matches.
+      def expand(dir)
+        File.realpath(dir)
+      rescue SystemCallError
+        File.expand_path(dir)
+      end
+    end
+  end
+end

data/lib/gemchat/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Gemchat
+  VERSION = "0.1.0"
+end

data/lib/gemchat.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "time"
+require "rbconfig"
+require "rdoc"
+require "rdoc/ri/driver"
+require "open3"
+
+require_relative "gemchat/version"
+require_relative "gemchat/errors"
+require_relative "gemchat/paths"
+require_relative "gemchat/env"
+require_relative "gemchat/ri"
+require_relative "gemchat/symbols"
+require_relative "gemchat/chunker"
+require_relative "gemchat/markdown"
+require_relative "gemchat/prose"
+require_relative "gemchat/stopwords"
+require_relative "gemchat/hosted"
+require_relative "gemchat/models"
+require_relative "gemchat/embedder"
+require_relative "gemchat/store"
+require_relative "gemchat/lockfile"
+require_relative "gemchat/manifest"
+require_relative "gemchat/trust"
+require_relative "gemchat/hook"
+require_relative "gemchat/plugin_index"
+require_relative "gemchat/init"
+require_relative "gemchat/indexer"
+require_relative "gemchat/cli"
+
+module Gemchat
+  def self.root
+    Paths.root
+  end
+
+  def self.config_home
+    Paths.config_home
+  end
+
+  def self.version
+    Gemchat::VERSION
+  end
+end

data/plugins.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# Bundler plugin entry point. This file executes on every `bundle install` for
+# anyone who clones a project that declares `plugin "gemchat"`, so it is treated
+# as untrusted input.
+#
+# Three rules, in priority order:
+#
+#   1. Never raise. A plugins.rb that raises is wrapped in MalformattedPlugin and
+#      fails the whole install (exit 29, no lockfile written). Nothing here may
+#      propagate.
+#   2. Carry no logic. Dual declaration installs two physical copies of gemchat
+#      and Bundler 4 writes no PLUGINS lockfile section, so nothing pins the
+#      plugin copy to the dependency copy. Requiring the *bundled* copy makes
+#      drift harmless: this file only locates and loads.
+#   3. Say one line, or nothing. The hook fires on CI and deploy installs too.
+#
+# This file lives in the gem, never in a consuming project. A project-root
+# plugins.rb is silently never executed.
+#
+# Every Bundler reference is root-scoped with `::`. Bundler loads this file with
+# `load(path, true)`, which wraps it in an anonymous module, and `register_plugin`
+# is mid-flight inside `module Bundler::Plugin` while this runs — so a bare
+# `Bundler::Plugin` here can resolve against the wrapper and raise NameError,
+# which then aborts the install. `::Bundler` cannot be shadowed that way.
+module Bundler
+  module Gemchat
+    HOST_GEM = "gemchat"
+    SUPPORTED = Gem::Requirement.new(">= 0.1.0")
+
+    class << self
+      def auto_index
+        # Delegates to the bundled copy. This file deliberately holds no logic:
+        # a dual `gem` + `plugin` declaration installs two physical copies and
+        # Bundler 4 pins neither to the other, so anything implemented here
+        # would exist twice and could disagree with the version the project
+        # actually depends on. See §15.4.
+        spec = host_spec
+        return report("#{HOST_GEM} is not in the bundle") unless spec
+        return report("skipped: outside supported range") unless SUPPORTED.satisfied_by?(spec.version)
+
+        lib = File.join(spec.full_gem_path, "lib")
+        $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+        require "gemchat/hook"
+        ::Gemchat::Hook.auto_index
+      rescue => e
+        report "auto-index skipped (#{e.class}: #{e.message})"
+      end
+
+      def host_spec
+        @host_spec ||= ::Bundler.load.specs.find { |s| s.name == HOST_GEM }
+      end
+
+      def report(message)
+        warn "[gemchat] #{message}"
+      end
+    end
+  end
+end
+
+# Registration, not execution. A plugins.rb that merely *runs* on load is
+# executed once, during the plugin phase of the very `bundle install` that
+# installed it, and never again: Bundler records which hooks a plugin
+# registered into .bundle/plugin/index, and `hook_plugins` gates every later
+# invocation on that record. Without this line the index shows `hooks:` empty
+# and `bundle add` / `bundle update` are silent no-ops despite the plugin being
+# installed and having printed output once already.
+#
+# The event name must be the string. `Events::GEM_AFTER_INSTALL_ALL` happens to
+# be that same String so it also works, but :after_install_all raises
+# ArgumentError, which register_plugin rewraps as MalformattedPlugin.
+::Bundler::Plugin.add_hook("after-install-all") do
+  Bundler::Gemchat.auto_index
+end

data/sig/gemchat.rbs

@@ -0,0 +1,4 @@
+module Gemchat
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

data/skills/gemchat/SKILL.md

@@ -0,0 +1,158 @@
+---
+name: gemchat
+description: Answer questions about the gems in this project's bundle — their API, configuration, and version-specific behaviour — from a local, offline index. Use when asked how a bundled gem works, what a method does, or how to use a library already in the Gemfile. Triggers on gem names, "how do I use X", method or class names from a dependency, and error messages from a gem.
+---
+
+# gemchat
+
+`gemchat` indexes the documentation of the gems in **this project's own
+`Gemfile.lock`** and answers questions about them offline. It is a local tool: no
+account, no API key, nothing leaves the machine.
+
+## Reach for this when
+
+- Someone asks how a gem **in this bundle** works, or what a method does.
+- An error message came from a dependency and you need the surrounding API.
+- You need the signature or the configuration options of something installed.
+
+## Do not reach for this when
+
+- The question is about **this application's** runtime — routes, schema, job
+  state, request logs. That is Rails Hyperdrive's job, not a documentation index.
+- The gem is **not in the bundle**. gemchat indexes `Gemfile.lock`; a gem you are
+  merely considering is not there.
+
+## Pick the right command
+
+This is the part worth getting right, because the two search commands fail in
+**opposite** directions and using the wrong one returns nothing at all.
+
+| Question shape | Command | Why |
+|---|---|---|
+| An exact identifier — `Rake::FileList#exclude`, an error string, a flag | `gemchat search` | Instant, no model needed, and exact. |
+| A natural-language question — "how do I list every task" | `gemchat query` | Needs vectors; use after `embed`. |
+| A paraphrase you want only the vector arm for | `gemchat vsearch` | Same requirement. |
+
+**`gemchat search` does not do keyword search.** It matches exact phrases. A
+question like *"how do I list all the available tasks"* returns **no results**,
+because those words do not appear together in any chunk. That is expected
+behaviour, not a broken index — reach for `query` instead.
+
+`gemchat query` takes a leading type that routes it:
+
+```sh
+gemchat query "how do I list every task"          # both engines, fused (default)
+gemchat query "lex: Rake::Task#enhance"           # BM25 only
+gemchat query "vec: stop a process on a signal"   # vectors only
+```
+
+`lex:` never loads a model, so it is the cheap choice for a symbol.
+
+## Before searching, run `gemchat status`
+
+It is one line each, and it tells you whether an answer is even possible:
+
+```
+index:     5 gems · 1095 chunks        # what is indexed
+manifest:  up to date                  # index matches the current lockfile
+vectors:   not downloaded (146MB) → gemchat embed
+hosted:    not configured → gemchat.org/settings
+```
+
+- `vectors: not downloaded` means `query` and `vsearch` **cannot answer yet**.
+  They do not download anything themselves; `gemchat embed` is the explicit step.
+- `manifest: stale` means the lockfile has moved since the last index. Re-run
+  `gemchat index`, or answers may be about a version that is no longer installed.
+
+## Cost
+
+- `gemchat search` — instant, free, no model on disk.
+- `gemchat embed` — one-time, downloads a 146MB model, then embeds the index.
+  Every later semantic search is local and fast.
+- If the model is not downloaded, `query` degrades to BM25 and says so on stderr.
+  It does not fail silently and it does not download behind your back.
+
+## Reading a result
+
+Each result names its source, and that changes how you should treat it:
+
+```
+ 1. [rake  13.4.2  ri  ] Rake::FileList#exclude
+    exclude(*patterns, &block)  (lib/rake/file_list.rb:150)
+```
+
+- `ri` is API reference generated from the installed gem's own source. The
+  `path:line` is the real definition site — open it before asserting behaviour.
+- `readme`, `guides` and `doc` are the gem's hand-written prose, split by
+  heading. Use these for *why* and *when*, not for signatures.
+
+If `path:line` is missing on an `ri` hit, the definition is not recoverable by
+parsing (a C extension, `define_method`, or a macro). Say so rather than
+guessing a file.
+
+## Version fidelity
+
+The index is built from `Gemfile.lock`, so local answers are about **the versions
+actually installed** — usually what you want. Check `manifest: up to date` before
+relying on that; if it is stale, the answer may describe a version that is no
+longer there.
+
+### Hosted backend
+
+`gemchat query --hosted` asks gemchat.org instead. It is opt-in and **never
+happens automatically** — there is no route from a failed or unconfigured hosted
+query to a local answer, so you will get an error rather than a local result you
+did not ask for. Do not retry it locally and present that as the answer.
+
+It sends this lockfile's exact gem versions, and the response says which ones it
+could scope to:
+
+```
+note:  (scoped to 13.4.2; no per-version index for sidekiq)
+```
+
+Report that note. A gem with no per-version index on the server is answered from
+its full docs, which may not be the pinned release — so a scoped note is fidelity,
+and an absent one is not.
+
+Two things that do not exist on the hosted side:
+
+- **`lex:` has no hosted equivalent.** It is BM25, which is local-only.
+  `gemchat query --hosted lex: …` refuses and points at `gemchat search`. Use
+  `gemchat search` for a symbol.
+- **`search` and `vsearch` are local-only**, whatever the backend is configured.
+
+If the server does not have a gem you named, the result says so and points at
+`--auto-index`, which asks gemchat.org to index it. That spends the caller's
+quota, so it is opt-in and the message is a suggestion, not something to pass
+through on your own.
+
+## Invocation
+
+Use `bundle exec gemchat` when gemchat is a development dependency in this
+project's Gemfile (which is what `gemchat init` sets up), so you get the project's
+bundle. Use bare `gemchat` when it is installed globally.
+
+## When the index is wrong
+
+- `gemchat reindex` discards the index and the generated ri, then builds it
+  again. Use it when `status` reports a stale manifest, or after upgrading
+  gemchat, whose store format is version-gated and will refuse to open a store
+  it did not write.
+- `gemchat version` prints the version, which is worth quoting when a search
+  behaves unexpectedly.
+
+## Setup
+
+`gemchat init` is the only command that modifies a Gemfile. It adds gemchat as a
+development dependency and a Bundler plugin, and grants trust for this project.
+After that, any `bundle install`, `bundle add` or `bundle update` that moves the
+lockfile re-indexes automatically and prints one line. Anything else is silent.
+
+`gemchat init --hosted` additionally records the hosted backend as this
+project's default, so later `gemchat query` calls use it without a flag. It still
+needs a key: `GEMCHAT_API_KEY`, or a `credentials` file under the gemchat config
+home. `gemchat status` says which of those is in play.
+
+If the project has no `Gemfile.lock`, or the gems are not installed, `index`
+cannot work — those gems have to be on disk to be documented.

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: gemchat
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Saroj Maharjan (zoras)
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rllama
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: prism
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: |
+  gemchat indexes the documentation for the gems in your project's own bundle and
+  serves BM25 search over it from the command line, with no account, no API key and
+  no network.
+
+  It is a local tool, not the hosted gemchat.org service. An optional mode can defer
+  to gemchat.org if you would rather not keep a local index.
+
+  Direct dependencies are indexed by default and ri documentation is generated
+  locally, because Bundler installs gems without documentation. Results carry the
+  exact locked version and real API signatures, which is the point: an agent can
+  look up how to call a library without reading a 1500-line lockfile into context.
+email:
+- zoras@users.noreply.github.com
+executables:
+- gemchat
+extensions: []
+extra_rdoc_files: []
+files:
+- AGENTS.md
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- docs/plans/gemchat-local-bundle-index.md
+- exe/gemchat
+- lib/gemchat.rb
+- lib/gemchat/chunker.rb
+- lib/gemchat/cli.rb
+- lib/gemchat/embedder.rb
+- lib/gemchat/env.rb
+- lib/gemchat/errors.rb
+- lib/gemchat/hook.rb
+- lib/gemchat/hosted.rb
+- lib/gemchat/indexer.rb
+- lib/gemchat/init.rb
+- lib/gemchat/lockfile.rb
+- lib/gemchat/manifest.rb
+- lib/gemchat/markdown.rb
+- lib/gemchat/models.rb
+- lib/gemchat/paths.rb
+- lib/gemchat/plugin_index.rb
+- lib/gemchat/prose.rb
+- lib/gemchat/ri.rb
+- lib/gemchat/stopwords.rb
+- lib/gemchat/store.rb
+- lib/gemchat/symbols.rb
+- lib/gemchat/trust.rb
+- lib/gemchat/version.rb
+- plugins.rb
+- sig/gemchat.rbs
+- skills/gemchat/SKILL.md
+homepage: https://github.com/zoras/gemchat
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/zoras/gemchat
+  source_code_uri: https://github.com/zoras/gemchat
+  changelog_uri: https://github.com/zoras/gemchat/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.22
+specification_version: 4
+summary: Local, offline documentation index for the gems in your own Gemfile.lock
+test_files: []