gem-skill 0.1.3 → 0.2.2

data/lib/gem/skill/verifier.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+require "ruby_llm/providers/lms"
+require "ruby_llm/providers/apfel"
+
+module Gem::Skill
+  # Second-pass quality gate for a generated SKILL.md.
+  #
+  # Generation synthesizes prose sources (README, changelog, examples) which are
+  # frequently wrong or stale about exact signatures. The verifier re-checks the
+  # generated skill against the gem's ACTUAL source code — the only source of
+  # truth — and corrects mismatched method signatures, default argument values,
+  # visibility, return values, and behavioral claims.
+  #
+  # Whether the skill actually changed is decided by a deterministic diff of the
+  # content before and after, not by trusting the model's self-report, so callers
+  # can rely on #changed? for an exit code and a "fixed" flag.
+  class Verifier
+    BEGIN_MARK = "===BEGIN SKILL==="
+    END_MARK   = "===END SKILL==="
+
+    SYSTEM_INSTRUCTIONS = <<~SYSTEM
+      You verify a generated Claude Code SKILL.md for a Ruby gem against the gem's
+      ACTUAL SOURCE CODE. The source code is the only source of truth. READMEs,
+      changelogs, and docstrings are frequently stale or wrong; when the SKILL.md
+      disagrees with the source, the source always wins.
+
+      Check every concrete claim against the source: method signatures, default
+      argument values, keyword vs positional arguments, public/private/protected
+      visibility, return values, constant and class/module names, default option
+      values, and described runtime behavior (including what arguments a yielded
+      block actually receives). Correct anything the source contradicts.
+
+      Rules:
+      - Do NOT invent APIs, methods, or options that are absent from the source.
+      - Do NOT restructure, re-style, or "improve" content that is already correct.
+        Preserve correct text verbatim so the diff stays minimal.
+      - Only change what the source proves is wrong.
+    SYSTEM
+
+    PROMPT = <<~PROMPT
+      Verify the SKILL.md below for "%<gem_name>s" v%<version>s against the gem's
+      source code. Correct every claim the source contradicts.
+
+      Output ONLY the full corrected SKILL.md in raw Markdown (even if you change
+      nothing), wrapped exactly between these marker lines and with no other text:
+      %<begin_mark>s
+      <corrected SKILL.md here>
+      %<end_mark>s
+
+      ============================================================
+      CURRENT SKILL.md
+      ============================================================
+
+      %<skill>s
+
+      ============================================================
+      GEM SOURCE CODE (ground truth)
+      ============================================================
+
+      %<source>s
+    PROMPT
+
+    # content:    the (possibly corrected) skill markdown
+    # changed:    true iff content differs from the original (diff-based)
+    # verifiable: false when no source was available to check against
+    # model:      the model used for verification
+    Result = Data.define(:content, :changed, :verifiable, :model) do
+      def changed? = changed
+    end
+
+    attr_reader :gem_name, :version, :model
+
+    def initialize(gem_name, version, model: Generator::DEFAULT_MODEL)
+      @gem_name = gem_name
+      @version  = version
+      @model    = model
+    end
+
+    # Verify skill_content against the gem source. Returns a Result.
+    def verify(skill_content)
+      fetcher = Fetcher.new(gem_name, version)
+      source  = fetcher.source_code
+      if source.nil? || source.strip.empty?
+        return Result.new(content: skill_content, changed: false, verifiable: false, model: model)
+      end
+
+      raw       = build_chat.ask(format_prompt(skill_content, source)).content.to_s
+      # Re-apply frontmatter to both sides so the diff compares like-for-like and
+      # the stored skill always keeps valid frontmatter, even if the model dropped it.
+      original  = Frontmatter.build(gem_name, version, skill_content)
+      corrected = Frontmatter.build(gem_name, version, extract_skill(raw, skill_content))
+      changed   = normalize(corrected) != normalize(original)
+
+      Result.new(content: (changed ? corrected : skill_content), changed: changed,
+                 verifiable: true, model: model)
+    rescue RubyLLM::Error => e
+      raise Error, e.message
+    end
+
+    private
+
+    def build_chat
+      model_id, provider = Gem::Skill.parse_model(model)
+      RubyLLM.chat(model: model_id, provider: provider).with_instructions(SYSTEM_INSTRUCTIONS)
+    end
+
+    def format_prompt(skill_content, source)
+      format(
+        PROMPT,
+        gem_name:   gem_name,
+        version:    version,
+        begin_mark: BEGIN_MARK,
+        end_mark:   END_MARK,
+        skill:      skill_content,
+        source:     source
+      )
+    end
+
+    # Pull the corrected skill out from between the markers. If the model didn't
+    # honor the protocol, fall back to the original so we never corrupt the cache.
+    def extract_skill(raw, fallback)
+      start = raw.index(BEGIN_MARK)
+      return fallback unless start
+
+      body = raw[(start + BEGIN_MARK.length)..]
+      stop = body.index(END_MARK)
+      body = body[0...stop] if stop
+
+      body = body.to_s.strip
+      body.empty? ? fallback : body
+    end
+
+    def normalize(text)
+      text.to_s.gsub(/[ \t]+$/, "").strip
+    end
+  end
+end

data/lib/gem/skill/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Gem::Skill
-  VERSION = "0.1.3"
+  VERSION = "0.2.2"
 end

data/lib/gem/skill.rb

@@ -3,7 +3,9 @@
 require_relative "skill/version"
 require_relative "skill/cache"
 require_relative "skill/fetcher"
+require_relative "skill/frontmatter"
 require_relative "skill/generator"
+require_relative "skill/verifier"
 require_relative "skill/linker"
 require_relative "skill/lockfile"
 require_relative "skill/runner"
@@ -11,6 +13,16 @@ require_relative "skill/runner"
 module Gem::Skill
   class Error < StandardError; end
 
+  # Exit status used when --verify found and corrected mistakes in a generated
+  # skill (grep-style: 0 = clean, 1 = error, 2 = verify applied fixes).
+  EXIT_VERIFY_FIXED = 2
+
+  # The bundled "router" skill that teaches assistants how to find cached gem
+  # skills in ~/.gem/skills. `gem skill setup` copies it into the assistants'
+  # default skill roots. Lives at the repo/gem root, beside README/CHANGELOG.
+  ROUTER_SKILL_NAME = "ruby-gem-skills"
+  ROUTER_SKILL_DIR  = File.expand_path("../../#{ROUTER_SKILL_NAME}", __dir__)
+
   ENV_KEY_MAP = {
     anthropic_api_key:  "ANTHROPIC_API_KEY",
     openai_api_key:     "OPENAI_API_KEY",
@@ -18,14 +30,37 @@ module Gem::Skill
     mistral_api_key:    "MISTRAL_API_KEY",
     deepseek_api_key:   "DEEPSEEK_API_KEY",
     openrouter_api_key: "OPENROUTER_API_KEY",
-    xai_api_key:        "XAI_API_KEY"
+    xai_api_key:        "XAI_API_KEY",
+    lms_api_base:       "LMS_API_BASE",
+    lms_api_key:        "LMS_API_KEY",
+    apfel_api_base:     "APFEL_API_BASE",
+    apfel_api_key:      "APFEL_API_KEY"
   }.freeze
 
+  # Split a "provider/model" string into [model_id, provider_symbol] when the
+  # prefix names a registered RubyLLM provider, e.g.
+  #   "lms/qwen/qwen3.8-27b" -> ["qwen/qwen3.8-27b", :lms]
+  # Otherwise the string is a bare model id: ["gpt-5.5", nil]. A bare id is
+  # resolved by RubyLLM's own provider preference, so ids that contain a "/"
+  # but don't start with a provider slug (e.g. "qwen/...") pass through intact.
+  def self.parse_model(model_string)
+    prefix, rest = model_string.to_s.split("/", 2)
+    return [model_string, nil] unless rest && RubyLLM::Provider.providers.key?(prefix.to_sym)
+
+    [rest, prefix.to_sym]
+  end
+
+  # HTTP read timeout in seconds (GEMSKILL_REQUEST_TIMEOUT to override).
+  # RubyLLM's default of 300 is tuned for hosted APIs; a local model (lms/apfel)
+  # can legitimately take longer than that to finish one skill generation.
+  REQUEST_TIMEOUT = ENV.fetch("GEMSKILL_REQUEST_TIMEOUT", 900).to_i
+
   # Configure RubyLLM from environment variables. Called automatically by the
   # CLI commands so users don't need a separate initializer for standalone use.
   # No-op if RubyLLM is already configured (e.g. in a Rails app).
   def self.configure_llm!
     RubyLLM.configure do |config|
+      config.request_timeout = REQUEST_TIMEOUT
       ENV_KEY_MAP.each do |attr, env_var|
         value = ENV[env_var]
         config.public_send(:"#{attr}=", value) if value

data/ruby-gem-skills/SKILL.md

@@ -0,0 +1,53 @@
+---
+name: ruby-gem-skills
+description: "Locate and load gem-skill's cached, version-specific SKILL.md for a Ruby gem. Use whenever working with, debugging, or writing Ruby code that uses a third-party gem (Bundler/Gemfile projects or installed gems) to get accurate, version-pinned API knowledge before relying on memory."
+---
+
+# ruby-gem-skills
+
+Detailed, version-specific knowledge for installed Ruby gems is generated by the
+`gem-skill` tool and cached on this machine at:
+
+```
+~/.gem/skills/<gem_name>/<version>/SKILL.md
+```
+
+(Use `$GEMSKILL_DIR` instead of `~/.gem/skills` if that variable is set.)
+
+Most assistants do not scan that directory by default, so follow this procedure
+to find and load a gem's skill before answering from memory.
+
+## When to use
+
+- Writing or debugging Ruby code that calls a third-party gem's API.
+- Verifying exact method signatures, options, defaults, or behavior for a
+  specific gem version.
+
+## How to load a gem's skill
+
+1. Determine the gem version in use:
+   - In a project with a `Gemfile.lock`, read the locked version for the gem.
+   - Otherwise run `gem list <gem_name>` (or `bundle show <gem_name>`) to find
+     the installed version.
+2. Read the cached skill for that exact version:
+   ```
+   ~/.gem/skills/<gem_name>/<version>/SKILL.md
+   ```
+3. If it exists, treat its contents as the authoritative, version-pinned
+   reference for that gem and prefer it over training memory. Skills are
+   version-specific — always match the version actually in use.
+
+## If the skill is missing
+
+Tell the user how to generate it:
+
+- One gem: `gem skill install <gem_name>`
+- Every gem in a project (from the project root): `bundle skill install`
+- Verify an existing skill against the gem's real source: `gem skill verify <gem_name>`
+
+## Notes
+
+- A `metadata.json` sits beside each `SKILL.md` with provenance (model, sources,
+  and verification status). It is for tooling and humans, not required reading.
+- A green checkmark next to a version in `gem skill list` means that skill was
+  verified against the gem's actual source code.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gem-skill
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.2
 platform: ruby
 authors:
 - Dewayne VanHoozer
@@ -29,14 +29,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: ruby_llm-providers-apfel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby_llm-providers-lms
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
@@ -65,6 +93,7 @@ files:
 - ".envrc"
 - ".github/workflows/deploy-github-pages.yml"
 - CHANGELOG.md
+- CLAUDE.md
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -82,14 +111,17 @@ files:
 - lib/gem/skill/cli/bundle_command.rb
 - lib/gem/skill/cli/gem_command.rb
 - lib/gem/skill/fetcher.rb
+- lib/gem/skill/frontmatter.rb
 - lib/gem/skill/generator.rb
 - lib/gem/skill/linker.rb
 - lib/gem/skill/lockfile.rb
 - lib/gem/skill/runner.rb
+- lib/gem/skill/verifier.rb
 - lib/gem/skill/version.rb
 - lib/rubygems_plugin.rb
 - mkdocs.yml
 - plugins.rb
+- ruby-gem-skills/SKILL.md
 - scripts/e2e_test
 - sig/gem/skill.rbs
 homepage: https://github.com/madbomber/gem-skill
@@ -117,14 +149,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: '3.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.14
+rubygems_version: 4.0.21
 specification_version: 4
 summary: Generate and manage Claude Code AI skills from Ruby gem documentation.
 test_files: []