ruby_llm-skills 0.3.0 → 0.5.0.pre1

data/lib/ruby_llm/skills/marketplace/config.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Skills
+    module Marketplace
+      # Knobs for fetching marketplaces: caps, the GitHub token, and the
+      # guard a marketplace author's URL must pass.
+      #
+      # @example
+      #   RubyLLM::Skills::Marketplace.configure do |config|
+      #     config.github_token = ENV["MARKETPLACE_GITHUB_TOKEN"]
+      #     config.url_guard = ->(uri) { raise "nope" unless uri.host.end_with?(".example.com") }
+      #   end
+      #
+      class Config
+        MEBIBYTE = 1024 * 1024
+
+        # Sent to GitHub hosts only; lifts the rate limit and reaches private repositories.
+        attr_accessor :github_token
+        # One plugin archive (compressed and expanded), one file inside it, files per plugin, skills per plugin.
+        attr_accessor :max_archive_bytes, :max_file_bytes, :max_files, :max_skills
+        attr_accessor :user_agent
+        # Called with each URI hop of a URL a marketplace author supplied
+        # (a hosted marketplace.json, an archive source). Raise to refuse;
+        # return an IP address string to pin the connection to the address
+        # that passed, so a DNS answer cannot change between check and connect.
+        attr_accessor :url_guard
+
+        def initialize
+          @github_token = ENV.fetch("GITHUB_TOKEN", nil)
+          @max_archive_bytes = 64 * MEBIBYTE
+          @max_file_bytes = 16 * MEBIBYTE
+          @max_files = 2000
+          @max_skills = 200
+          @user_agent = "ruby_llm-skills/#{RubyLLM::Skills::VERSION}"
+          @url_guard = nil
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/skills/marketplace/fetcher.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module RubyLLM
+  module Skills
+    module Marketplace
+      # Where a marketplace's bytes come from: one adapter per source kind.
+      # +head+ answers "did the upstream move?" cheaply (a commit sha or a
+      # tree hash), +catalog_files+ fetches only the marketplace file(s),
+      # +source_tree+ fetches one plugin's content (the marketplace's own tree
+      # for relative sources, memoized so ten relative plugins cost one
+      # download; another repository's archive for github / gitlab sources;
+      # a .tar.gz for archive sources). Nothing here runs git.
+      #
+      module Fetcher
+        Head = Data.define(:sha, :etag, :unchanged) do
+          def unchanged? = unchanged
+        end
+
+        # +files+ is the plugin root as `{ path => bytes }`; +sha+ identifies the
+        # content (a commit, a tree hash, an archive digest).
+        Fetched = Data.define(:files, :sha, :upstream_url)
+
+        # @param source [Locator::Source]
+        # @return [Base]
+        def self.for(source)
+          case source.kind
+          when "github" then Github.new(source)
+          when "gitlab" then Gitlab.new(source)
+          when "url" then Url.new(source)
+          when "directory" then Directory.new(source)
+          else raise FetchError, "unknown marketplace kind #{source.kind.inspect}"
+          end
+        end
+
+        class Base
+          attr_reader :source
+
+          def initialize(source)
+            @source = source
+            @tarballs = {}
+            @shas = {}
+          end
+
+          def head(previous_sha: nil, previous_etag: nil)
+            raise NotImplementedError
+          end
+
+          # `{ path => bytes }` for the marketplace files that exist at +head+.
+          def catalog_files(head)
+            raise NotImplementedError
+          end
+
+          # The plugin content for one marketplace entry.
+          #
+          # @param plugin_source [Manifest::PluginSource]
+          # @param head [Head]
+          # @return [Fetched]
+          def source_tree(plugin_source, head:)
+            case plugin_source.kind
+            when "relative" then relative_tree(plugin_source, head)
+            when "github" then repo_tree(GithubRepo.new(plugin_source.repo), plugin_source)
+            when "gitlab" then repo_tree(GitlabRepo.new(plugin_source.repo), plugin_source)
+            when "archive" then archive_tree(plugin_source)
+            else raise FetchError, "unsupported source (#{plugin_source.reason})"
+            end
+          end
+
+          # The commit a source resolves to right now, without downloading it
+          # (nil for an archive, whose identity is its digest).
+          def source_sha(plugin_source, head:)
+            case plugin_source.kind
+            when "relative" then head.sha
+            when "github" then repo_sha(GithubRepo.new(plugin_source.repo), plugin_source)
+            when "gitlab" then repo_sha(GitlabRepo.new(plugin_source.repo), plugin_source)
+            end
+          end
+
+          private
+
+          def relative_tree(plugin_source, head)
+            Fetched.new(files: read_tree(head.sha, plugin_source.path), sha: head.sha, upstream_url: tree_url(head.sha, plugin_source.path))
+          end
+
+          def repo_sha(repo, plugin_source)
+            plugin_source.sha || @shas[[repo.class, repo.repo, plugin_source.ref]] ||= repo.commit(plugin_source.ref || repo.default_branch).first
+          end
+
+          def repo_tree(repo, plugin_source)
+            sha = repo_sha(repo, plugin_source)
+            Fetched.new(files: Tarball.read(tarball(repo, sha), subdir: plugin_source.path), sha: sha, upstream_url: repo.tree_url(sha, plugin_source.path))
+          end
+
+          # One download per repository commit however many plugins live in it.
+          def tarball(repo, sha)
+            @tarballs[[repo.class, repo.repo, sha]] ||= repo.tarball(sha)
+          end
+
+          def archive_tree(plugin_source)
+            response = Http.get(plugin_source.url, public: true)
+            raise FetchError, "archive returned HTTP #{response.status}" unless response.success?
+
+            digest = Digest::SHA256.hexdigest(response.body)
+            raise FetchError, "archive sha256 mismatch for #{plugin_source.url}" if plugin_source.sha256 && plugin_source.sha256 != digest
+
+            files = Tarball.read(response.body, strip_root: false)
+            Fetched.new(files: strip_single_root(files), sha: digest, upstream_url: plugin_source.url)
+          end
+
+          PLUGIN_MARKERS = %w[skills/ commands/ agents/ .claude-plugin/ .codex-plugin/ .cursor-plugin/ SKILL.md plugin.json].freeze
+
+          # An archive may carry the plugin at its top or one folder down: a
+          # single root directory that itself holds a plugin marker (a
+          # `skills/` directory, a manifest, a root `SKILL.md`) is a wrapper
+          # and is stripped; a root that is the plugin's own `skills/` is not.
+          def strip_single_root(files)
+            roots = files.keys.map { |path| path.split("/", 2) }
+            return files if roots.any? { |parts| parts.size == 1 } || roots.map(&:first).uniq.size != 1
+
+            root = roots.first.first
+            wrapper = PLUGIN_MARKERS.any? do |marker|
+              marker.end_with?("/") ? files.keys.any? { |path| path.start_with?("#{root}/#{marker}") } : files.key?("#{root}/#{marker}")
+            end
+            return files unless wrapper
+
+            files.transform_keys { |path| path.split("/", 2).last }
+          end
+
+          def subtree(files, subdir, where)
+            prefix = Manifest.presence(subdir)
+            return files if prefix.nil?
+
+            subset = files.select { |path, _| path.start_with?("#{prefix}/") }.transform_keys { |path| path.delete_prefix("#{prefix}/") }
+            raise FetchError, "#{where}/#{prefix} has no files" if subset.empty?
+
+            subset
+          end
+        end
+
+        # A repository over one of the repo clients (GitHub, GitLab).
+        class Repository < Base
+          def repo
+            raise NotImplementedError
+          end
+
+          def head(previous_sha: nil, previous_etag: nil)
+            ref = source.ref || repo.default_branch
+            result = repo.commit(ref, etag: previous_etag)
+            return Head.new(sha: previous_sha, etag: previous_etag, unchanged: true) if result == :not_modified
+
+            sha, etag = result
+            Head.new(sha: sha, etag: etag, unchanged: sha == previous_sha)
+          end
+
+          def catalog_files(head)
+            Manifest::PATHS.each_with_object({}) do |path, files|
+              body = repo.file(head.sha, path)
+              files[path] = body if body
+            end
+          end
+
+          private
+
+          def read_tree(sha, subdir)
+            Tarball.read(tarball(repo, sha), subdir: subdir)
+          end
+
+          def tree_url(sha, subdir)
+            repo.tree_url(sha, subdir)
+          end
+        end
+
+        class Github < Repository
+          def repo
+            @repo ||= GithubRepo.new(source.locator)
+          end
+        end
+
+        class Gitlab < Repository
+          def repo
+            @repo ||= GitlabRepo.new(source.locator)
+          end
+        end
+
+        # A bare `https://…/marketplace.json`: the file is the whole upstream,
+        # so relative sources cannot be fetched (as in Claude Code); github,
+        # gitlab and archive entries work.
+        class Url < Base
+          def head(previous_sha: nil, previous_etag: nil)
+            response = fetch(etag: previous_etag)
+            return Head.new(sha: previous_sha, etag: previous_etag, unchanged: true) if response.not_modified?
+
+            @body = response.body
+            sha = Digest::SHA256.hexdigest(response.body)
+            Head.new(sha: sha, etag: response.etag, unchanged: sha == previous_sha)
+          end
+
+          def catalog_files(_head)
+            {Manifest::PATHS.first => @body || fetch.body}
+          end
+
+          private
+
+          def fetch(etag: nil)
+            response = Http.get(source.locator, public: true, max_bytes: Manifest::MAX_BYTES, etag: etag)
+            raise FetchError, "marketplace URL returned HTTP #{response.status}" unless response.success? || response.not_modified?
+
+            response
+          end
+
+          def read_tree(_sha, _subdir)
+            raise FetchError, "relative sources are not available from a hosted marketplace.json"
+          end
+
+          def tree_url(_sha, _subdir)
+            source.locator
+          end
+        end
+
+        # A local folder holding a marketplace file. The tree's sha256 is the
+        # head, so an update is a local read that does nothing until the
+        # folder changes.
+        class Directory < Base
+          def root
+            dir = File.expand_path(source.locator)
+            raise FetchError, "marketplace directory #{source.locator} does not exist" unless File.directory?(dir)
+
+            dir
+          end
+
+          def head(previous_sha: nil, previous_etag: nil)
+            @files = nil
+            sha = Tarball.tree_sha256(files)
+            Head.new(sha: sha, etag: nil, unchanged: sha == previous_sha)
+          end
+
+          def catalog_files(_head)
+            files.slice(*Manifest::PATHS)
+          end
+
+          private
+
+          def files
+            @files ||= Tarball.from_directory(root)
+          end
+
+          def read_tree(_sha, subdir)
+            subtree(files, subdir, source.locator)
+          end
+
+          # A local folder has no public tree to link to.
+          def tree_url(_sha, _subdir)
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/skills/marketplace/github_repo.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module RubyLLM
+  module Skills
+    module Marketplace
+      # One GitHub repository over the REST API and its archive hosts: the
+      # default branch and a ref's head commit from api.github.com, single
+      # files from raw.githubusercontent.com, the tree as a tarball from
+      # codeload.github.com. The optional token lifts the rate limit and
+      # reaches private repositories; it is sent to these hosts only.
+      #
+      class GithubRepo
+        API = "api.github.com"
+        RAW = "raw.githubusercontent.com"
+        CODELOAD = "codeload.github.com"
+        HOSTS = [API, RAW, CODELOAD].freeze
+        SMALL = 16 * 1024 * 1024
+
+        attr_reader :repo
+
+        def initialize(repo, token: Marketplace.config.github_token)
+          @repo = Manifest.normalize_repo(repo)
+          raise FetchError, "invalid GitHub repository #{repo.inspect}" unless @repo.match?(Manifest::REPO_PATTERN)
+
+          @token = Manifest.presence(token)
+        end
+
+        def default_branch
+          Http.json(api("repos/#{repo}")).fetch("default_branch", "main").to_s
+        end
+
+        # The head commit of +ref+ (a branch, tag or sha): [sha, etag], or
+        # :not_modified when the ETag still matches.
+        def commit(ref, etag: nil)
+          response = api("repos/#{repo}/commits/#{ERB::Util.url_encode(ref)}", etag: etag, allow_not_modified: true)
+          return :not_modified if response.not_modified?
+
+          sha = Http.json(response)["sha"].to_s
+          raise FetchError, "#{repo}@#{ref}: no commit sha in the response" unless sha.match?(Manifest::SHA_PATTERN)
+
+          [sha, response.etag]
+        end
+
+        # One file at +sha+, or nil when it does not exist.
+        def file(sha, path)
+          response = Http.get("https://#{RAW}/#{repo}/#{sha}/#{path}", hosts: HOSTS, headers: headers, max_bytes: SMALL)
+          return nil if response.not_found?
+          raise FetchError, "#{repo}: #{path} returned HTTP #{response.status}" unless response.success?
+
+          response.body
+        end
+
+        def tarball(sha)
+          response = Http.get("https://#{CODELOAD}/#{repo}/tar.gz/#{sha}", hosts: HOSTS, headers: headers)
+          raise FetchError, "#{repo}: archive returned HTTP #{response.status}" unless response.success?
+
+          response.body
+        end
+
+        # `{ "tag" =>, "url" =>, "name" =>, "published_at" => }` for a tag that has a GitHub release, else nil.
+        def release(tag)
+          response = api("repos/#{repo}/releases/tags/#{ERB::Util.url_encode(tag)}", allow_not_found: true)
+          return nil if response.not_found?
+
+          data = Http.json(response)
+          {"tag" => tag, "url" => data["html_url"].to_s, "name" => data["name"].to_s, "published_at" => data["published_at"].to_s}
+        end
+
+        def tree_url(sha, subdir = nil)
+          ["https://github.com/#{repo}/tree/#{sha}", Manifest.presence(subdir)].compact.join("/")
+        end
+
+        private
+
+        def api(path, etag: nil, allow_not_modified: false, allow_not_found: false)
+          response = Http.get("https://#{API}/#{path}", hosts: HOSTS, headers: headers.merge("Accept" => "application/vnd.github+json"),
+            max_bytes: SMALL, etag: etag)
+          return response if response.success? || (allow_not_modified && response.not_modified?) || (allow_not_found && response.not_found?)
+          if [403, 429].include?(response.status) && response.headers["x-ratelimit-remaining"].to_s == "0"
+            raise FetchError, "GitHub rate limit reached for #{repo}"
+          end
+
+          raise FetchError, "#{repo}: GitHub API returned HTTP #{response.status} for #{path.split("/").last}"
+        end
+
+        def headers
+          @token ? {"Authorization" => "Bearer #{@token}"} : {}
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/skills/marketplace/gitlab_repo.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module RubyLLM
+  module Skills
+    module Marketplace
+      # One gitlab.com project over API v4: a ref's head commit, single raw
+      # files and the repository archive. Public projects only.
+      #
+      class GitlabRepo
+        HOST = "gitlab.com"
+        HOSTS = [HOST].freeze
+        SMALL = 16 * 1024 * 1024
+
+        attr_reader :repo
+
+        def initialize(project)
+          @repo = Manifest.normalize_repo(project)
+          raise FetchError, "invalid GitLab project #{project.inspect}" unless @repo.match?(Manifest::PROJECT_PATTERN)
+        end
+
+        def default_branch
+          Http.json(api("")).fetch("default_branch", "main").to_s
+        end
+
+        def commit(ref, etag: nil)
+          response = api("/repository/commits/#{ERB::Util.url_encode(ref)}", etag: etag, allow_not_modified: true)
+          return :not_modified if response.not_modified?
+
+          sha = Http.json(response)["id"].to_s
+          raise FetchError, "#{repo}@#{ref}: no commit id in the response" unless sha.match?(Manifest::SHA_PATTERN)
+
+          [sha, response.etag]
+        end
+
+        def file(sha, path)
+          response = Http.get("#{base}/repository/files/#{ERB::Util.url_encode(path)}/raw?ref=#{ERB::Util.url_encode(sha)}", hosts: HOSTS, max_bytes: SMALL)
+          return nil if response.not_found?
+          raise FetchError, "#{repo}: #{path} returned HTTP #{response.status}" unless response.success?
+
+          response.body
+        end
+
+        def tarball(sha)
+          response = Http.get("#{base}/repository/archive.tar.gz?sha=#{ERB::Util.url_encode(sha)}", hosts: HOSTS)
+          raise FetchError, "#{repo}: archive returned HTTP #{response.status}" unless response.success?
+
+          response.body
+        end
+
+        def tree_url(sha, subdir = nil)
+          ["https://#{HOST}/#{repo}/-/tree/#{sha}", Manifest.presence(subdir)].compact.join("/")
+        end
+
+        private
+
+        def base
+          "https://#{HOST}/api/v4/projects/#{ERB::Util.url_encode(repo)}"
+        end
+
+        def api(path, etag: nil, allow_not_modified: false)
+          response = Http.get("#{base}#{path}", hosts: HOSTS, max_bytes: SMALL, etag: etag)
+          return response if response.success? || (allow_not_modified && response.not_modified?)
+          raise FetchError, "GitLab rate limit reached for #{repo}" if response.status == 429
+
+          raise FetchError, "#{repo}: GitLab API returned HTTP #{response.status}"
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/skills/marketplace/http.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "uri"
+require "json"
+
+module RubyLLM
+  module Skills
+    module Marketplace
+      # The one HTTP door for marketplace fetches: HTTPS only, a body cap,
+      # redirects followed by hand, conditional requests through ETags.
+      #
+      # Two admission modes: +hosts:+ for the adapters' own well-known hosts,
+      # +public: true+ for a URL a marketplace author supplied, where every
+      # hop must pass +Config#url_guard+ when one is configured.
+      #
+      module Http
+        OPEN_TIMEOUT = 10
+        READ_TIMEOUT = 120
+        MAX_REDIRECTS = 3
+
+        Response = Data.define(:status, :body, :headers) do
+          def success? = status.between?(200, 299)
+
+          def not_modified? = status == 304
+
+          def not_found? = status == 404
+
+          def etag
+            value = headers["etag"].to_s
+            value.empty? ? nil : value
+          end
+        end
+
+        class << self
+          # @param url [String] an https URL
+          # @param hosts [Array<String>] hosts admitted by name
+          # @param public [Boolean] admit any https host that passes the url_guard
+          # @param headers [Hash] extra request headers
+          # @param max_bytes [Integer] abandon the body past this many bytes
+          # @param etag [String, nil] sent as If-None-Match
+          # @return [Response]
+          # @raise [FetchError]
+          def get(url, hosts: [], public: false, headers: {}, max_bytes: Marketplace.config.max_archive_bytes, etag: nil)
+            uri = parse(url)
+            request_headers = {"User-Agent" => Marketplace.config.user_agent}.merge(headers)
+            request_headers["If-None-Match"] = etag if etag
+
+            (MAX_REDIRECTS + 1).times do
+              address = admit!(uri, hosts: hosts, public: public)
+              response = perform(uri, request_headers, max_bytes, address: address)
+              location = response.headers["location"]
+              return response unless response.status.between?(300, 399) && location
+
+              uri = parse(URI.join(uri.to_s, location).to_s)
+            end
+
+            raise FetchError, "#{uri.host}: too many redirects"
+          rescue Timeout::Error, SocketError, SystemCallError, OpenSSL::SSL::SSLError, IOError => e
+            raise FetchError, "#{uri&.host}: #{e.class.name}"
+          end
+
+          def json(response)
+            parsed = JSON.parse(response.body.to_s)
+            parsed.is_a?(Hash) ? parsed : {}
+          rescue JSON::ParserError
+            raise FetchError, "malformed JSON response"
+          end
+
+          private
+
+          def parse(url)
+            URI.parse(url.to_s)
+          rescue URI::InvalidURIError
+            raise FetchError, "invalid URL #{url.to_s.inspect}"
+          end
+
+          # The address to connect to (nil: by name) once the hop is admitted.
+          def admit!(uri, hosts:, public:)
+            host = uri.host.to_s.downcase
+            raise FetchError, "refusing #{uri}: not an https URL" unless uri.is_a?(URI::HTTPS) && !host.empty?
+            raise FetchError, "refusing #{uri}: URLs with credentials are not allowed" if uri.userinfo
+            return nil if hosts.map(&:downcase).include?(host)
+            raise FetchError, "refusing #{host.inspect}: not an allowlisted https host" unless public
+
+            address = Marketplace.config.url_guard&.call(uri)
+            address.is_a?(String) ? address : nil
+          end
+
+          # Streams the body and stops reading the moment it passes the cap,
+          # so an upstream that answers with gigabytes never occupies more
+          # than the cap in memory. Pinned to +address+ when given, Net::HTTP
+          # still names the host for SNI and the certificate check.
+          def perform(uri, headers, max_bytes, address: nil)
+            body = String.new(encoding: Encoding::BINARY)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.ipaddr = address if address
+            http.use_ssl = true
+            http.open_timeout = OPEN_TIMEOUT
+            http.read_timeout = READ_TIMEOUT
+            http.start do
+              request = Net::HTTP::Get.new(uri.request_uri, headers)
+              http.request(request) do |response|
+                response.read_body do |chunk|
+                  raise FetchError, "#{uri.host}: response exceeds #{max_bytes} bytes" if body.bytesize + chunk.bytesize > max_bytes
+
+                  body << chunk.b
+                end
+                return Response.new(status: response.code.to_i, body: body, headers: response_headers(response))
+              end
+            end
+          end
+
+          def response_headers(response)
+            response.each_header.to_h { |key, value| [key.downcase, value] }
+          end
+        end
+      end
+    end
+  end
+end