ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/models.rb

@@ -1,207 +1,237 @@
 # frozen_string_literal: true
 
 require 'date'
-require 'json'
 
 module RubyLLM
-  # Registry of available AI models and their capabilities.
+  # A Models registry is the catalog of AI models RubyLLM knows about,
+  # including their capabilities, context windows, and pricing. The global
+  # registry is available through RubyLLM.models.
+  #
+  #   RubyLLM.models.find 'claude-sonnet-5'
+  #   RubyLLM.models.by_provider(:openai).chat_models
+  #   RubyLLM.models.refresh
+  #
+  # Filter methods return new Models instances, so calls chain. Models is
+  # enumerable over its Model entries. Class-level calls such as
+  # Models.find delegate to the global registry.
   class Models
     include Enumerable
 
-    MODELS_DEV_PROVIDER_MAP = {
+    MODELS_DEV_PROVIDER_MAP = { # :nodoc:
       'openai' => 'openai',
       'anthropic' => 'anthropic',
       'google' => 'gemini',
       'google-vertex' => 'vertexai',
       'amazon-bedrock' => 'bedrock',
+      'cohere' => 'cohere',
       'deepseek' => 'deepseek',
       'mistral' => 'mistral',
+      'ollama-cloud' => 'ollama_cloud',
       'openrouter' => 'openrouter',
       'perplexity' => 'perplexity',
+      'perplexity-agent' => 'perplexity',
       'xai' => 'xai'
     }.freeze
-    MODELS_DEV_INPUT_MODALITIES = %w[text image audio pdf video file].freeze
-    MODELS_DEV_OUTPUT_MODALITIES = %w[text image audio video embeddings moderation].freeze
-    MODELS_DEV_AUTHORITY_CAPABILITIES = %w[function_calling structured_output reasoning vision].freeze
+    MODELS_DEV_INPUT_MODALITIES = %w[text image audio pdf video file].freeze # :nodoc:
+    MODELS_DEV_OUTPUT_MODALITIES = %w[text image audio video embeddings moderation rerank].freeze # :nodoc:
+    # First-party providers outrank the aggregators that resell their models.
     PROVIDER_PREFERENCE = %w[
       openai
       anthropic
       gemini
-      vertexai
-      bedrock
-      openrouter
       deepseek
       mistral
+      cohere
       perplexity
       xai
+      vertexai
+      bedrock
+      openrouter
       azure
+      ollama_cloud
       ollama
       gpustack
-    ].freeze
+    ].freeze # :nodoc:
     INSTANCE_DELEGATES = (Enumerable.instance_methods(false) + %i[
       all
       each
       find
+      listed
+      unlisted
       chat_models
       embedding_models
       audio_models
       image_models
       by_family
       by_provider
-      load_from_json!
-      load_from_database!
+      load_from_json
+      load_from_store
       save_to_json
-    ]).uniq.freeze
+    ]).uniq.freeze # :nodoc:
 
     class << self
+      # The providers whose model list could not be fetched during the last
+      # refresh, as hashes of +:name+, +:slug+, and +:error+. Their previous
+      # models are kept, so an unreported failure leaves stale entries behind.
+      attr_reader :last_provider_failures
+
       INSTANCE_DELEGATES.each do |method_name|
         define_method(method_name) do |*args, **kwargs, &block|
-          if kwargs.empty?
-            instance.public_send(method_name, *args, &block)
-          else
-            instance.public_send(method_name, *args, **kwargs, &block)
-          end
+          instance.public_send(method_name, *args, **kwargs, &block)
         end
       end
 
-      def instance
+      def instance # :nodoc:
         @instance ||= new
       end
 
-      def schema_file
-        File.expand_path('models_schema.json', __dir__)
+      def bundled_registry_file # :nodoc:
+        File.expand_path('models.json', __dir__)
       end
 
-      def load_models(file = RubyLLM.config.model_registry_file)
-        source = RubyLLM.config.model_registry_source
-        if source && file == RubyLLM.config.model_registry_file
-          models = source.read
-          return models if models.any?
+      def load_models # :nodoc:
+        base = models_from_store(RubyLLM.config.model_registry_store) ||
+               models_from_file(RubyLLM.config.model_registry_file) ||
+               models_from_bundle
+
+        merge_models(models_from_provider_gems, base)
+      end
 
-          RubyLLM.logger.debug { 'Model registry source is empty, falling back to JSON registry' }
+      def models_from_provider_gems # :nodoc:
+        Provider.model_registry_files.flat_map do |provider, file|
+          Array(models_from_file(file)).select { |model| model.provider == provider.to_s }
         end
+      end
 
-        read_from_json(file)
+      def models_from_store(store) # :nodoc:
+        return unless store
+
+        models = Array(store.read)
+        return models unless models.empty?
+
+        RubyLLM.logger.debug { 'Model registry store is empty, falling back to the registry file' }
+        nil
       end
 
-      def read_from_json(file = RubyLLM.config.model_registry_file)
-        data = File.exist?(file) ? File.read(file) : '[]'
-        models = JSON.parse(data, symbolize_names: true).map { |model| Model::Info.new(model) }
-        filter_models(models)
-      rescue JSON::ParserError
-        []
+      def models_from_file(file) # :nodoc:
+        return unless file
+
+        models = Registry.read(file)
+        models unless models.nil? || models.empty?
+      rescue ModelRegistryError => e
+        RubyLLM.logger.warn("Ignoring invalid model registry file #{file}: #{e.message}")
+        nil
+      end
+
+      def models_from_bundle # :nodoc:
+        Registry.read(bundled_registry_file) || begin
+          RubyLLM.logger.warn(
+            "Bundled model registry is missing: #{bundled_registry_file}. " \
+            'Refresh the registry to rebuild it.'
+          )
+          []
+        end
       end
 
-      def read_from_database
-        ModelRegistry::ActiveRecordSource.new.read
+      def fetch_published_registry(etag: nil) # :nodoc:
+        Registry::PublishedSource.new.fetch(etag:)
       end
 
-      def refresh!(remote_only: false)
-        # Replaces the process-wide model registry. Call save_to_json when the
-        # refreshed registry should also be persisted.
+      # Refreshes the global model registry from the published catalog and
+      # configured providers. Returns the global Models instance. See
+      # #refresh for the +remote_only:+ option.
+      def refresh(remote_only: false)
+        instance.refresh(remote_only: remote_only)
+      end
+
+      def refresh_from_providers(remote_only: false) # :nodoc:
+        instance.refresh_from_providers(remote_only: remote_only)
+      end
+
+      # :stopdoc:
+
+      # Fetches and merges models directly from upstream provider APIs and
+      # models.dev for the maintainer registry builder.
+      def fetch_merged_models(remote_only: false) # :nodoc:
         RubyLLM.instrument('models.refresh.ruby_llm', remote_only:) do |payload|
-          existing_models = load_existing_models
+          existing_models = read_existing_models
 
           provider_fetch = fetch_provider_models(remote_only: remote_only)
+          @last_provider_failures = provider_fetch[:failed]
           log_provider_fetch(provider_fetch)
+          payload[:failed_providers] = provider_fetch[:failed].map { |failure| failure[:slug] }
 
           models_dev_fetch = fetch_models_dev_models(existing_models)
           log_models_dev_fetch(models_dev_fetch)
 
           merged_models = merge_with_existing(existing_models, provider_fetch, models_dev_fetch)
           payload[:model_count] = merged_models.size
-          @instance = new(merged_models)
+          merged_models
         end
       end
 
-      def fetch_provider_models(remote_only: true) # rubocop:disable Metrics/PerceivedComplexity
+      def fetch_provider_models(remote_only: true) # :nodoc:
         config = RubyLLM.config
-        provider_classes = remote_only ? Provider.remote_providers.values : Provider.providers.values
-        configured_classes = if remote_only
-                               Provider.configured_remote_providers(config)
-                             else
-                               Provider.configured_providers(config)
-                             end
-        configured = configured_classes.select { |klass| provider_classes.include?(klass) }
+        providers = remote_only ? Provider.configured_remote_providers(config) : Provider.configured_providers(config)
+        providers = providers.reject { |provider| Provider.model_registry_files.key?(provider.slug.to_sym) }
         result = {
-          models: [],
-          fetched_providers: [],
-          configured_names: configured.map(&:name),
-          failed: []
+          models: [], fetched_providers: [], configured_names: providers.map(&:display_name), failed: [], empty: []
         }
 
-        provider_classes.each do |provider_class|
-          next if remote_only && provider_class.local?
-          next unless provider_class.configured?(config)
-
-          begin
-            result[:models].concat(provider_class.new(config).list_models)
+        providers.each do |provider_class|
+          models = provider_class.new(config).list_models
+          if models.empty?
+            result[:empty] << { name: provider_class.display_name, slug: provider_class.slug }
+          else
+            result[:models].concat(models)
             result[:fetched_providers] << provider_class.slug
-          rescue StandardError => e
-            result[:failed] << { name: provider_class.name, slug: provider_class.slug, error: e }
           end
+        rescue StandardError => e
+          result[:failed] << { name: provider_class.display_name, slug: provider_class.slug, error: e }
         end
 
-        result[:fetched_providers].uniq!
         result
       end
 
-      # Backwards-compatible wrapper used by specs.
-      def fetch_from_providers(remote_only: true)
-        fetch_provider_models(remote_only: remote_only)[:models]
-      end
-
-      def resolve(model_id, provider: nil, assume_exists: false, config: nil) # rubocop:disable Metrics/PerceivedComplexity
+      def resolve(model_id, provider: nil, assume_model_exists: false, config: nil,
+                  operation: nil, default_model: nil) # rubocop:disable Metrics/PerceivedComplexity
         config ||= RubyLLM.config
         provider_class = provider ? Provider.providers[provider.to_sym] : nil
+        if operation && provider_class && !provider_class.model_required?(operation:)
+          raise ArgumentError, "#{operation} does not accept a model" unless model_id.nil?
 
-        if provider_class
-          temp_instance = provider_class.new(config)
-          assume_exists = true if temp_instance.local? || temp_instance.assume_models_exist?
+          return [nil, provider_class.new(config)]
         end
+        model_id ||= default_model
+        assume_model_exists = true if provider_class&.local? || provider_class&.assume_models_exist?
 
-        if assume_exists
-          raise ArgumentError, 'Provider must be specified if assume_exists is true' unless provider
+        if assume_model_exists
+          raise ArgumentError, 'Provider must be specified if assume_model_exists is true' unless provider
 
-          provider_class ||= raise_unknown_provider(provider)
-          provider_instance = provider_class.new(config)
+          provider_class ||= Provider.resolve!(provider)
 
-          model = if provider_instance.local?
-                    begin
-                      Models.find(model_id, provider)
-                    rescue ModelNotFoundError
-                      nil
-                    end
-                  end
+          model = begin
+            Models.find(model_id, provider: provider, config: config)
+          rescue ModelNotFoundError
+            nil
+          end
 
-          model ||= Model::Info.default(model_id, provider_instance.slug)
+          model ||= Model.default(model_id, provider_class.slug)
         else
-          model = Models.find model_id, provider
-          provider_class = Provider.providers[model.provider.to_sym] || raise_unknown_provider(model.provider)
-          provider_instance = provider_class.new(config)
+          model = Models.find model_id, provider: provider, config: config
+          provider_class = Provider.resolve!(model.provider)
         end
-        [model, provider_instance]
+        [model, provider_class.new(config)]
       end
 
-      def fetch_models_dev_models(existing_models) # rubocop:disable Metrics/PerceivedComplexity
+      def fetch_models_dev_models(existing_models) # :nodoc:
         RubyLLM.logger.info 'Fetching models from models.dev API...'
 
-        connection = Connection.basic do |f|
+        connection = Transport::Connection.basic do |f|
           f.request :json
           f.response :json, parser_options: { symbolize_names: true }
         end
-        response = connection.get 'https://models.dev/api.json'
-        providers = response.body || {}
-
-        models = providers.flat_map do |provider_key, provider_data|
-          provider_slug = MODELS_DEV_PROVIDER_MAP[provider_key.to_s]
-          next [] unless provider_slug
-
-          (provider_data[:models] || {}).values.map do |model_data|
-            Model::Info.new(models_dev_model_to_info(model_data, provider_slug, provider_key.to_s))
-          end
-        end
-        { models: models.reject { |model| model.provider.nil? || model.id.nil? }, fetched: true }
+        { models: parse_models_dev_catalog(connection.get('https://models.dev/api.json').body), fetched: true }
       rescue StandardError => e
         RubyLLM.logger.warn("Failed to fetch models.dev (#{e.class}: #{e.message}). Keeping existing.")
         {
@@ -210,18 +240,33 @@ module RubyLLM
         }
       end
 
-      def load_existing_models
-        existing_models = instance&.all
-        existing_models = read_from_json if existing_models.nil? || existing_models.empty?
-        existing_models
+      # An answer RubyLLM cannot read a single model out of is no answer:
+      # only a catalog carrying models may overrule what the registry holds.
+      def parse_models_dev_catalog(body) # :nodoc:
+        raise ModelRegistryError, "models.dev returned #{body.class} instead of a catalog" unless body.is_a?(Hash)
+
+        models = body.flat_map { |provider_key, data| models_dev_provider_models(provider_key, data) }
+        raise ModelRegistryError, 'models.dev returned no models RubyLLM knows a provider for' if models.empty?
+
+        models
       end
 
-      def raise_unknown_provider(provider)
-        available = Provider.providers.keys.join(', ')
-        raise Error, "Unknown provider: #{provider.inspect}. Available providers: #{available}"
+      def models_dev_provider_models(provider_key, provider_data) # :nodoc:
+        provider_slug = MODELS_DEV_PROVIDER_MAP[provider_key.to_s]
+        return [] unless provider_slug
+
+        (provider_data[:models] || {}).values.filter_map do |model_data|
+          model = Model.new(models_dev_model_attributes(model_data, provider_slug, provider_key.to_s))
+          model unless model.provider.nil? || model.id.nil?
+        end
       end
 
-      def log_provider_fetch(provider_fetch)
+      def read_existing_models # :nodoc:
+        existing_models = instance.all
+        existing_models.empty? ? load_models : existing_models
+      end
+
+      def log_provider_fetch(provider_fetch) # :nodoc:
         RubyLLM.logger.info "Fetching models from providers: #{provider_fetch[:configured_names].join(', ')}"
         provider_fetch[:failed].each do |failure|
           RubyLLM.logger.warn(
@@ -229,15 +274,18 @@ module RubyLLM
             'Keeping existing.'
           )
         end
+        Array(provider_fetch[:empty]).each do |provider|
+          RubyLLM.logger.warn("#{provider[:name]} listed no models. Keeping existing.")
+        end
       end
 
-      def log_models_dev_fetch(models_dev_fetch)
+      def log_models_dev_fetch(models_dev_fetch) # :nodoc:
         return if models_dev_fetch[:fetched]
 
         RubyLLM.logger.warn('Using cached models.dev data due to fetch failure.')
       end
 
-      def merge_with_existing(existing_models, provider_fetch, models_dev_fetch)
+      def merge_with_existing(existing_models, provider_fetch, models_dev_fetch) # :nodoc:
         existing_by_provider = existing_models.group_by(&:provider)
         preserved_models = existing_by_provider
                            .except(*provider_fetch[:fetched_providers])
@@ -254,70 +302,50 @@ module RubyLLM
         merge_models(provider_models, models_dev_models)
       end
 
-      def merge_models(provider_models, models_dev_models)
+      def merge_models(provider_models, models_dev_models) # :nodoc:
         models_dev_by_key = index_by_key(models_dev_models)
         provider_by_key = index_by_key(provider_models)
+        provider_by_alias = index_provider_aliases(provider_models)
 
         all_keys = models_dev_by_key.keys | provider_by_key.keys
 
         models = all_keys.map do |key|
-          models_dev_model = find_models_dev_model(key, models_dev_by_key)
-          provider_model = provider_by_key[key]
+          provider_model = provider_by_key[key] || provider_by_alias[key]
+          models_dev_model = find_models_dev_model(key, models_dev_by_key, provider_model)
 
           if models_dev_model && provider_model
             add_provider_metadata(models_dev_model, provider_model)
           elsif models_dev_model
             models_dev_model
           else
-            provider_model
+            augment_model_capabilities(provider_model)
           end
         end
 
-        filter_models(models).sort_by { |m| [m.provider, m.id] }
+        models.sort_by { |m| [m.provider, m.id] }
       end
 
-      def filter_models(models)
-        models.reject do |model|
-          model.provider.to_s == 'vertexai' && model.id.to_s.include?('/')
-        end
-      end
-
-      def find_models_dev_model(key, models_dev_by_key)
-        # Direct match
+      def find_models_dev_model(key, models_dev_by_key, provider_model = nil) # :nodoc:
         return models_dev_by_key[key] if models_dev_by_key[key]
 
         provider, model_id = key.split(':', 2)
-        if provider == 'bedrock'
-          normalized_id = model_id.sub(/^[a-z]{2}\./, '')
-          context_override = nil
-          normalized_id = normalized_id.gsub(/:(\d+)k\b/) do
-            context_override = Regexp.last_match(1).to_i * 1000
-            ''
-          end
-          bedrock_model = models_dev_by_key["bedrock:#{normalized_id}"]
-          if bedrock_model
-            data = bedrock_model.to_h.merge(id: model_id)
-            data[:context_window] = context_override if context_override
-            return Model::Info.new(data)
-          end
-        end
-
-        # VertexAI uses same models as Gemini
-        return unless provider == 'vertexai'
-
-        gemini_model = models_dev_by_key["gemini:#{model_id}"]
-        return unless gemini_model
-
-        # Return Gemini's models.dev data but with VertexAI as provider
-        Model::Info.new(gemini_model.to_h.merge(provider: 'vertexai'))
+        Provider.resolve(provider)&.models_dev_alias(model_id, models_dev_by_key, provider_model)
       end
 
-      def index_by_key(models)
+      def index_by_key(models) # :nodoc:
         models.to_h do |model|
           ["#{model.provider}:#{model.id}", model]
         end
       end
 
+      def index_provider_aliases(models) # :nodoc:
+        models.each_with_object({}) do |model, aliases|
+          Array(model.metadata[:aliases]).each do |alias_id|
+            aliases["#{model.provider}:#{alias_id}"] ||= model
+          end
+        end
+      end
+
       def add_provider_metadata(models_dev_model, provider_model) # rubocop:disable Metrics/PerceivedComplexity
         data = models_dev_model.to_h
         data[:name] = provider_model.name if blank_value?(data[:name])
@@ -325,16 +353,51 @@ module RubyLLM
         data[:created_at] = provider_model.created_at if blank_value?(data[:created_at])
         data[:context_window] = provider_model.context_window if blank_value?(data[:context_window])
         data[:max_output_tokens] = provider_model.max_output_tokens if blank_value?(data[:max_output_tokens])
+        data[:knowledge_cutoff] = provider_model.knowledge_cutoff if blank_value?(data[:knowledge_cutoff])
         data[:modalities] = provider_model.modalities.to_h if blank_value?(data[:modalities])
-        data[:pricing] = provider_model.pricing.to_h if blank_value?(data[:pricing])
+        if models_dev_model.type == :chat && provider_model.type != :chat
+          data[:modalities] = provider_model.modalities.to_h
+        end
+        data[:pricing] = Support::Utils.deep_merge(provider_model.pricing.to_h, data[:pricing].to_h)
         data[:metadata] = provider_model.metadata.merge(data[:metadata] || {})
-        provider_capabilities = provider_model.capabilities - MODELS_DEV_AUTHORITY_CAPABILITIES
-        data[:capabilities] = (models_dev_model.capabilities + provider_capabilities).uniq
+        data[:capabilities] = merge_capabilities(models_dev_model, provider_model, data[:modalities])
         normalize_embedding_modalities(data)
-        Model::Info.new(data)
+        Model.new(data)
       end
 
-      def normalize_embedding_modalities(data)
+      def merge_capabilities(models_dev_model, provider_model, modalities) # :nodoc:
+        denied = models_dev_reported_capabilities(models_dev_model) - models_dev_model.capabilities
+        reported = (models_dev_model.capabilities + provider_model.capabilities).uniq - denied
+        augment_capabilities(provider_model.provider, reported, provider_model.id, modalities)
+      end
+
+      def augment_model_capabilities(model) # :nodoc:
+        capabilities = augment_capabilities(model.provider, model.capabilities, model.id, model.modalities.to_h)
+        return model if capabilities == model.capabilities
+
+        Model.new(model.to_h.merge(capabilities: capabilities))
+      end
+
+      def augment_capabilities(provider_slug, capabilities, model_id, modalities) # :nodoc:
+        augmenter = Provider.resolve(provider_slug)&.capabilities
+        return capabilities unless augmenter
+
+        augmenter.augment(capabilities, model_id: model_id, modalities: modalities.to_h)
+      end
+
+      # models.dev leaves a field out where it has no opinion, so only the
+      # capabilities it reports on can overrule what a provider claims.
+      def models_dev_reported_capabilities(models_dev_model) # :nodoc:
+        metadata = models_dev_model.metadata
+        reported = []
+        reported << 'function_calling' unless metadata[:tool_call].nil?
+        reported << 'structured_output' unless metadata[:structured_output].nil?
+        reported << 'reasoning' unless metadata[:reasoning].nil? && metadata[:reasoning_options].nil?
+        reported << 'vision' unless models_dev_model.modalities.input.empty?
+        reported
+      end
+
+      def normalize_embedding_modalities(data) # :nodoc:
         return unless data[:id].to_s.include?('embedding')
 
         modalities = data[:modalities].to_h
@@ -343,7 +406,7 @@ module RubyLLM
         data[:modalities] = modalities
       end
 
-      def blank_value?(value)
+      def blank_value?(value) # :nodoc:
         return true if value.nil?
         return value.empty? if value.is_a?(String) || value.is_a?(Array)
 
@@ -356,19 +419,19 @@ module RubyLLM
         false
       end
 
-      def models_dev_model_to_info(model_data, provider_slug, provider_key)
+      def models_dev_model_attributes(model_data, provider_slug, provider_key) # :nodoc:
         modalities = normalize_models_dev_modalities(model_data[:modalities])
-        capabilities = models_dev_capabilities(model_data, modalities)
+        capabilities = models_dev_capabilities(model_data, modalities, provider_slug)
 
         created_date = [model_data[:release_date], model_data[:last_updated]]
                        .find { |value| !value.to_s.strip.empty? }
 
         data = {
-          id: model_data[:id],
+          id: models_dev_model_id(model_data[:id], provider_slug),
           name: model_data[:name] || model_data[:id],
           provider: provider_slug,
           family: model_data[:family],
-          created_at: Utils.iso_date_prefix_to_utc_midnight_string(created_date),
+          created_at: Support::Utils.iso_date_prefix_to_utc_midnight_string(created_date),
           context_window: model_data.dig(:limit, :context),
           max_output_tokens: model_data.dig(:limit, :output),
           knowledge_cutoff: normalize_models_dev_knowledge(model_data[:knowledge]),
@@ -382,16 +445,22 @@ module RubyLLM
         data
       end
 
-      def models_dev_capabilities(model_data, modalities)
+      def models_dev_model_id(id, provider_slug) # :nodoc:
+        provider = Provider.resolve(provider_slug)
+        provider ? provider.models_dev_model_id(id) : id
+      end
+
+      def models_dev_capabilities(model_data, modalities, provider_slug) # :nodoc:
         capabilities = []
         capabilities << 'function_calling' if model_data[:tool_call]
         capabilities << 'structured_output' if model_data[:structured_output]
         capabilities << 'reasoning' if model_data[:reasoning] || model_data[:reasoning_options]
         capabilities << 'vision' if modalities[:input].intersect?(%w[image video pdf])
-        capabilities.uniq
+        capabilities << 'video' if modalities[:input].include?('video')
+        augment_capabilities(provider_slug, capabilities.uniq, model_data[:id], modalities)
       end
 
-      def models_dev_pricing(cost)
+      def models_dev_pricing(cost) # :nodoc:
         return {} unless cost
 
         text_standard = {
@@ -408,12 +477,27 @@ module RubyLLM
         }.compact
 
         pricing = {}
-        pricing[:text_tokens] = { standard: text_standard } if text_standard.any?
+        text_tokens = models_dev_text_tokens_pricing(text_standard, cost)
+        pricing[:text_tokens] = text_tokens if text_tokens
         pricing[:audio_tokens] = { standard: audio_standard } if audio_standard.any?
         pricing
       end
 
-      def models_dev_metadata(model_data, provider_key)
+      def models_dev_text_tokens_pricing(text_standard, cost) # :nodoc:
+        long_context, threshold = Model::PricingCategory.long_context_from_cost(cost)
+
+        return nil if text_standard.empty? && long_context.nil?
+
+        text_tokens = {}
+        text_tokens[:standard] = text_standard if text_standard.any?
+        if long_context
+          text_tokens[:long_context] = long_context
+          text_tokens[:long_context_threshold] = threshold if threshold
+        end
+        text_tokens
+      end
+
+      def models_dev_metadata(model_data, provider_key) # :nodoc:
         metadata = {
           source: 'models.dev',
           provider_id: provider_key,
@@ -423,6 +507,9 @@ module RubyLLM
           last_updated: model_data[:last_updated],
           status: model_data[:status],
           interleaved: model_data[:interleaved],
+          tool_call: model_data[:tool_call],
+          structured_output: model_data[:structured_output],
+          reasoning: model_data[:reasoning],
           reasoning_options: model_data[:reasoning_options],
           cost: model_data[:cost],
           limit: model_data[:limit],
@@ -431,7 +518,7 @@ module RubyLLM
         metadata.compact
       end
 
-      def normalize_models_dev_modalities(modalities)
+      def normalize_models_dev_modalities(modalities) # :nodoc:
         normalized = { input: [], output: [] }
         return normalized unless modalities
 
@@ -440,7 +527,7 @@ module RubyLLM
         normalized
       end
 
-      def normalize_models_dev_knowledge(value)
+      def normalize_models_dev_knowledge(value) # :nodoc:
         return if value.nil?
         return value if value.is_a?(Date)
 
@@ -450,108 +537,286 @@ module RubyLLM
       end
     end
 
-    def initialize(models = nil)
-      @models = self.class.filter_models(models || self.class.load_models)
+    # :startdoc:
+
+    def initialize(models = nil) # :nodoc:
+      @models = models || self.class.load_models
     end
 
-    # Replaces this registry instance with models loaded from JSON.
-    def load_from_json!(file = RubyLLM.config.model_registry_file)
-      @models = self.class.read_from_json(file)
+    # Replaces the models in this registry with those read from the JSON
+    # +file+. The default is the configured
+    # <tt>RubyLLM.config.model_registry_file</tt>. A missing or invalid
+    # file falls back to the registry bundled with the gem.
+    def load_from_json(file = RubyLLM.config.model_registry_file)
+      @models = self.class.models_from_file(file) || self.class.models_from_bundle
+      self
     end
 
-    # Replaces this registry instance with models loaded from the configured
-    # ActiveRecord model class.
-    def load_from_database!
-      @models = self.class.read_from_database
+    # Replaces the models in this registry with entries from the configured
+    # model-registry store.
+    def load_from_store
+      store = RubyLLM.config.model_registry_store
+      raise ModelRegistryError, 'No model registry store is configured' unless store
+
+      @models = Array(store.read)
+      self
     end
 
-    # Persists this registry instance to JSON without changing the global
-    # RubyLLM.models instance.
+    # Exports this registry to +file+ as pretty-printed JSON. The default is
+    # the configured <tt>RubyLLM.config.model_registry_file</tt>. A regular
+    # #refresh already persists to the active registry store.
+    #
+    #   RubyLLM.models.save_to_json('/tmp/models.json')
+    #
     def save_to_json(file = RubyLLM.config.model_registry_file)
-      File.write(file, JSON.pretty_generate(all.map(&:to_h)))
+      Registry::FileStore.new(file).write(all)
+      self
     end
 
+    # Returns an array of the Model entries the configured provider still
+    # lists. Models it has stopped listing are left out; #find still resolves
+    # them, and #unlisted reports them.
     def all
+      all_including_unlisted.reject(&:unlisted?)
+    end
+
+    # Returns an array of the Model entries the configured provider has
+    # stopped listing. Only a store that keeps them, such as the Rails model
+    # table, ever reports one.
+    #
+    #   RubyLLM.models.unlisted.map(&:id)
+    #
+    def unlisted
+      all_including_unlisted.select(&:unlisted?)
+    end
+
+    # Returns an array of the Model entries the configured provider still
+    # lists. Reads the same as #all, which already excludes the rest.
+    alias listed all
+
+    def all_including_unlisted # :nodoc:
       @models
     end
 
+    # Yields each Model in the registry.
+    #
+    #   RubyLLM.models.each { |model| puts model.id }
+    #
     def each(&)
       all.each(&)
     end
 
-    def find(model_id, provider = nil)
+    # Returns the Model matching +model_id+, resolving aliases along the
+    # way. Without +provider+, picks the preferred provider that carries
+    # the model, first-party providers before aggregators. Raises
+    # RubyLLM::ModelNotFoundError if no model matches.
+    #
+    #   RubyLLM.models.find 'gpt-5.6'
+    #   RubyLLM.models.find 'claude-sonnet-5', provider: :bedrock
+    #
+    def find(model_id, provider: nil, config: nil)
       if provider
-        find_with_provider(model_id, provider)
+        find_with_provider(model_id, provider, config)
       else
         find_without_provider(model_id)
       end
     end
 
+    # Returns a new Models registry containing only chat models.
     def chat_models
-      self.class.new(all.select { |m| m.type == 'chat' })
+      select_models { |m| m.type == :chat }
     end
 
+    # Returns a new Models registry containing only embedding models.
     def embedding_models
-      self.class.new(all.select { |m| m.type == 'embedding' || m.modalities.output.include?('embeddings') })
+      select_models { |m| m.type == :embedding || m.modalities.output.include?('embeddings') }
     end
 
+    # Returns a new Models registry containing only models with audio
+    # output.
     def audio_models
-      self.class.new(all.select { |m| m.type == 'audio' || m.modalities.output.include?('audio') })
+      select_models { |m| m.type == :audio || m.modalities.output.include?('audio') }
     end
 
+    # Returns a new Models registry containing only models with image
+    # output.
     def image_models
-      self.class.new(all.select { |m| m.type == 'image' || m.modalities.output.include?('image') })
+      select_models { |m| m.type == :image || m.modalities.output.include?('image') }
     end
 
+    # Returns a new Models registry containing only models in +family+.
+    #
+    #   RubyLLM.models.by_family('claude3_sonnet')
+    #
     def by_family(family)
-      self.class.new(all.select { |m| m.family == family.to_s })
+      select_models { |m| m.family == family.to_s }
     end
 
+    # Returns a new Models registry containing only models from +provider+.
+    # Accepts a symbol or a string.
+    #
+    #   RubyLLM.models.by_provider(:openai).select { |model| model.supports?(:vision) }
+    #
     def by_provider(provider)
-      self.class.new(all.select { |m| m.provider == provider.to_s })
+      select_models { |m| m.provider == provider.to_s }
+    end
+
+    # Replaces the registry with the latest published RubyLLM catalog,
+    # merged with models discovered from configured providers. The result is
+    # saved to the platform cache, or to the database in Rails applications.
+    # Pass +remote_only:+ +true+ to skip local providers such as Ollama and
+    # GPUStack. Returns +self+.
+    #
+    # Raises ModelRegistryError when the catalog cannot be fetched or the
+    # result cannot be persisted, leaving the current registry unchanged.
+    #
+    #   RubyLLM.models.refresh
+    #   RubyLLM.models.refresh(remote_only: true).chat_models
+    #
+    def refresh(remote_only: false)
+      RubyLLM.instrument('models.refresh.ruby_llm', remote_only:) do |payload|
+        published = fetch_published_models
+        main_models = merge_discovered_models(published.models, remote_only:)
+        merged_models = self.class.merge_models(self.class.models_from_provider_gems, main_models)
+        persisted_models = RubyLLM.config.model_registry_store ? merged_models : main_models
+        persist_registry!(persisted_models, published:)
+        @models = stored_models || merged_models
+        payload.merge!(model_count: all.size, not_modified: published.not_modified)
+      end
+      self
     end
 
-    def refresh!(remote_only: false)
-      self.class.refresh!(remote_only: remote_only)
+    def refresh_from_providers(remote_only: false) # :nodoc:
+      @models = self.class.fetch_merged_models(remote_only: remote_only)
+      self
     end
 
-    def resolve(model_id, provider: nil, assume_exists: false, config: nil)
-      self.class.resolve(model_id, provider: provider, assume_exists: assume_exists, config: config)
+    def resolve(model_id, provider: nil, assume_model_exists: false, config: nil) # :nodoc:
+      self.class.resolve(model_id, provider: provider, assume_model_exists: assume_model_exists, config: config)
     end
 
     private
 
-    def find_with_provider(model_id, provider)
-      resolved_id = Aliases.resolve(model_id, provider)
-      resolved_id = resolve_bedrock_region_id(resolved_id) if provider.to_s == 'bedrock'
-      all.find { |m| m.id == resolved_id && m.provider == provider.to_s } ||
-        all.find { |m| m.id == model_id && m.provider == provider.to_s } ||
-        raise_model_not_found(model_id, provider: provider)
+    # Filters keep the unlisted entries so #find and #unlisted still see them
+    # after a chain such as by_provider(:openai).unlisted.
+    def select_models(&)
+      self.class.new(all_including_unlisted.select(&))
     end
 
-    def resolve_bedrock_region_id(model_id)
-      region = RubyLLM.config.bedrock_region.to_s
-      return model_id if region.empty?
+    def file_store
+      return if RubyLLM.config.model_registry_store
+      return unless RubyLLM.config.model_registry_file
 
-      candidate_id = Providers::Bedrock::Models.with_region_prefix(model_id, region)
-      return model_id if candidate_id == model_id
+      Registry::FileStore.new(RubyLLM.config.model_registry_file)
+    end
 
-      candidate = all.find { |m| m.provider == 'bedrock' && m.id == candidate_id }
-      return model_id unless candidate
+    # The ETag identifies the published catalog, not the merged registry, so
+    # it may only be sent while the catalog it stands for is still on disk.
+    def published_store
+      file = file_store
+      Registry::FileStore.new("#{file.path}.published.json") if file
+    end
 
-      inference_types = Array(candidate.metadata[:inference_types] || candidate.metadata['inference_types'])
-      Providers::Bedrock::Models.normalize_inference_profile_id(model_id, inference_types, region)
+    def fetch_published_models
+      cached = published_catalog
+      result = self.class.fetch_published_registry(etag: (file_store.etag if cached))
+      result.models ||= cached
+      result.models ? result : self.class.fetch_published_registry
     end
 
-    def find_without_provider(model_id)
-      exact_matches = all.select { |m| m.id == model_id }
-      return preferred_match(exact_matches) if exact_matches.any?
+    def published_catalog
+      models = published_store&.read
+      models unless models.nil? || models.empty?
+    rescue ModelRegistryError
+      nil
+    end
+
+    def merge_discovered_models(published, remote_only:)
+      provider_fetch = self.class.fetch_provider_models(remote_only: remote_only)
+      self.class.log_provider_fetch(provider_fetch)
+      preserved = preserved_providers(provider_fetch, published)
+      preserved_models = all.select { |model| preserved.include?(model.provider) }
+      self.class.merge_models(provider_fetch[:models] + preserved_models, published)
+    end
+
+    # A provider that answered replaces its own models and the published
+    # catalog replaces what it covers. Everything else survives the refresh,
+    # including the local providers a remote_only run never asks.
+    def preserved_providers(provider_fetch, published)
+      failed = provider_fetch[:failed].map { |failure| failure[:slug] }
+      covered = provider_fetch[:fetched_providers] + published.map(&:provider)
+      provider_gems = Provider.model_registry_files.keys.map(&:to_s)
+      failed | (all.map(&:provider).uniq - covered - provider_gems)
+    end
+
+    def persist_registry!(models, published:)
+      store = RubyLLM.config.model_registry_store
+      if store
+        raise ModelRegistryError, "Model registry store #{store.class} is read-only" unless store.respond_to?(:write)
+
+        store.write(self.class.new(models))
+        return
+      end
+
+      file = file_store
+      raise ModelRegistryError, 'No writable model registry store is configured' unless file
+
+      write_published_catalog(published)
+      file.write(models, etag: published.etag)
+    rescue ModelRegistryError
+      raise
+    rescue StandardError => e
+      destination = store_description(store) || file&.path || 'the configured store'
+      raise ModelRegistryError, "Could not save the model registry to #{destination}: #{e.message}"
+    end
+
+    # A store keeps entries the merge dropped, such as the unlisted rows a
+    # Rails application still references, so its answer wins over the merge.
+    def stored_models
+      store = RubyLLM.config.model_registry_store
+      return unless store.respond_to?(:read)
+
+      models = Array(store.read)
+      models unless models.empty?
+    rescue StandardError => e
+      RubyLLM.logger.debug { "Could not re-read the model registry store: #{e.message}" }
+      nil
+    end
+
+    def write_published_catalog(published)
+      published_store.write(published.models) unless published.not_modified
+    end
+
+    def store_description(store)
+      return unless store
+
+      store.respond_to?(:description) ? store.description : store.class.name
+    end
+
+    def find_with_provider(model_id, provider, config = nil)
+      resolved_id = Aliases.resolve(model_id, provider)
+      resolved_id = resolve_provider_registry_id(resolved_id, provider, config)
+      all_including_unlisted.find { |m| m.id == resolved_id && m.provider == provider.to_s } ||
+        all_including_unlisted.find { |m| m.id == model_id && m.provider == provider.to_s } ||
+        raise_model_not_found(model_id, provider: provider)
+    end
+
+    def resolve_provider_registry_id(model_id, provider, config = nil)
+      provider_class = Provider.resolve(provider)
+      return model_id unless provider_class
 
+      provider_class.resolve_registry_id(model_id, self, config || RubyLLM.config)
+    end
+
+    # A name can be one provider's exact id and another's alias:
+    # claude-opus-4 is exact on vertexai, an alias on anthropic.
+    # Provider preference settles it, not the kind of match.
+    def find_without_provider(model_id)
       resolved_id = Aliases.resolve(model_id)
-      alias_matches = all.select { |m| m.id == resolved_id }
-      return preferred_match(alias_matches) if alias_matches.any?
+      matches = all_including_unlisted.select { |m| [model_id, resolved_id].include?(m.id) }
+                                      .sort_by { |m| m.id == model_id ? 0 : 1 }
 
-      raise_model_not_found(model_id)
+      preferred_match(matches) || raise_model_not_found(model_id)
     end
 
     def raise_model_not_found(model_id, provider: nil)
@@ -562,10 +827,7 @@ module RubyLLM
     end
 
     def refresh_registry_guidance
-      rails_model = RubyLLM.config.model_registry_class
-      'If the model exists at the provider, refresh the registry with `RubyLLM.models.refresh!` ' \
-        'and persist it with `RubyLLM.models.save_to_json`. ' \
-        "Rails model registries can call `#{rails_model}.refresh!` instead."
+      'If the model exists at the provider, refresh the registry with `RubyLLM.models.refresh`.'
     end
 
     def preferred_match(candidates)
@@ -573,7 +835,7 @@ module RubyLLM
 
       candidates.min_by do |model|
         index = PROVIDER_PREFERENCE.index(model.provider)
-        index || PROVIDER_PREFERENCE.length
+        [model.unlisted? ? 1 : 0, index || PROVIDER_PREFERENCE.length]
       end
     end
   end