completion-kit 0.16.3 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6481d39133e4f2fd59c4fe262608363e206e5a374863055489021cbde84dfa11
-  data.tar.gz: c3e7db262f0c8d97526eb7057af0d4724dfbc331051640ff02d64129936e6bec
+  metadata.gz: 971ebc337d2fa495fc31e82e98428aaee6f007ab3353f61d560df95f0cd15f78
+  data.tar.gz: 6aaa974cef5da25388cc19c4f3ce34b0c0b9fe447523c4430a7ac313351d3035
 SHA512:
-  metadata.gz: 34158b719e5bc88757f81c43f6f676e236db71cd935008f3910e7b94b2560b9c420dd0dec0ef7428a31bfd30d4544b674f39337b0745d539dd53449b2534de0a
-  data.tar.gz: fffb6ff7102277d203378100ad0e58319f2fd34f1867d3563f3fdf8e5f409a42a29cb5a602d8be143c512374ba51617c8580b5bfe89a6703fb2ddbef020e324b
+  metadata.gz: 6db71b9dfddb72596ec78ec9dd1c012b541e384680d71763cc2e753f11df2c734e9d74f2b30613f00f3aaf5697fd5d4b28125fe728294aa1220986363545a495
+  data.tar.gz: 3dafbf56819d71a56d6b346002feba0bfc5b84eb019e618d3607dfb857996d0e0132c24086198c95b73dfcffa6002a3bb82366ea14be5bb3bb136284c6292b85

data/README.md

@@ -270,16 +270,20 @@ end
 
 `tenant_scope` runs as each engine model's `default_scope` (use `unscoped` to bypass). `tenant_scope_columns` is appended to every engine uniqueness validation. Adding the tenant columns and composite unique indexes lives in your host migrations. Both defaults (`nil`, `[]`) are no-ops.
 
-Two further hooks let a host shape the runs index page without overriding the controller or the view:
+One hook lets a host apply run-history retention everywhere the engine lists, counts, or traverses runs, without overriding controllers or views:
 
 ```ruby
 CompletionKit.configure do |config|
-  config.runs_index_scope = -> { where(created_at: 90.days.ago..) }
-  config.runs_index_footer_partial = "runs/retention_notice"
+  config.runs_display_scope = -> { where(created_at: 90.days.ago..) }
+  config.runs_display_footer_partial = "runs/retention_notice"
 end
 ```
 
-`runs_index_scope` is a callable evaluated against the runs index relation, in the same bare-`where` style as `tenant_scope` (it runs via `instance_exec`, so write it zero-arg with the relation as `self`, like a Rails `scope` lambda). It must return a relation: a callable that returns `nil` or anything non-chainable raises when the list renders. Use it to apply list-only filters such as run-history retention, rather than a global `default_scope` that would also null `Run` associations everywhere they are traversed. `runs_index_footer_partial` names a partial rendered below the list; it receives the shown runs as a `runs` local. Use it for a notice like "older runs are hidden, upgrade to see them" — your host owns the retention rule in `runs_index_scope`, so it computes the hidden count itself. Both default to `nil` (no-ops), leaving standalone behaviour unchanged.
+`runs_display_scope` is a callable evaluated against a `Run` relation, in the same bare-`where` style as `tenant_scope` (it runs via `instance_exec`, so write it zero-arg with the relation as `self`, like a Rails `scope` lambda). It must return a relation: a callable that returns `nil` or anything non-chainable raises when a list renders. The engine applies it through `Run.display_scoped` at every run list and count it owns (the runs index, prompt and dataset show pages, the compare picker, new-run tag defaults, the v1 API index and its `X-Total-Count`, the MCP `runs_list` tool, the API reference recent-runs panel, and provider-credential usage stats) and through `Run.visible_run_ids` for child records that traverse runs (the metric trust-panel sample and the agreement examples shown on a metric page). Use it for list-only retention rather than a global `default_scope`, which would null `Run` associations everywhere they are traversed.
+
+Deliberately exempt, because they must still see every run: id-addressed single-run lookups (`runs#show`, the MCP `runs_get` tool, the v1 API show), delete-confirmation cascade counts, the auto-generated run-name counter, and the judge few-shot seeding that learns from corrected examples even on hidden runs.
+
+`runs_display_footer_partial` names a partial rendered below the runs list on the index and the prompt and dataset show pages; it receives the shown runs as a `runs` local. Use it for a notice like "older runs are hidden, upgrade to see them" — your host owns the retention rule in `runs_display_scope`, so it computes the hidden count itself. Both default to `nil` (no-ops), leaving standalone behaviour unchanged.
 
 ## Contributing
 

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -5,7 +5,7 @@ module CompletionKit
         before_action :set_run, only: [:show, :update, :destroy, :generate, :retry_failures, :rerun, :regrade, :compare]
 
         def index
-          scope = Run.includes(:tags)
+          scope = Run.includes(:tags).display_scoped
           scope = scope.where(status: params[:status]) if params[:status].present?
           scope = scope.where(prompt_id: params[:prompt_id]) if params[:prompt_id].present?
           scope = scope.where(dataset_id: params[:dataset_id]) if params[:dataset_id].present?

data/app/controllers/completion_kit/api_reference_controller.rb

@@ -2,7 +2,7 @@ module CompletionKit
   class ApiReferenceController < ApplicationController
     def index
       @published_prompts = Prompt.current_versions.order(name: :asc)
-      @recent_runs = Run.includes(:prompt).order(created_at: :desc).limit(10)
+      @recent_runs = Run.includes(:prompt).display_scoped.order(created_at: :desc).limit(10)
       @datasets = Dataset.order(name: :asc)
       @metrics = Metric.order(name: :asc)
       @metric_groups = MetricGroup.includes(:metrics).order(name: :asc)

data/app/controllers/completion_kit/datasets_controller.rb

@@ -8,7 +8,7 @@ module CompletionKit
     end
 
     def show
-      @runs = @dataset.runs.includes(:prompt, :responses).order(created_at: :desc)
+      @runs = @dataset.runs.includes(:prompt, :responses).order(created_at: :desc).display_scoped
       respond_to do |format|
         format.html
         format.csv do

data/app/controllers/completion_kit/prompts_controller.rb

@@ -11,6 +11,7 @@ module CompletionKit
       @runs = Run.where(prompt_id: @prompt.family_versions.select(:id))
                  .includes(:prompt, :dataset, responses: :reviews)
                  .order(created_at: :desc)
+                 .display_scoped
     end
     
     def new

data/app/controllers/completion_kit/runs_controller.rb

@@ -5,9 +5,7 @@ module CompletionKit
     before_action :load_form_collections, only: [:new, :edit, :create, :update]
 
     def index
-      scope = Run.includes(:prompt, :dataset, :tags, responses: :reviews).order(created_at: :desc)
-      index_scope = CompletionKit.config.runs_index_scope
-      scope = scope.instance_exec(&index_scope) if index_scope
+      scope = Run.includes(:prompt, :dataset, :tags, responses: :reviews).order(created_at: :desc).display_scoped
       @runs = apply_tag_filter(scope)
     end
 
@@ -33,7 +31,7 @@ module CompletionKit
       @run = Run.new(prompt_id: params[:prompt_id])
       prompt = Prompt.find_by(id: @run.prompt_id)
       if prompt
-        last_run = Run.where(prompt_id: prompt.family_versions.ids).order(created_at: :desc).first
+        last_run = Run.where(prompt_id: prompt.family_versions.ids).display_scoped.order(created_at: :desc).first
         @run.tag_names = last_run.tag_names if last_run
       end
     end
@@ -86,6 +84,7 @@ module CompletionKit
       if other_id.blank?
         @other_runs = Run.where(dataset_id: @run.dataset_id, prompt_id: @run.prompt_id)
                           .where.not(id: @run.id)
+                          .display_scoped
                           .order(created_at: :desc)
                           .limit(50)
         return render(:compare_picker)

data/app/helpers/completion_kit/application_helper.rb

@@ -1,5 +1,11 @@
 module CompletionKit
   module ApplicationHelper
+    def ck_runs_display_footer(runs)
+      partial = CompletionKit.config.runs_display_footer_partial
+      return unless partial
+      render partial, runs: runs
+    end
+
     def ck_button_classes(tone = :dark, variant: :solid)
       base = "ck-button"
 

data/app/models/completion_kit/provider_credential.rb

@@ -65,7 +65,7 @@ module CompletionKit
     def judge_count
       model_ids = Model.where(provider: provider).pluck(:model_id)
       return 0 if model_ids.empty?
-      Run.where(judge_model: model_ids).distinct.count(:judge_model)
+      Run.where(judge_model: model_ids).display_scoped.distinct.count(:judge_model)
     end
 
     def last_used_at
@@ -74,6 +74,7 @@ module CompletionKit
       prompt_scope = Prompt.where(llm_model: model_ids).select(:id)
       Run.where("prompt_id IN (:prompts) OR judge_model IN (:models)",
                 prompts: prompt_scope, models: model_ids)
+         .display_scoped
          .where.not(status: "pending")
          .maximum(:created_at)
     end

data/app/models/completion_kit/run.rb

@@ -21,6 +21,15 @@ module CompletionKit
     before_validation :set_default_status, on: :create
     before_validation :set_auto_name, on: :create
 
+    def self.display_scoped
+      filter = CompletionKit.config.runs_display_scope
+      filter ? all.instance_exec(&filter) : all
+    end
+
+    def self.visible_run_ids
+      display_scoped.select(:id)
+    end
+
     # A judge-only run grades a pre-existing column on the dataset instead of
     # generating new outputs. No prompt is attached; the response text is read
     # from row[output_column]; no LLM generation happens.

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -57,7 +57,7 @@ module CompletionKit
       }.freeze
 
       def self.list(_args)
-        text_result(Run.order(created_at: :desc).map(&:as_json))
+        text_result(Run.display_scoped.order(created_at: :desc).map(&:as_json))
       end
 
       def self.get(args)

data/app/services/completion_kit/metric_agreement_examples.rb

@@ -29,7 +29,7 @@ module CompletionKit
     end
 
     def agreements_for(metric, verdict:, limit:)
-      base = Agreement.where(metric_id: metric.id, verdict: verdict)
+      base = Agreement.where(metric_id: metric.id, verdict: verdict, run_id: Run.visible_run_ids)
       current_version = MetricVersion.current.find_by(metric_id: metric.id)
       scoped = current_version ? base.where(metric_version_id: current_version.id) : base
       effective = scoped.exists? ? scoped : base

data/app/services/completion_kit/metric_agreement_stats.rb

@@ -49,7 +49,7 @@ module CompletionKit
     end
 
     def call
-      scope = Agreement.where(metric_id: @metric.id)
+      scope = Agreement.where(metric_id: @metric.id, run_id: Run.visible_run_ids)
       if @metric_version
         scope = scope.where(metric_version_id: @metric_version.id)
       elsif !@all_versions

data/app/views/completion_kit/agreements/_trust_panel.html.erb

@@ -4,8 +4,8 @@
 <% current_metric_version = metric && CompletionKit::MetricVersion.current.find_by(metric_id: metric.id) %>
 <% target_response = if (stats.sample_size.zero? || stats.counter_only?) && metric && current_metric_version
      created_by = CompletionKit.config.username.presence || "operator"
-     verdicted_ids = CompletionKit::Agreement.where(metric_id: metric.id, created_by: created_by, metric_version_id: current_metric_version.id).pluck(:response_id)
-     CompletionKit::Response.joins(:reviews)
+     verdicted_ids = CompletionKit::Agreement.where(metric_id: metric.id, created_by: created_by, metric_version_id: current_metric_version.id, run_id: CompletionKit::Run.visible_run_ids).pluck(:response_id)
+     CompletionKit::Response.where(run_id: CompletionKit::Run.visible_run_ids).joins(:reviews)
        .where(reviews: { metric_id: metric.id, metric_version_id: current_metric_version.id })
        .where.not(reviews: { ai_score: nil })
        .where.not(id: verdicted_ids)

data/app/views/completion_kit/datasets/index.html.erb

@@ -36,7 +36,7 @@
             <% end %>
           </td>
           <td data-label="Rows"><%= dataset.row_count %></td>
-          <td data-label="Used in"><%= dataset.runs.count %></td>
+          <td data-label="Used in"><%= dataset.runs.display_scoped.count %></td>
           <td data-label="Created" class="ck-meta-copy"><time datetime="<%= dataset.created_at.iso8601 %>"><%= dataset.created_at.strftime("%b %-d, %Y") %></time></td>
           <td class="ck-results-table__arrow">&rarr;</td>
         </tr>

data/app/views/completion_kit/datasets/show.html.erb

@@ -71,3 +71,5 @@
     <%= render "completion_kit/runs/table", runs: @runs %>
   </section>
 <% end %>
+
+<%= ck_runs_display_footer(@runs) %>

data/app/views/completion_kit/prompts/index.html.erb

@@ -51,8 +51,8 @@
             <% end %>
           </td>
           <td><span class="ck-chip"><%= prompt.llm_model %></span></td>
-          <% family_runs = CompletionKit::Run.where(prompt_id: prompt.family_versions.select(:id)) %>
-          <% current_version_runs = prompt.runs.includes(responses: :reviews) %>
+          <% family_runs = CompletionKit::Run.where(prompt_id: prompt.family_versions.select(:id)).display_scoped %>
+          <% current_version_runs = prompt.runs.display_scoped.includes(responses: :reviews) %>
           <% best_score = current_version_runs.map(&:avg_score).compact.max %>
           <td>
             <% if best_score %>

data/app/views/completion_kit/prompts/show.html.erb

@@ -64,7 +64,7 @@
       </thead>
       <tbody>
         <% versions.each do |v| %>
-          <% best_score = v.runs.map(&:avg_score).compact.max %>
+          <% best_score = v.runs.display_scoped.map(&:avg_score).compact.max %>
           <% pred = predecessor_of[v] %>
           <tr class="<%= "ck-results-table__row--active" if v.id == @prompt.id %>" onclick="window.location='<%= prompt_path(v) %>'" style="cursor: pointer;">
             <td>
@@ -143,6 +143,8 @@
   </section>
 <% end %>
 
+<%= ck_runs_display_footer(@runs) %>
+
 <% suggestions = CompletionKit::Suggestion.where(prompt_id: @prompt.family_versions.select(:id)).order(created_at: :desc) %>
 <% if suggestions.any? %>
   <section class="ck-card--spaced">

data/app/views/completion_kit/runs/index.html.erb

@@ -23,6 +23,4 @@
   <div class="ck-empty">No runs yet.&ensp;<%= link_to "Create your first run →", new_run_path, class: "ck-link" %></div>
 <% end %>
 
-<% if CompletionKit.config.runs_index_footer_partial %>
-  <%= render CompletionKit.config.runs_index_footer_partial, runs: @runs %>
-<% end %>
+<%= ck_runs_display_footer(@runs) %>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.16.3"
+  VERSION = "0.17.0"
 end

data/lib/completion_kit.rb

@@ -10,7 +10,7 @@ module CompletionKit
     attr_accessor :username, :password, :auth_strategy, :api_token
     attr_accessor :tenant_scope, :tenant_scope_columns
     attr_accessor :api_reference_authentication_partial
-    attr_accessor :runs_index_scope, :runs_index_footer_partial
+    attr_accessor :runs_display_scope, :runs_display_footer_partial
     attr_accessor :api_rate_limit, :web_rate_limit
     attr_accessor :allow_loopback_endpoints
     attr_accessor :judge_agreement_enabled

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.16.3
+  version: 0.17.0
 platform: ruby
 authors:
 - Damien Bastin