raif 1.4.0 → 1.5.0

data/app/assets/javascript/raif/admin/copy_to_clipboard_controller.js

@@ -0,0 +1,132 @@
+import { Controller } from "@hotwired/stimulus";
+
+const COPY_ICON = `<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16" aria-hidden="true">
+<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>
+<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>
+</svg>`;
+
+const CHECK_ICON = `<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16" aria-hidden="true">
+<path d="M13.854 3.646a.5.5 0 0 1 0 .708l-7 7a.5.5 0 0 1-.708 0l-3.5-3.5a.5.5 0 1 1 .708-.708L6.5 10.293l6.646-6.647a.5.5 0 0 1 .708 0z"/>
+</svg>`;
+
+export default class extends Controller {
+  connect() {
+    this.attachAll();
+
+    this.observer = new MutationObserver((mutations) => {
+      for (const mutation of mutations) {
+        for (const node of mutation.addedNodes) {
+          if (!(node instanceof Element)) continue;
+          if (node.matches && node.matches("pre")) {
+            this.attach(node);
+          }
+          if (node.querySelectorAll) {
+            node.querySelectorAll("pre").forEach((pre) => this.attach(pre));
+          }
+        }
+      }
+    });
+
+    this.observer.observe(this.element, { childList: true, subtree: true });
+  }
+
+  disconnect() {
+    if (this.observer) {
+      this.observer.disconnect();
+      this.observer = null;
+    }
+  }
+
+  attachAll() {
+    this.element.querySelectorAll("pre").forEach((pre) => this.attach(pre));
+  }
+
+  attach(pre) {
+    if (pre.dataset.raifCopyAttached === "true") return;
+    if (pre.closest("[data-raif-copy-skip]")) return;
+    pre.dataset.raifCopyAttached = "true";
+
+    const wrapper = document.createElement("div");
+    wrapper.className = "raif-copyable-pre";
+    const classesToCopy = ["mb-0", "mb-2", "mb-3", "mt-1", "mt-2", "mt-3"];
+    classesToCopy.forEach((cls) => {
+      if (pre.classList.contains(cls)) {
+        wrapper.classList.add(cls);
+        pre.classList.remove(cls);
+      }
+    });
+
+    pre.parentNode.insertBefore(wrapper, pre);
+    wrapper.appendChild(pre);
+
+    const button = document.createElement("button");
+    button.type = "button";
+    button.className = "btn btn-sm btn-outline-secondary raif-copy-btn";
+    button.setAttribute("aria-label", "Copy to clipboard");
+    button.setAttribute("title", "Copy to clipboard");
+    button.innerHTML = COPY_ICON;
+    button.addEventListener("click", (event) => {
+      event.preventDefault();
+      this.copy(pre, button);
+    });
+
+    wrapper.appendChild(button);
+  }
+
+  async copy(pre, button) {
+    const text = pre.textContent;
+    let succeeded = false;
+
+    if (navigator.clipboard && window.isSecureContext) {
+      try {
+        await navigator.clipboard.writeText(text);
+        succeeded = true;
+      } catch (_err) {
+        succeeded = false;
+      }
+    }
+
+    if (!succeeded) {
+      succeeded = this.fallbackCopy(text);
+    }
+
+    this.showFeedback(button, succeeded);
+  }
+
+  fallbackCopy(text) {
+    const textarea = document.createElement("textarea");
+    textarea.value = text;
+    textarea.setAttribute("readonly", "");
+    textarea.style.position = "absolute";
+    textarea.style.left = "-9999px";
+    document.body.appendChild(textarea);
+    textarea.select();
+    let ok = false;
+    try {
+      ok = document.execCommand("copy");
+    } catch (_err) {
+      ok = false;
+    }
+    document.body.removeChild(textarea);
+    return ok;
+  }
+
+  showFeedback(button, succeeded) {
+    const originalHTML = button.innerHTML;
+    const originalTitle = button.getAttribute("title");
+    const originalAriaLabel = button.getAttribute("aria-label");
+    const message = succeeded ? "Copied" : "Copy failed";
+    button.innerHTML = succeeded ? CHECK_ICON : "!";
+    button.setAttribute("title", message);
+    button.setAttribute("aria-label", message);
+    button.classList.add(succeeded ? "raif-copy-btn-success" : "raif-copy-btn-error");
+
+    clearTimeout(button._raifCopyTimer);
+    button._raifCopyTimer = setTimeout(() => {
+      button.innerHTML = originalHTML;
+      button.setAttribute("title", originalTitle || "Copy to clipboard");
+      button.setAttribute("aria-label", originalAriaLabel || "Copy to clipboard");
+      button.classList.remove("raif-copy-btn-success", "raif-copy-btn-error");
+    }, 1500);
+  }
+}

data/app/assets/javascript/raif/admin/cost_estimate_controller.js

@@ -0,0 +1,80 @@
+import { Controller } from "@hotwired/stimulus";
+
+export default class extends Controller {
+  static targets = ["checkbox", "modelSelect", "judgeModelSelect", "judgeTypeSelect", "estimate"];
+  static values = { pricing: Object };
+
+  calculate() {
+    if (!this.hasModelSelectTarget || !this.hasEstimateTarget) return;
+
+    const modelKey = this.modelSelectTarget.value;
+    const pricing = this.pricingValue[modelKey];
+
+    if (!pricing) {
+      this.hideEstimate();
+      return;
+    }
+
+    const selected = this.checkboxTargets.filter((cb) => cb.checked);
+
+    if (selected.length === 0) {
+      this.hideEstimate();
+      return;
+    }
+
+    let totalPromptTokens = 0;
+    let totalCompletionTokens = 0;
+
+    selected.forEach((cb) => {
+      totalPromptTokens += parseInt(cb.dataset.promptTokens || 0, 10);
+      totalCompletionTokens += parseInt(cb.dataset.completionTokens || 0, 10);
+    });
+
+    const taskInputCost = totalPromptTokens * pricing.input;
+    const taskOutputCost = totalCompletionTokens * pricing.output;
+    const taskTotalCost = taskInputCost + taskOutputCost;
+
+    let judgeCost = null;
+    const judgeType = this.hasJudgeTypeSelectTarget ? this.judgeTypeSelectTarget.value : "";
+    if (judgeType) {
+      const judgeModelKey = this.hasJudgeModelSelectTarget ? this.judgeModelSelectTarget.value : "";
+      const judgePricing = this.pricingValue[judgeModelKey];
+
+      if (judgePricing) {
+        // Rough estimate: judge input ≈ prompt + completion tokens from the task, judge output ≈ 500 tokens
+        const judgeInputTokens = totalPromptTokens + totalCompletionTokens;
+        const judgeOutputTokens = selected.length * 500;
+        judgeCost = judgeInputTokens * judgePricing.input + judgeOutputTokens * judgePricing.output;
+      }
+    }
+
+    const totalCost = taskTotalCost + (judgeCost || 0);
+    const avgTokens = Math.round((totalPromptTokens + totalCompletionTokens) / selected.length);
+
+    let html = `<strong>Estimated cost: ${this.formatCurrency(totalCost)}</strong>`;
+    html += `<br><small class="text-muted">Task runs: ${this.formatCurrency(taskTotalCost)} (${selected.length} tasks &times; ~${this.formatNumber(avgTokens)} tokens avg)`;
+    if (judgeCost !== null) {
+      html += `<br>Judge runs: ~${this.formatCurrency(judgeCost)}`;
+    }
+    html += "</small>";
+
+    this.estimateTarget.innerHTML = html;
+    this.estimateTarget.classList.remove("d-none");
+  }
+
+  hideEstimate() {
+    this.estimateTarget.innerHTML = "";
+    this.estimateTarget.classList.add("d-none");
+  }
+
+  formatCurrency(amount) {
+    if (amount < 0.01) {
+      return `$${amount.toFixed(4)}`;
+    }
+    return `$${amount.toFixed(2)}`;
+  }
+
+  formatNumber(num) {
+    return num.toLocaleString();
+  }
+}

data/app/assets/javascript/raif/admin/judge_config_controller.js

@@ -0,0 +1,23 @@
+import { Controller } from "@hotwired/stimulus";
+
+export default class extends Controller {
+  static targets = ["binary", "scored", "comparative", "summarization", "sharedOptions"];
+
+  toggle(event) {
+    const selectedType = event.target.value;
+    const typeMap = {
+      "Raif::Evals::LlmJudges::Binary": this.binaryTarget,
+      "Raif::Evals::LlmJudges::Scored": this.scoredTarget,
+      "Raif::Evals::LlmJudges::Comparative": this.comparativeTarget,
+      "Raif::Evals::LlmJudges::Summarization": this.summarizationTarget,
+    };
+
+    Object.values(typeMap).forEach((el) => el.classList.add("d-none"));
+    this.sharedOptionsTarget.classList.add("d-none");
+
+    if (typeMap[selectedType]) {
+      typeMap[selectedType].classList.remove("d-none");
+      this.sharedOptionsTarget.classList.remove("d-none");
+    }
+  }
+}

data/app/assets/javascript/raif/admin/select_all_checkboxes_controller.js

@@ -0,0 +1,33 @@
+import { Controller } from "@hotwired/stimulus";
+
+export default class extends Controller {
+  static targets = ["selectAll", "checkbox", "selectedCount", "batchButton"];
+  static values = { batchButtonLabel: String };
+
+  connect() {
+    if (this.hasBatchButtonTarget) {
+      this.batchButtonLabelValue = this.batchButtonTarget.textContent.trim();
+    }
+  }
+
+  toggle() {
+    const checked = this.selectAllTarget.checked;
+    this.checkboxTargets.forEach((cb) => (cb.checked = checked));
+    this.updateCount();
+    this.dispatch("toggled");
+  }
+
+  updateCount() {
+    const count = this.checkboxTargets.filter((cb) => cb.checked).length;
+    this.selectedCountTargets.forEach((el) => {
+      el.textContent = count;
+    });
+    if (this.hasBatchButtonTarget) {
+      this.batchButtonTarget.disabled = count === 0;
+      this.batchButtonTarget.textContent =
+        count > 0
+          ? `${this.batchButtonLabelValue} (${count})`
+          : this.batchButtonLabelValue;
+    }
+  }
+}

data/app/assets/javascript/raif/admin/sortable_table_controller.js

@@ -0,0 +1,51 @@
+import { Controller } from "@hotwired/stimulus";
+
+export default class extends Controller {
+  static targets = ["header", "body"];
+
+  sort(event) {
+    const th = event.currentTarget;
+    const colIndex = parseInt(th.dataset.colIndex, 10);
+    const type = th.dataset.sortType || "string";
+    const currentDir = th.dataset.sortDir;
+    const newDir = currentDir === "asc" ? "desc" : "asc";
+
+    // Reset all headers
+    this.headerTargets.forEach((header) => {
+      header.dataset.sortDir = "";
+      header.querySelector(".sort-indicator")?.remove();
+    });
+
+    th.dataset.sortDir = newDir;
+
+    // Add sort indicator
+    const indicator = document.createElement("span");
+    indicator.className = "sort-indicator ms-1";
+    indicator.textContent = newDir === "asc" ? "\u25B2" : "\u25BC";
+    th.appendChild(indicator);
+
+    const rows = Array.from(this.bodyTarget.querySelectorAll("tr"));
+    rows.sort((a, b) => {
+      const aCell = a.children[colIndex];
+      const bCell = b.children[colIndex];
+      if (!aCell || !bCell) return 0;
+
+      let aVal = (aCell.dataset.sortValue || aCell.textContent).trim();
+      let bVal = (bCell.dataset.sortValue || bCell.textContent).trim();
+
+      if (type === "number") {
+        aVal = parseFloat(aVal) || 0;
+        bVal = parseFloat(bVal) || 0;
+      } else {
+        aVal = aVal.toLowerCase();
+        bVal = bVal.toLowerCase();
+      }
+
+      if (aVal < bVal) return newDir === "asc" ? -1 : 1;
+      if (aVal > bVal) return newDir === "asc" ? 1 : -1;
+      return 0;
+    });
+
+    rows.forEach((row) => this.bodyTarget.appendChild(row));
+  }
+}

data/app/assets/javascript/raif/admin/table_search_controller.js

@@ -0,0 +1,15 @@
+import { Controller } from "@hotwired/stimulus";
+
+export default class extends Controller {
+  static targets = ["input", "row"];
+
+  filter() {
+    const terms = this.inputTarget.value.toLowerCase().trim().split(/\s+/).filter(Boolean);
+
+    this.rowTargets.forEach((row) => {
+      const text = row.dataset.searchable || row.textContent.toLowerCase();
+      const visible = terms.length === 0 || terms.every((term) => text.includes(term));
+      row.style.display = visible ? "" : "none";
+    });
+  }
+}

data/app/assets/javascript/raif/admin/tom_select_controller.js

@@ -0,0 +1,33 @@
+import { Controller } from "@hotwired/stimulus";
+
+export default class extends Controller {
+  connect() {
+    if (typeof TomSelect === "undefined") return;
+
+    const isMultiple = this.element.hasAttribute("multiple");
+    const blankOption = this.element.querySelector('option[value=""]');
+    const placeholder = blankOption ? blankOption.textContent : "";
+
+    const options = {
+      allowEmptyOption: !isMultiple,
+      placeholder: placeholder,
+      plugins: {},
+    };
+
+    if (isMultiple) {
+      options.plugins.remove_button = { title: "Remove" };
+    }
+
+    this.tomSelect = new TomSelect(this.element, options);
+
+    if (blankOption && !isMultiple) {
+      blankOption.textContent = "";
+    }
+  }
+
+  disconnect() {
+    if (this.tomSelect) {
+      this.tomSelect.destroy();
+    }
+  }
+}

data/app/assets/javascript/raif_admin.js

@@ -0,0 +1,23 @@
+import "@hotwired/turbo-rails"
+import { application } from "controllers/application"
+
+import JudgeConfigController from "raif/admin/judge_config_controller"
+application.register("raif--judge-config", JudgeConfigController)
+
+import SelectAllCheckboxesController from "raif/admin/select_all_checkboxes_controller"
+application.register("raif--select-all-checkboxes", SelectAllCheckboxesController)
+
+import CostEstimateController from "raif/admin/cost_estimate_controller"
+application.register("raif--cost-estimate", CostEstimateController)
+
+import TomSelectController from "raif/admin/tom_select_controller"
+application.register("raif--tom-select", TomSelectController)
+
+import TableSearchController from "raif/admin/table_search_controller"
+application.register("raif--table-search", TableSearchController)
+
+import SortableTableController from "raif/admin/sortable_table_controller"
+application.register("raif--sortable-table", SortableTableController)
+
+import CopyToClipboardController from "raif/admin/copy_to_clipboard_controller"
+application.register("raif--copy-to-clipboard", CopyToClipboardController)

data/app/assets/javascript/raif_admin_sprockets.js

@@ -0,0 +1,24 @@
+// Sprockets entry point for Raif Admin.
+// This is bundled by esbuild into app/assets/builds/raif_admin_sprockets.js
+// and provides a self-contained Stimulus setup for host apps that don't use importmaps.
+import { Application, Controller } from "@hotwired/stimulus"
+
+// Make Controller available globally so the imported controller files can extend it
+window.Stimulus = { Controller: Controller }
+
+const application = Application.start()
+
+import JudgeConfigController from "./raif/admin/judge_config_controller"
+application.register("raif--judge-config", JudgeConfigController)
+
+import SelectAllCheckboxesController from "./raif/admin/select_all_checkboxes_controller"
+application.register("raif--select-all-checkboxes", SelectAllCheckboxesController)
+
+import CostEstimateController from "./raif/admin/cost_estimate_controller"
+application.register("raif--cost-estimate", CostEstimateController)
+
+import TomSelectController from "./raif/admin/tom_select_controller"
+application.register("raif--tom-select", TomSelectController)
+
+import CopyToClipboardController from "./raif/admin/copy_to_clipboard_controller"
+application.register("raif--copy-to-clipboard", CopyToClipboardController)

data/app/assets/stylesheets/raif_admin.scss

@@ -157,7 +157,7 @@ body.raif-admin {
     border-radius: $radius;
     box-shadow: 0 4px 6px rgba(50, 50, 93, 0.11), 0 1px 3px rgba(0, 0, 0, 0.08);
     background-color: white;
-    overflow: hidden;
+    overflow-x: auto;
   }
 
   // Badges
@@ -220,6 +220,55 @@ body.raif-admin {
     white-space: pre-wrap;
   }
 
+  // Copy-to-clipboard wrapper around pre blocks
+  .raif-copyable-pre {
+    position: relative;
+
+    > pre {
+      margin-bottom: 0;
+    }
+
+    .raif-copy-btn {
+      position: absolute;
+      top: 0.5rem;
+      right: 0.5rem;
+      padding: 0.25rem 0.5rem;
+      line-height: 1;
+      background-color: white;
+      box-shadow: 0 1px 2px rgba(50, 50, 93, 0.12);
+      opacity: 0;
+      transition: opacity 0.15s ease, background-color 0.15s ease, color 0.15s ease;
+
+      svg {
+        display: block;
+      }
+
+      &:focus,
+      &:focus-visible {
+        opacity: 1;
+      }
+
+      &.raif-copy-btn-success {
+        opacity: 1;
+        color: white;
+        background-color: $accent-color;
+        border-color: $accent-color;
+      }
+
+      &.raif-copy-btn-error {
+        opacity: 1;
+        color: white;
+        background-color: $danger-color;
+        border-color: $danger-color;
+      }
+    }
+
+    &:hover .raif-copy-btn,
+    &:focus-within .raif-copy-btn {
+      opacity: 1;
+    }
+  }
+
   // List groups
   .list-group-item {
     border-color: $border-color;

data/app/controllers/raif/admin/agents_controller.rb

@@ -6,7 +6,33 @@ module Raif
       include Pagy::Backend
 
       def index
-        @pagy, @agents = pagy(Raif::Agent.order(created_at: :desc))
+        @agent_types = Raif::Agent.distinct.pluck(:type)
+        @selected_type = params[:agent_type].present? ? params[:agent_type] : "all"
+
+        @selected_status = params[:status].present? ? params[:status].to_sym : :all
+
+        @selected_llm_model_key = params[:llm_model_key].presence
+        @llm_model_keys = Raif::Agent.distinct.order(:llm_model_key).pluck(:llm_model_key)
+
+        agents = Raif::Agent.order(created_at: :desc)
+        agents = agents.where(type: @selected_type) if @selected_type.present? && @selected_type != "all"
+
+        if @selected_status.present? && @selected_status != :all
+          case @selected_status
+          when :completed
+            agents = agents.completed
+          when :failed
+            agents = agents.failed
+          when :running
+            agents = agents.started.where(completed_at: nil, failed_at: nil)
+          when :pending
+            agents = agents.where(started_at: nil)
+          end
+        end
+
+        agents = agents.where(llm_model_key: @selected_llm_model_key) if @selected_llm_model_key.present?
+
+        @pagy, @agents = pagy(agents)
       end
 
       def show

data/app/controllers/raif/admin/configs_controller.rb

@@ -33,6 +33,7 @@ module Raif
           { key: "evals_default_llm_judge_model_key", value: @config.evals_default_llm_judge_model_key },
           { key: "evals_verbose_output", value: @config.evals_verbose_output },
           { key: "google_api_key", value: mask_sensitive_value(@config.google_api_key) },
+          { key: "google_embedding_models_enabled", value: @config.google_embedding_models_enabled },
           { key: "google_models_enabled", value: @config.google_models_enabled },
           { key: "llm_api_requests_enabled", value: @config.llm_api_requests_enabled },
           { key: "llm_request_max_retries", value: @config.llm_request_max_retries },

data/app/controllers/raif/admin/llms_controller.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Raif
+  module Admin
+    class LlmsController < Raif::Admin::ApplicationController
+
+      def index
+        @llms = Raif.llm_registry.map do |_key, config|
+          llm_class = config[:llm_class]
+          llm_class.new(**config.except(:llm_class))
+        end
+
+        @llms.sort_by!(&:name)
+
+        @provider_names = @llms.map { |llm| llm.class.name.demodulize }.uniq.sort
+        @llm_names = @llms.map(&:name).sort
+
+        @selected_providers = Array(params[:providers]).reject(&:blank?)
+        @selected_names = Array(params[:names]).reject(&:blank?)
+
+        @llms = @llms.select { |llm| @selected_providers.include?(llm.class.name.demodulize) } if @selected_providers.present?
+        @llms = @llms.select { |llm| @selected_names.include?(llm.name) } if @selected_names.present?
+      end
+
+    end
+  end
+end

data/app/controllers/raif/admin/model_completions_controller.rb

@@ -7,6 +7,8 @@ module Raif
 
       def index
         @selected_status = params[:status].present? ? params[:status].to_sym : :all
+        @selected_llm_model_key = params[:llm_model_key].presence
+        @llm_model_keys = Raif::ModelCompletion.distinct.order(:llm_model_key).pluck(:llm_model_key)
 
         model_completions = Raif::ModelCompletion.order(created_at: :desc)
 
@@ -23,6 +25,10 @@ module Raif
           end
         end
 
+        if @selected_llm_model_key.present?
+          model_completions = model_completions.where(llm_model_key: @selected_llm_model_key)
+        end
+
         @pagy, @model_completions = pagy(model_completions)
       end
 

data/app/controllers/raif/admin/prompt_studio/agents_controller.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Raif
+  module Admin
+    module PromptStudio
+      class AgentsController < BaseController
+        def index
+          @agent_types = Raif::Agent.distinct.pluck(:type).sort
+          @selected_type = params[:agent_type] if params[:agent_type].present?
+          @llm_model_keys = Raif::Agent.where(type: @selected_type).distinct.pluck(:llm_model_key).compact.sort if @selected_type.present?
+
+          if @selected_type.present?
+            agents = apply_filters(Raif::Agent.where(type: @selected_type)).order(created_at: :desc)
+            @pagy, @agents = pagy(agents)
+          end
+        end
+
+        def show
+          @agent = Raif::Agent.find(params[:id])
+          @comparison = build_prompt_comparison(@agent)
+        end
+      end
+    end
+  end
+end

data/app/controllers/raif/admin/prompt_studio/base_controller.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Raif
+  module Admin
+    module PromptStudio
+      class BaseController < Raif::Admin::ApplicationController
+        include Pagy::Backend
+
+      private
+
+        def build_prompt_comparison(record)
+          Raif::PromptStudioComparisonBuilder.build(record)
+        end
+
+        def apply_filters(scope)
+          scope = scope.where("#{scope.table_name}.created_at >= ?", Time.zone.parse(params[:created_after])) if params[:created_after].present?
+          scope = scope.where(
+            "#{scope.table_name}.created_at <= ?",
+            Time.zone.parse(params[:created_before]).end_of_day
+          ) if params[:created_before].present?
+          scope = scope.where(llm_model_key: params[:llm_model_key]) if params[:llm_model_key].present?
+          scope
+        end
+
+        helper_method :prompt_studio_runs_enabled?
+        def prompt_studio_runs_enabled?
+          Raif.config.prompt_studio_runs_enabled
+        end
+      end
+    end
+  end
+end