completion-kit 0.1.0.rc1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cdecf15deb685a524186a2bd4ba48268e10475da0d2cc2914969318893268f70
+  data.tar.gz: ce395abf147434f9a825f79902e47b171b074038d20c06bf7b86b80ba70eaa00
+SHA512:
+  metadata.gz: 9d13fd1d1863c87ca0f7ed78eb1853ac921ff39862b84c678e5b5c9977f1832edcfb50c52b2371f2dc88ad6360be22a78d39ef53385239939a1ac9297df444ff
+  data.tar.gz: bf95df5b178ccfe455350d216a894836a85cfeff5c117d0206b87211ef638c1d690790350168b5a9ad84e8e3895add8c89acdea98738a1c0d2025115679d939f

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,192 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/homemade-software-inc/completion-kit/main/docs/logo.png" alt="CompletionKit logo" width="120" />
+</p>
+
+# CompletionKit
+
+[![CI](https://github.com/homemade-software-inc/completion-kit/actions/workflows/ci.yml/badge.svg)](https://github.com/homemade-software-inc/completion-kit/actions/workflows/ci.yml)
+![coverage](https://img.shields.io/badge/coverage-100%25-brightgreen)
+![dependencies](https://img.shields.io/badge/dependencies-7-blue)
+[![Dependabot](https://img.shields.io/badge/dependabot-enabled-blue?logo=dependabot)](https://github.com/homemade-software-inc/completion-kit/network/updates)
+
+You need to know whether your prompts produce the output you expect, consistently, across real data. CompletionKit gives you that, inside your Rails app.
+
+Mount the engine, bring your prompts and datasets, and every input runs through a model you pick. Each output is scored against your own metrics and rubrics by an LLM-as-judge. When you change a prompt, re-run the same dataset and see exactly what got better and what broke — and when the scores tell you something's off, CompletionKit can suggest an improved version of the prompt based on the reviews, which you inspect as a diff and apply as a new version.
+
+Drive it from the web UI, from the REST API, or from Claude Code and other MCP-aware agents via the built-in Model Context Protocol server. All three share the same state — your prompts, runs, datasets, and scores are one source of truth.
+
+It's the difference between "this prompt seems to work" and "this prompt scores 4.3 out of 5 across 200 inputs, up from 3.8 last version."
+
+![Prompts index](https://raw.githubusercontent.com/homemade-software-inc/completion-kit/main/docs/screenshots/prompts.png)
+
+![Prompt detail with metrics and rubrics](https://raw.githubusercontent.com/homemade-software-inc/completion-kit/main/docs/screenshots/prompt-detail.png)
+
+![Test run with scored results](https://raw.githubusercontent.com/homemade-software-inc/completion-kit/main/docs/screenshots/test-run.png)
+
+## Setup
+
+```ruby
+gem "completion-kit"
+```
+
+```bash
+bin/rails generate completion_kit:install
+bin/rails db:migrate
+```
+
+Set your provider keys via environment variables or the generated initializer:
+
+```bash
+OPENAI_API_KEY=...
+ANTHROPIC_API_KEY=...
+LLAMA_API_KEY=...
+LLAMA_API_ENDPOINT=...
+```
+
+Available models are discovered dynamically from each provider's API.
+
+### Encryption keys
+
+Provider API keys are stored using [Rails Active Record encryption](https://guides.rubyonrails.org/active_record_encryption.html), so the host app must have encryption keys configured. If you haven't set them up already:
+
+```bash
+bin/rails db:encryption:init
+```
+
+Copy the generated keys into `config/credentials.yml.enc` under `active_record_encryption`, or set the equivalent environment variables. CompletionKit won't boot without valid keys in production.
+
+## Authentication
+
+CompletionKit requires authentication in production. In development, routes are open by default (with a log warning).
+
+### Basic Auth (recommended for simple setups)
+
+```ruby
+# config/initializers/completion_kit.rb
+CompletionKit.configure do |c|
+  c.username = "admin"
+  c.password = ENV["COMPLETION_KIT_PASSWORD"]
+end
+```
+
+### Custom Auth (Devise, etc.)
+
+```ruby
+# config/initializers/completion_kit.rb
+CompletionKit.configure do |c|
+  c.auth_strategy = ->(controller) { controller.authenticate_user! }
+end
+```
+
+Only one mode can be active — setting both raises a `ConfigurationError`.
+
+## Usage
+
+1. Create a prompt with `{{variable}}` placeholders
+2. Create a test run and paste CSV data (headers match variable names)
+3. Generate outputs, run AI review, inspect scored results
+
+## Programmatic access
+
+CompletionKit exposes every resource through both a REST JSON API and an MCP server. Both share the same bearer-token auth, so configure once and use either interface:
+
+```ruby
+# config/initializers/completion_kit.rb
+CompletionKit.configure { |c| c.api_token = ENV["COMPLETION_KIT_API_TOKEN"] }
+```
+
+### Concepts
+
+These are the objects you'll work with, whether through the UI, the REST API, or the MCP server:
+
+- **Prompt** — A named, versioned template with `{{variable}}` placeholders. Publishing a prompt freezes its template so runs always reference a known version; editing a published prompt creates a new version.
+- **Dataset** — A CSV of real inputs. Column headers match the prompt's `{{variable}}` names, and each row becomes one test case.
+- **Run** — A single execution of a prompt against a dataset. Tracks progress, stores outputs, and records which metrics were used for scoring.
+- **Response** — The model's output for one row of the dataset, with any reviews attached.
+- **Metric** — One evaluation dimension: a name, an instruction, evaluation steps, and 1–5-star rubric bands. The judge uses a metric to score a response.
+- **Criteria** — A named, reusable bundle of metrics you can apply to a run in one step.
+- **Provider Credential** — An API key for a model provider (OpenAI, Anthropic, Ollama, OpenRouter). Encrypted at rest using Rails' Active Record encryption, and never returned through the API.
+
+### REST API
+
+```bash
+curl -H "Authorization: Bearer $TOKEN" \
+  http://localhost:3000/completion_kit/api/v1/prompts
+
+curl -X POST http://localhost:3000/completion_kit/api/v1/prompts \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"name":"summarizer","template":"Summarize: {{text}}","llm_model":"gpt-4.1"}'
+```
+
+Mount the engine, then visit **`/completion_kit/api_reference`** in your running app for per-endpoint documentation with copy-to-clipboard curl examples pre-filled with your token.
+
+### MCP server
+
+CompletionKit also runs a [Model Context Protocol](https://modelcontextprotocol.io) server at the `/mcp` path within the engine mount, exposing the same resources as 36 tools (one per CRUD action plus process actions like `runs_generate` and `prompts_publish`). Point Claude Code, Cursor, or any other MCP client at it:
+
+```json
+{
+  "mcpServers": {
+    "completion-kit": {
+      "url": "https://your-app.com/completion_kit/mcp",
+      "headers": { "Authorization": "Bearer YOUR_TOKEN" }
+    }
+  }
+}
+```
+
+The in-app API reference page also ships install snippets you can copy straight into your MCP client config.
+
+## Standalone App
+
+CompletionKit ships with a standalone Rails app you can deploy as a hosted service.
+
+### Quick Start
+
+```bash
+cd standalone
+bundle install
+bin/rails completion_kit:install:migrations
+bin/rails db:migrate
+bin/rails server
+```
+
+Visit `http://localhost:3000` for the home page, or `http://localhost:3000/completion_kit` for the engine UI.
+
+### Configuration
+
+Set environment variables:
+
+| Variable | Purpose | Default |
+|----------|---------|---------|
+| `COMPLETION_KIT_API_TOKEN` | Bearer token for REST API and MCP access | (none — API disabled) |
+| `COMPLETION_KIT_USERNAME` | Web UI login username | `admin` |
+| `COMPLETION_KIT_PASSWORD` | Web UI login password | (none — open in dev) |
+| `DATABASE_URL` | PostgreSQL connection string (production) | SQLite in dev |
+
+### Deploying
+
+Any Rails-friendly host works — Fly, Heroku, Render, self-managed Docker, etc. Point your host at a Postgres instance via `DATABASE_URL`, set the environment variables above, and run `cd standalone && bin/rails db:migrate` on each deploy.
+
+When the gem ships a new engine migration, install it into your standalone app locally and commit the generated file before pushing:
+
+```bash
+cd standalone
+bin/rails completion_kit:install:migrations
+bin/rails db:migrate
+git add db/migrate/ && git commit -m "install new engine migration"
+```
+
+That way your host's `db:migrate` picks up the new file on the next deploy. Don't run `completion_kit:install:migrations` on the host itself — migration files are source artifacts, they belong in git.
+
+## Development
+
+```bash
+bundle install
+bundle exec rspec
+```
+
+## License
+
+[MIT](https://opensource.org/licenses/MIT)

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/setup"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = "--default-path spec"
+end
+
+task default: :spec

data/app/assets/config/completion_kit_manifest.js

@@ -0,0 +1 @@
+//= link_directory ../stylesheets/completion_kit .css

data/app/assets/config/manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images
+//= link_directory ../stylesheets .css
+//= link_directory ../stylesheets/completion_kit .css

data/app/assets/images/completion_kit/logo.svg

@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="2.4 2.3 88.74 89.1" width="512" height="512">
+  <path d="M57.7,56.4c1.2-1.2,1.8-3.1,1.4-5c-0.5-2-2.1-3.6-4.1-4c-3.5-0.7-6.6,1.9-6.6,5.3c0,1.5,0.6,2.8,1.5,3.8H37.4l0-12.6c-1.2,1.2-3.1,1.8-5,1.4c-2-0.5-3.6-2.1-4-4.1c-0.7-3.5,1.9-6.6,5.3-6.6c1.5,0,2.8,0.6,3.8,1.5V23.5H10.1c-3.1,0-5.7,2.5-5.7,5.7v54.5c0,3.1,2.5,5.7,5.7,5.7h27.2h27.2c3.1,0,5.7-2.5,5.7-5.7V56.4H57.7z" fill="#0891B2"/>
+  <path d="M54.7,37.5c-0.1-1.7,0.6-3.5,2.2-4.8c1.6-1.2,3.9-1.4,5.7-0.5c3.2,1.7,3.8,5.7,1.6,8.3c-0.9,1.1-2.3,1.8-3.6,1.9l9.6,8.1l17.5-20.9c2-2.4,1.7-6-0.7-8L66.2,4.3l-8.1,9.6c-0.2-1.7-1.2-3.4-3-4.3c-1.8-0.9-4.1-0.7-5.7,0.5c-2.8,2.2-2.8,6.3-0.2,8.4c1.1,0.9,2.5,1.4,3.9,1.2L45,29.5L54.7,37.5z" fill="#06B6D4"/>
+  <path d="M70.2,52.6c-0.5,0-0.9-0.2-1.3-0.5L59.3,44c-0.6-0.5-0.9-1.4-0.6-2.1c0.2-0.8,0.9-1.3,1.7-1.4c0.9-0.1,1.7-0.5,2.3-1.2c0.7-0.8,0.9-1.8,0.7-2.8c-0.2-1-0.8-1.9-1.8-2.4c-1.1-0.6-2.6-0.5-3.6,0.3c-1,0.7-1.5,1.8-1.4,3c0.1,0.8-0.4,1.6-1.1,1.9c-0.7,0.4-1.6,0.3-2.2-0.2L43.8,31c-0.4-0.3-0.7-0.8-0.7-1.4c0-0.5,0.1-1.1,0.5-1.5l5.9-7.1c-0.5-0.3-1-0.6-1.5-1c-1.7-1.4-2.7-3.6-2.6-5.8c0-2.2,1.1-4.3,2.9-5.7c2.2-1.8,5.3-2.1,7.9-0.7c1.1,0.6,2,1.3,2.6,2.3l6-7.1C65,2.6,65.5,2.3,66,2.3c0.5,0,1.1,0.1,1.5,0.5l20.9,17.5c1.6,1.3,2.5,3.2,2.7,5.2s-0.4,4-1.8,5.6L71.8,51.9c-0.3,0.4-0.8,0.7-1.4,0.7C70.4,52.6,70.3,52.6,70.2,52.6z M64.4,43.1l5.6,4.7l16.2-19.3c0.6-0.8,0.9-1.7,0.8-2.7s-0.5-1.9-1.3-2.5L66.4,7.1l-6.8,8.1c-0.5,0.6-1.4,0.9-2.1,0.6c-0.8-0.2-1.3-0.9-1.4-1.7c-0.1-1.2-0.8-2.2-1.9-2.7c-1.1-0.6-2.6-0.4-3.6,0.3c-0.9,0.7-1.3,1.6-1.4,2.7c0,1,0.4,2,1.2,2.7c0.7,0.6,1.5,0.9,2.4,0.8c0.8-0.1,1.6,0.4,1.9,1.1s0.3,1.6-0.2,2.2l-6.8,8.1l5.6,4.7c0.5-1,1.3-2,2.2-2.7c2.3-1.7,5.4-2,7.9-0.6c2,1.1,3.4,3,3.8,5.2s-0.2,4.5-1.6,6.2C65.4,42.4,64.9,42.8,64.4,43.1z" fill="#475569"/>
+  <path d="M70.3,54.4H61c0.3-1.1,0.3-2.3,0-3.5c-0.6-2.8-2.8-5-5.6-5.5c-2.2-0.5-4.5,0.1-6.3,1.5c-1.7,1.4-2.7,3.5-2.7,5.8c0,0.6,0.1,1.2,0.2,1.8h-7.3V43.8c0-0.8-0.5-1.5-1.2-1.8c-0.7-0.3-1.6-0.1-2.2,0.4c-0.8,0.8-2,1.2-3.2,0.9c-1.2-0.3-2.2-1.3-2.5-2.5c-0.2-1.1,0-2.1,0.7-2.9c0.7-0.8,1.6-1.3,2.6-1.3c0.9,0,1.7,0.3,2.4,1c0.6,0.6,1.4,0.7,2.2,0.4c0.7-0.3,1.2-1,1.2-1.8V23.5c0-1.1-0.9-2-2-2H10.1c-4.2,0-7.7,3.4-7.7,7.7v54.5c0,4.2,3.4,7.7,7.7,7.7h54.5c4.2,0,7.7-3.4,7.7-7.7V56.4C72.3,55.3,71.4,54.4,70.3,54.4z M10.1,25.5h25.2v7.3c-0.6-0.1-1.2-0.2-1.8-0.2c-2.2,0-4.3,1-5.8,2.7s-2,4-1.5,6.3c0.6,2.8,2.8,5,5.5,5.6c1.2,0.3,2.4,0.2,3.5,0v7.3H24.8c-0.8,0-1.5,0.5-1.8,1.2c-0.3,0.7-0.1,1.6,0.4,2.2c0.8,0.8,1.1,2,0.9,3.2c-0.3,1.2-1.3,2.2-2.6,2.5c-1.1,0.2-2.1,0-2.9-0.7c-0.8-0.7-1.3-1.6-1.3-2.6c0-0.9,0.3-1.7,1-2.4c0.6-0.6,0.7-1.4,0.4-2.2c-0.3-0.7-1-1.2-1.8-1.2H6.5V29.2C6.5,27.1,8.1,25.5,10.1,25.5z M36.6,74.9c-0.7,0.3-1.2,1-1.2,1.8v10.6H10.1c-2,0-3.7-1.6-3.7-3.7V58.4h7.3c-0.1,0.6-0.2,1.2-0.2,1.8c0,2.2,1,4.3,2.7,5.8c1.7,1.4,4,2,6.3,1.5c2.8-0.6,5-2.8,5.6-5.5c0.3-1.2,0.2-2.4,0-3.5h7.3V69c0,0.8,0.5,1.5,1.2,1.8c0.7,0.3,1.6,0.1,2.2-0.4c0.8-0.8,2-1.1,3.2-0.9c1.2,0.3,2.2,1.3,2.5,2.6c0.2,1.1,0,2.1-0.7,2.9c-0.7,0.8-1.6,1.3-2.6,1.3c-0.9,0-1.7-0.3-2.4-1C38.2,74.7,37.3,74.6,36.6,74.9z M68.3,83.6c0,2-1.6,3.7-3.7,3.7H39.4V80c0.6,0.1,1.2,0.2,1.8,0.2c2.2,0,4.3-1,5.8-2.7c1.4-1.7,2-4,1.5-6.3c-0.6-2.8-2.8-5-5.5-5.6c-1.2-0.3-2.4-0.2-3.5,0v-7.3H50c0.8,0,1.5-0.5,1.8-1.2c0.3-0.7,0.1-1.6-0.4-2.2c-0.6-0.6-1-1.5-1-2.4c0-1,0.5-2,1.3-2.6c0.8-0.7,1.8-0.9,2.9-0.7c1.2,0.3,2.3,1.3,2.6,2.5c0.3,1.2-0.1,2.4-0.9,3.2c-0.6,0.6-0.7,1.4-0.4,2.2c0.3,0.7,1,1.2,1.8,1.2h10.6V83.6z" fill="#475569"/>
+</svg>

data/app/assets/javascripts/completion_kit/evaluation_steps_controller.js

@@ -0,0 +1,25 @@
+document.addEventListener("DOMContentLoaded", function () {
+  document.addEventListener("click", function (event) {
+    var addBtn = event.target.closest("[data-action='evaluation-steps#add']");
+    if (addBtn) {
+      var container = addBtn.closest("[data-controller='evaluation-steps']");
+      var list = container.querySelector("[data-evaluation-steps-target='list']");
+      var row = document.createElement("div");
+      row.className = "ck-step-row";
+      row.setAttribute("data-evaluation-steps-target", "row");
+      row.innerHTML =
+        '<input type="text" name="metric[evaluation_steps][]" value="" class="ck-input" placeholder="Describe this evaluation step..." />' +
+        '<button type="button" class="ck-icon-btn" data-action="evaluation-steps#remove" aria-label="Remove step">' +
+        '<svg viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="1.75"><path d="M3 6h18"/><path d="M19 6v14c0 1-1 2-2 2H7c-1 0-2-1-2-2V6"/><path d="M8 6V4c0-1 1-2 2-2h4c1 0 2 1 2 2v2"/><line x1="10" y1="11" x2="10" y2="17"/><line x1="14" y1="11" x2="14" y2="17"/></svg>' +
+        "</button>";
+      list.appendChild(row);
+      row.querySelector("input").focus();
+    }
+
+    var removeBtn = event.target.closest("[data-action='evaluation-steps#remove']");
+    if (removeBtn) {
+      var stepRow = removeBtn.closest("[data-evaluation-steps-target='row']");
+      if (stepRow) stepRow.remove();
+    }
+  });
+});