completion-kit 0.29.2 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +4 -1
  3. data/README.md +94 -127
  4. data/app/assets/stylesheets/completion_kit/application.css +29 -0
  5. data/app/controllers/completion_kit/api/v1/base_controller.rb +1 -0
  6. data/app/controllers/completion_kit/api/v1/prompts_controller.rb +14 -2
  7. data/app/controllers/completion_kit/prompts_controller.rb +1 -6
  8. data/app/controllers/completion_kit/runs_controller.rb +8 -1
  9. data/app/models/completion_kit/prompt.rb +7 -0
  10. data/app/services/completion_kit/mcp_tools/agreements.rb +4 -2
  11. data/app/services/completion_kit/mcp_tools/base.rb +7 -1
  12. data/app/services/completion_kit/mcp_tools/datasets.rb +12 -6
  13. data/app/services/completion_kit/mcp_tools/imports.rb +2 -1
  14. data/app/services/completion_kit/mcp_tools/judges.rb +4 -2
  15. data/app/services/completion_kit/mcp_tools/metric_groups.rb +10 -5
  16. data/app/services/completion_kit/mcp_tools/metric_versions.rb +6 -3
  17. data/app/services/completion_kit/mcp_tools/metrics.rb +12 -6
  18. data/app/services/completion_kit/mcp_tools/prompts.rb +18 -9
  19. data/app/services/completion_kit/mcp_tools/provider_credentials.rb +10 -5
  20. data/app/services/completion_kit/mcp_tools/responses.rb +4 -6
  21. data/app/services/completion_kit/mcp_tools/runs.rb +18 -11
  22. data/app/services/completion_kit/mcp_tools/tags.rb +10 -5
  23. data/app/views/completion_kit/api_reference/_body.html.erb +4 -6
  24. data/app/views/completion_kit/provider_credentials/_form.html.erb +4 -2
  25. data/app/views/completion_kit/runs/compare_picker.html.erb +5 -3
  26. data/lib/completion_kit/version.rb +1 -1
  27. data/lib/completion_kit.rb +2 -1
  28. metadata +1 -1
@@ -4,6 +4,12 @@ module CompletionKit
4
4
  DEFAULT_PAGE_LIMIT = 50
5
5
  MAX_PAGE_LIMIT = 500
6
6
 
7
+ READ_ONLY = {readOnlyHint: true, destructiveHint: false, openWorldHint: false}.freeze
8
+ ADDITIVE = {readOnlyHint: false, destructiveHint: false, openWorldHint: false}.freeze
9
+ DESTRUCTIVE = {readOnlyHint: false, destructiveHint: true, openWorldHint: false}.freeze
10
+ OPEN_WORLD_ADDITIVE = {readOnlyHint: false, destructiveHint: false, openWorldHint: true}.freeze
11
+ OPEN_WORLD_DESTRUCTIVE = {readOnlyHint: false, destructiveHint: true, openWorldHint: true}.freeze
12
+
7
13
  def page_bounds(args)
8
14
  limit = args["limit"].to_i
9
15
  limit = DEFAULT_PAGE_LIMIT if limit <= 0
@@ -14,7 +20,7 @@ module CompletionKit
14
20
  end
15
21
 
16
22
  def definitions
17
- self::TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
23
+ self::TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema], annotations: config[:annotations]} }
18
24
  end
19
25
 
20
26
  def call(name, arguments)
@@ -9,17 +9,20 @@ module CompletionKit
9
9
 
10
10
  TOOLS = {
11
11
  "datasets_list" => {
12
- description: "List all datasets",
12
+ description: "List every test dataset in your CompletionKit account, newest first. Returns each dataset's id, name, full CSV data, tags and timestamps, so the result can be large when your datasets are big. Use it to find a dataset to test a prompt against or to score AI responses on. To read one dataset, use datasets_get.",
13
+ annotations: Base::READ_ONLY,
13
14
  inputSchema: {type: "object", properties: {}, required: []},
14
15
  handler: :list
15
16
  },
16
17
  "datasets_get" => {
17
- description: "Get a dataset by ID",
18
+ description: "Get one test dataset by its ID. Returns the dataset's name, its full CSV data (header row first, then one test case per row), its tags and its timestamps. Use it to check a dataset's columns and rows before you test a prompt or start a run against it.",
19
+ annotations: Base::READ_ONLY,
18
20
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
19
21
  handler: :get
20
22
  },
21
23
  "datasets_create" => {
22
- description: "Create a dataset with CSV data. First row is the header. Two column names are recognized specially: \"expected_output\" is each row's answer key (ground truth) given to the judge and to checks that compare against the row's expected value, and \"actual_output\" is a pre-made output to score in a prompt-less run. Both are overridable per run (expected_column / output_column). Every column is also available to the prompt as a variable.",
24
+ description: "Create a test dataset from CSV text included in the call. The first row is the header, and each later row is one test case. Every column is available to the prompt as a variable. Two column names are recognized specially: \"expected_output\" is each row's answer key (ground truth) given to the judge and to checks that compare against the row's expected value, and \"actual_output\" is a pre-made output to score in a prompt-less run. Both are overridable per run (expected_column / output_column). Optional tag_names labels the dataset. Tag names are lowercased and may contain only unaccented letters, numbers, spaces, hyphens and underscores, up to 64 characters, and tags that don't exist yet are created. Returns the new dataset. The CSV is rejected if it is empty, is not valid UTF-8, has a blank header row or a header row over 64 KB, or is larger than the upload limit (25 MB by default). If the CSV is too big to put in the call, use datasets_create_from_url, which downloads files up to 10MB.",
25
+ annotations: Base::ADDITIVE,
23
26
  inputSchema: {
24
27
  type: "object",
25
28
  properties: {name: {type: "string"}, csv_data: {type: "string"}, tag_names: {type: "array", items: {type: "string"}}},
@@ -28,7 +31,8 @@ module CompletionKit
28
31
  handler: :create
29
32
  },
30
33
  "datasets_update" => {
31
- description: "Update a dataset",
34
+ description: "Change a test dataset's name, CSV data or tags. Passing csv_data replaces the whole CSV, and the dataset keeps no copy of the old version. Passing tag_names replaces all of the dataset's tags. Responses from runs that already ran keep the row values they used, but any run started after the change reads the new rows. If you remove or rename a column that a run needs (a variable in its prompt, its expected_column or its output column), that run cannot be started, finished, regraded, retried or rerun until the column is back. To add rows, read the dataset with datasets_get first and send back the full CSV. Returns the updated dataset. If tag_names is rejected, the name and CSV changes in the same call are still saved.",
35
+ annotations: Base::DESTRUCTIVE,
32
36
  inputSchema: {
33
37
  type: "object",
34
38
  properties: {id: {type: "integer"}, name: {type: "string"}, csv_data: {type: "string"}, tag_names: {type: "array", items: {type: "string"}}},
@@ -37,12 +41,14 @@ module CompletionKit
37
41
  handler: :update
38
42
  },
39
43
  "datasets_delete" => {
40
- description: "Delete a dataset",
44
+ description: "Permanently delete a test dataset. This also deletes every run that used it, with those runs' responses, scores, suggestions and human agreement labels. Prompts and metrics are kept. This cannot be undone, so confirm with the user before calling it. Returns a short confirmation.",
45
+ annotations: Base::DESTRUCTIVE,
41
46
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
42
47
  handler: :delete
43
48
  },
44
49
  "datasets_create_from_url" => {
45
- description: "Create a dataset by downloading CSV from a URL instead of inlining it. Use this for large datasets: pass a public http(s) URL and the server fetches the CSV directly, so the data never has to pass through the tool-call arguments. The URL is SSRF-checked and the download is capped at 10MB. First row is the header; the \"expected_output\" (answer key) and \"actual_output\" (pre-made output) columns are recognized specially, overridable per run.",
50
+ description: "Create a test dataset by downloading a CSV file from a public http(s) URL instead of pasting it into the call. Use this for large datasets: the server fetches the CSV directly, so the data never has to pass through the tool-call arguments. The URL is SSRF-checked, so it cannot point at private network addresses. Redirects are not followed and return an error, and files over 10MB are rejected. The first row is the header. The \"expected_output\" (answer key) and \"actual_output\" (pre-made output) columns are recognized specially, and both are overridable per run. Optional tag_names labels the dataset. Returns the new dataset, or an error that says why the URL, the download or the CSV was rejected.",
51
+ annotations: Base::OPEN_WORLD_ADDITIVE,
46
52
  inputSchema: {
47
53
  type: "object",
48
54
  properties: {
@@ -5,7 +5,8 @@ module CompletionKit
5
5
 
6
6
  TOOLS = {
7
7
  "promptfoo_import" => {
8
- description: "Import a promptfooconfig.yaml. Creates a prompt, a dataset from the test vars, and metrics from the assert blocks (llm-rubric/g-eval become judge metrics; contains/equals/regex/is-json become deterministic check metrics). Returns a summary of what mapped and what was skipped and why; nothing is dropped silently.",
8
+ description: "Import a promptfoo config (the full promptfooconfig.yaml text) so you can test its prompts and score their responses in CompletionKit. It creates a new prompt for each inline prompt template, one dataset from the tests' vars, and metrics from the assert blocks in tests and defaultTest, with identical asserts imported once. llm-rubric, g-eval, model-graded-closedqa and factuality become LLM judge metrics. contains, icontains, not-contains, not-icontains, equals, regex and is-json become deterministic check metrics. Prompts use the model named by the first provider, or gpt-4o if none is given, and are created even if no saved provider key serves that model. Prompts given as file:// references are skipped, so paste the file's contents instead. It only adds new records, never changes existing ones and does not start a run. Providers are only matched by name against your saved provider keys and reported as matched or unmatched. Any apiKey in the config is ignored, so remove keys before pasting. Returns a summary of what was created and of the prompts and assert types it skipped, with the reason for each. Assert options such as threshold, weight and provider, defaultTest vars, and asserts with no type are dropped without being listed in the summary. If a prompt or the dataset cannot be saved, the whole import is rolled back and nothing is created.",
9
+ annotations: Base::ADDITIVE,
9
10
  inputSchema: {
10
11
  type: "object",
11
12
  properties: {
@@ -5,7 +5,8 @@ module CompletionKit
5
5
 
6
6
  TOOLS = {
7
7
  "judges_replay" => {
8
- description: "Create a scoring run for the current judge over a dataset's existing outputs (wraps runs_create with prompt_id omitted and output_column supplied). This only sets up the run; call runs_generate to actually re-judge the outputs so you can compare against human verdicts.",
8
+ description: "Check an LLM judge against outputs you already have: set up a scoring run that grades a dataset column of existing outputs with one metric's current judge and the judge_model you name (wraps runs_create with prompt_id omitted and output_column supplied, defaulting to actual_output). This only sets up the run; call runs_generate to actually re-judge the outputs so you can compare against human verdicts. It returns the new run.",
9
+ annotations: Base::ADDITIVE,
9
10
  inputSchema: {
10
11
  type: "object",
11
12
  properties: {
@@ -20,7 +21,8 @@ module CompletionKit
20
21
  handler: :replay
21
22
  },
22
23
  "judges_compare" => {
23
- description: "Compare two versions of one metric's agreement stats side by side. Requires metric_id, metric_version_a_id, and metric_version_b_id (both versions must belong to that metric). Unavailable for check metrics.",
24
+ description: "Compare two versions of one metric's LLM judge to see which agrees more with human reviewers. It computes stats from the agreement verdicts already recorded under each version and saves nothing; verdicts on runs older than the account's history window are not counted. It requires metric_id, metric_version_a_id, and metric_version_b_id (both versions must belong to that metric). It returns, for each version, the number of verdicts, the agreement rate with its confidence range, borderline rate, mean absolute error and kappa, plus the change from A to B and a recommendation. The recommendation is need_more_data under 30 verdicts combined, recommend when B's agreement rate beats A's by more than 3 points, hold when it trails A's by more than 3 points, and no_change otherwise or when one version has no verdicts. It is unavailable for check metrics.",
25
+ annotations: Base::READ_ONLY,
24
26
  inputSchema: {
25
27
  type: "object",
26
28
  properties: {
@@ -5,17 +5,20 @@ module CompletionKit
5
5
 
6
6
  TOOLS = {
7
7
  "metric_groups_list" => {
8
- description: "List all metric groups",
8
+ description: "List every metric group in your workspace, newest first. A metric group is a saved, ordered set of metrics that you can attach to a run in one step, so runs are scored the same way each time. Returns each group's id, name, description, metric_ids, tags and timestamps. The full list comes back with no paging. Use it to find a metric_group_id for runs_create or runs_update.",
9
+ annotations: Base::READ_ONLY,
9
10
  inputSchema: {type: "object", properties: {}, required: []},
10
11
  handler: :list
11
12
  },
12
13
  "metric_groups_get" => {
13
- description: "Get a metric group by ID",
14
+ description: "Get one metric group by id. Returns its id, name, description, the ids of its metrics in order, tags and timestamps. Use metrics_get on each id to see how each metric scores AI responses.",
15
+ annotations: Base::READ_ONLY,
14
16
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
15
17
  handler: :get
16
18
  },
17
19
  "metric_groups_create" => {
18
- description: "Create a metric group",
20
+ description: "Create a metric group: a named, ordered set of metrics that you can attach to a run in one step with metric_group_id, so every run of a prompt is scored the same way. Send name, and optionally description, metric_ids in the order you want them and tag_names (any new tags are created). Each id must be a metric in this workspace, or the whole call fails. A run copies the group's metrics when you attach the group, so later changes to the group don't reach that run. Returns the saved group.",
21
+ annotations: Base::ADDITIVE,
19
22
  inputSchema: {
20
23
  type: "object",
21
24
  properties: {
@@ -28,7 +31,8 @@ module CompletionKit
28
31
  handler: :create
29
32
  },
30
33
  "metric_groups_update" => {
31
- description: "Update a metric group",
34
+ description: "Change a metric group's name, description, metrics or tags. This overwrites the group in place. metric_ids replaces the group's whole metric list in the order given, and an empty list removes every metric. Each id must be a metric in this workspace. Leave metric_ids out to keep the current list. tag_names replaces the whole tag set. Runs already created keep the metrics they had. Returns the updated group.",
35
+ annotations: Base::DESTRUCTIVE,
32
36
  inputSchema: {
33
37
  type: "object",
34
38
  properties: {
@@ -41,7 +45,8 @@ module CompletionKit
41
45
  handler: :update
42
46
  },
43
47
  "metric_groups_delete" => {
44
- description: "Delete a metric group",
48
+ description: "Permanently delete a metric group. Its tags are unlinked, though the tags themselves stay. The metrics in it, and any runs scored with it, stay as they are. This cannot be undone. Returns a confirmation message.",
49
+ annotations: Base::DESTRUCTIVE,
45
50
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
46
51
  handler: :delete
47
52
  }
@@ -5,7 +5,8 @@ module CompletionKit
5
5
 
6
6
  TOOLS = {
7
7
  "metric_versions_list" => {
8
- description: "List every MetricVersion (drafts + published) for a metric, newest first. Each row carries version_number, state, source, current flag, and timestamps.",
8
+ description: "List every version of one metric's judge or check, drafts and published, newest first. Use it to see how a judge changed over time, find drafts suggested by metrics_suggest_variants or saved from edits in the web app, or pick an older version to restore with metric_versions_publish. Each row carries id, version_number, state (draft or published), source, the current flag marking the live version, and timestamps, plus that version's instruction and rubric bands (judges) or check_config (checks). source is edit or suggestion for versions made those ways, and null for a version saved automatically the first time the metric scored a response or got a verdict. The current version can differ from the metric's live definition if metrics_update changed the metric after that version was saved.",
9
+ annotations: Base::READ_ONLY,
9
10
  inputSchema: {
10
11
  type: "object",
11
12
  properties: {
@@ -16,7 +17,8 @@ module CompletionKit
16
17
  handler: :list
17
18
  },
18
19
  "metric_versions_publish" => {
19
- description: "Publish a MetricVersion as the live version of its metric. Works for both 'draft → published' and 'revert to an older published version → current'. Transactionally flips current, demotes peers, and writes the version's instruction + rubric_bands back onto the metric so the judge grades against it.",
20
+ description: "Make a metric version the live judge or check that scores new AI responses. It works both to publish a draft and to switch back to an older published version. In one step it marks this version current, marks every other version of the metric not current, and writes this version's instruction and rubric bands (or check_config, for a check) onto the metric, replacing what the metric had. Responses already scored keep their scores. Rescore a run with runs_regrade if you want new ones. Human verdicts stay with the version they were given on. Other published versions stay saved, so you can switch back by publishing one, but a live definition that was never saved as a version is lost. A check version with an invalid check_config is refused. Returns the published version.",
21
+ annotations: Base::DESTRUCTIVE,
20
22
  inputSchema: {
21
23
  type: "object",
22
24
  properties: {
@@ -27,7 +29,8 @@ module CompletionKit
27
29
  handler: :publish
28
30
  },
29
31
  "metric_versions_dismiss" => {
30
- description: "Destroy a draft MetricVersion (use for either source: 'edit' or source: 'suggestion'). Published versions are refused. To demote a published version, publish a different one as current instead.",
32
+ description: "Permanently delete a draft version of a metric, whether it came from an edit in the web app or from metrics_suggest_variants. Use it to throw away a suggested judge rewrite you don't want. Published versions are refused. To stop using a published version, publish a different one as current instead. This cannot be undone. Returns the deleted version's id with destroyed: true.",
33
+ annotations: Base::DESTRUCTIVE,
31
34
  inputSchema: {
32
35
  type: "object",
33
36
  properties: {
@@ -44,17 +44,20 @@ module CompletionKit
44
44
 
45
45
  TOOLS = {
46
46
  "metrics_list" => {
47
- description: "List all metrics",
47
+ description: "List every metric in your workspace, newest first. A metric is how CompletionKit scores AI responses. It is either an LLM judge, which grades each response from 1 to 5 (5 is best) against an instruction and rubric, or a check, which scores a response by an exact rule such as contains, equals, regex, valid JSON, a JSON path value, length or number bounds, or list overlap. Returns each metric's id, name, key, type, tags and timestamps, plus its instruction and rubric bands (judges) or its check_config (checks). The full list comes back with no paging. Use it to find a metric id before you score a run, edit a judge or build a metric group.",
48
+ annotations: Base::READ_ONLY,
48
49
  inputSchema: {type: "object", properties: {}, required: []},
49
50
  handler: :list
50
51
  },
51
52
  "metrics_get" => {
52
- description: "Get a metric by ID",
53
+ description: "Get one metric by id. Returns its id, name, key, type, tags and timestamps, plus its live judge instruction and 1 to 5 rubric bands (for an LLM judge) or its check_config (for a check). Use it to read exactly how a metric scores AI responses before you change it or explain a score. To see its saved versions and drafts, use metric_versions_list.",
54
+ annotations: Base::READ_ONLY,
53
55
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
54
56
  handler: :get
55
57
  },
56
58
  "metrics_create" => {
57
- description: "Create a metric with evaluation criteria. #{CHECK_CONFIG_HINT}",
59
+ description: "Create a metric that scores AI responses. By default it is an LLM judge. Send name and instruction (what a good response looks like), and optionally rubric_bands describing 1 to 5 stars, where 5 is best. Any band you leave out gets a default description. For an exact rule instead of a judge, see the check details at the end. The metric's name must be unique in the workspace, ignoring case, spaces and most punctuation, and a renamed metric keeps its original name reserved. A clash fails with the error \"Key has already been taken\". tag_names adds tags and creates any that don't exist yet. Saving a metric scores nothing by itself. Attach it to a run with runs_create or runs_update, then score responses with runs_generate or runs_regrade. Returns the saved metric. #{CHECK_CONFIG_HINT}",
60
+ annotations: Base::ADDITIVE,
58
61
  inputSchema: {
59
62
  type: "object",
60
63
  properties: {
@@ -69,7 +72,8 @@ module CompletionKit
69
72
  handler: :create
70
73
  },
71
74
  "metrics_update" => {
72
- description: "Update a metric. #{CHECK_CONFIG_HINT}",
75
+ description: "Change a metric's name, judge instruction, rubric bands, check_config, type or tags. This overwrites the metric in place, and the new definition scores every response graded after it. It saves no draft or new version. If the metric already has a live version, scores and verdicts given after the change are still filed under that version, so judges_compare can't tell the old and new definitions apart. Responses already scored keep their old scores until you rescore the run with runs_regrade. Nothing tests the change before it goes live. Fields you leave out stay as they are, but a field you send replaces that whole field: rubric_bands resets any band you leave out to its default description, not its old one, and check_config replaces the whole config. tag_names replaces the whole tag set. A rename keeps the metric's original key, so its old name stays reserved. metric_type cannot change once the metric is in a run, has scores or has versions. Returns the updated metric. #{CHECK_CONFIG_HINT}",
76
+ annotations: Base::DESTRUCTIVE,
73
77
  inputSchema: {
74
78
  type: "object",
75
79
  properties: {
@@ -84,12 +88,14 @@ module CompletionKit
84
88
  handler: :update
85
89
  },
86
90
  "metrics_delete" => {
87
- description: "Delete a metric",
91
+ description: "Permanently delete a metric. This also deletes all of its versions and drafts and every human agree, disagree or borderline verdict recorded against them. It removes the metric from every run and metric group and unlinks its tags, though the tags themselves stay. Scores it already gave stay on their responses but are no longer linked to the metric. This cannot be undone. Returns a confirmation message.",
92
+ annotations: Base::DESTRUCTIVE,
88
93
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
89
94
  handler: :delete
90
95
  },
91
96
  "metrics_suggest_variants" => {
92
- description: "Ask the model to rewrite the metric's judge instruction in N variants targeted at the recent disagreements. Each variant is saved as a draft MetricVersion with source=\"suggestion\". Returns the persisted drafts. Stripe-metering hooks fire via ActiveSupport::Notifications under completion_kit.judge_suggestion.generated.",
97
+ description: "Get AI suggestions to improve an LLM judge that disagrees with people. It sends the judge's current instruction and rubric, plus up to 8 recent responses where a person disagreed with the judge's score and up to 6 that a person marked borderline (each input and output cut to 200 characters), to your own model provider. The model is asked for rewrites of the instruction, the rubric or both that close the gap. Each rewrite is saved as a draft metric version with source \"suggestion\", and earlier suggestion drafts are kept. The live judge does not change until you publish a draft with metric_versions_publish. Nothing tests a draft before you publish it. Once it is live and people have recorded verdicts on it with agreements_create, judges_compare shows whether it agrees with people more often than the version it replaced. Returns the saved drafts. It works best after you have recorded disagreements with agreements_create. Checks are refused, because they are exact and have nothing to suggest. count defaults to 1 and is capped at 3. model picks the model that writes the rewrites and defaults to your judge model. The call uses your provider key and runs while you wait. It fails with an error if no judging model is set up or the model you name is unknown. If the model's reply can't be read, nothing is saved and you get a \"no parseable variants\" error. A call your provider refuses, such as one with a bad key, usually ends in that same error.",
98
+ annotations: Base::ADDITIVE,
93
99
  inputSchema: {
94
100
  type: "object",
95
101
  properties: {
@@ -5,17 +5,20 @@ module CompletionKit
5
5
 
6
6
  TOOLS = {
7
7
  "prompts_list" => {
8
- description: "List all prompts",
8
+ description: "List every prompt in CompletionKit, newest first. Each version of a prompt is its own entry, so drafts and older versions appear next to the published one. Versions of the same prompt share a family_key, and current=true marks the published version that apps get when they fetch the prompt by name or family_key. Returns each version's id, name, description, template, llm_model, family_key, version_number, current, timestamps and tags. Use it to find a prompt's ID before you test it, edit it, compare its versions or ask for a suggested improvement.",
9
+ annotations: Base::READ_ONLY,
9
10
  inputSchema: {type: "object", properties: {}, required: []},
10
11
  handler: :list
11
12
  },
12
13
  "prompts_get" => {
13
- description: "Get a prompt by ID",
14
+ description: "Get one prompt version by its numeric ID. Returns its name, description, template text with {{variable}} placeholders, the model that answers it (llm_model), family_key, version_number, whether it is the published version (current), timestamps and tags. Use it to read a prompt before you edit it, test it in a run or compare it with another version.",
15
+ annotations: Base::READ_ONLY,
14
16
  inputSchema: {type: "object", properties: {id: {type: "integer", description: "Prompt ID"}}, required: ["id"]},
15
17
  handler: :get
16
18
  },
17
19
  "prompts_create" => {
18
- description: "Create a prompt",
20
+ description: "Create a new prompt to test. Give it a name, a template with {{variable}} placeholders that match your dataset's column names, and the model that should answer it (llm_model). The model name is checked against the models your saved provider keys serve. If CompletionKit has not found any models from a saved key yet, any model name is accepted, and runs fail until a key for that model's provider is added. The new prompt is version 1 of a new prompt family and is published right away. If another published prompt already has the same name, apps that fetch prompts by name get this new one instead. To change an existing prompt, use prompts_update so its versions stay together. tag_names creates any tags that don't exist yet. Returns the saved prompt with its ID. To test it, create a run with runs_create and start it with runs_generate.",
21
+ annotations: Base::DESTRUCTIVE,
19
22
  inputSchema: {
20
23
  type: "object",
21
24
  properties: {
@@ -28,7 +31,8 @@ module CompletionKit
28
31
  handler: :create
29
32
  },
30
33
  "prompts_update" => {
31
- description: "Update a prompt. If the prompt already has runs, this creates a new DRAFT version (current=false) rather than editing in place or publishing. Promote it with prompts_publish, so an agent's edits don't go live without a gate. If it has no runs, it is updated in place.",
34
+ description: "Edit a prompt's name, description, template, model or tags. Changing the template or model of the published version, or editing any version that already has runs, creates a new DRAFT version (current=false) instead of changing it, so an agent's edits don't go live without a gate. The draft copies the old version's name, description, template and model, with your changes applied. Run and compare the draft, then promote it with prompts_publish. An unpublished draft with no runs, or a name, description or tag change on a published version with no runs, is updated in place and the old values are not kept. Renaming a published version in place takes effect at once, so apps that fetch it by its old name stop finding it, while fetches by family_key or ID still work. tag_names replaces the whole tag list. A new draft does not copy the old version's tags, so pass tag_names to keep them, or the draft has no tags, even after it is published. Returns the saved version, which has a new ID when a draft was created.",
35
+ annotations: Base::DESTRUCTIVE,
32
36
  inputSchema: {
33
37
  type: "object",
34
38
  properties: {
@@ -41,17 +45,20 @@ module CompletionKit
41
45
  handler: :update
42
46
  },
43
47
  "prompts_delete" => {
44
- description: "Delete a prompt",
48
+ description: "Permanently delete one prompt version by ID, along with every run of that version and all of those runs' responses, scores, judge reviews, agreements and suggestions. Other versions of the prompt, and your datasets, metrics and tags, are kept. Apps that fetch this version by its ID get a not-found error. If you delete the published version, no other version is published in its place, so apps that fetch the prompt by family_key get a not-found error, and apps that fetch it by name get a not-found error or another published prompt with the same name, until you publish another version with prompts_publish. This cannot be undone. Returns a confirmation message.",
49
+ annotations: Base::DESTRUCTIVE,
45
50
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
46
51
  handler: :delete
47
52
  },
48
53
  "prompts_publish" => {
49
- description: "Publish a prompt version, making it the current version",
54
+ description: "Publish a prompt version, making it the current version. Apps that fetch the prompt by its name or family_key get this version from their next request, and the version that was published before stops being current. Use it to ship a draft from prompts_update once a run shows it scores at least as well, or to roll back by publishing an older version. Returns the published version.",
55
+ annotations: Base::DESTRUCTIVE,
50
56
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
51
57
  handler: :publish
52
58
  },
53
59
  "prompts_suggest_improvement" => {
54
- description: "Suggest an improved version of a prompt, grounded in a run's test results and judge feedback. Analyzes the run's responses, scores, and reviews, then returns reasoning plus a rewritten template (preserving {{variables}}) and persists it as a Suggestion. Requires a run that has a prompt (not a scoring-only run).",
60
+ description: "Suggest an improved version of a prompt, grounded in a run's test results and judge feedback. It reads up to 20 of the run's responses with their scores and reviews, then asks the prompt's own model, on your saved provider key, for reasoning plus a rewritten template that keeps the {{variables}}. It then tests the rewrite on up to 30 of the run's judge-scored rows and scores the new answers with the run's LLM-judge metrics and judge model. Returns suggestion_id, prompt_id, reasoning, suggested_template, original_template, validation (before and after averages and how many rows improved, regressed or stayed the same) and net_negative, which is true when the rewrite's average score fell or more rows got worse than better. If validation.tested is 0, nothing was checked and net_negative is false. That happens when the run has no LLM-judge metrics, has no judge-scored rows with both input and output, or every test call failed. It saves the result as a Suggestion. The prompt itself is not changed, so pass suggested_template to prompts_update to save it as a draft version. Requires a run that has a prompt (not a scoring-only run). Each call can make dozens of model calls on your saved provider keys and can take a while.",
61
+ annotations: Base::ADDITIVE,
55
62
  inputSchema: {
56
63
  type: "object",
57
64
  properties: {run_id: {type: "integer", description: "The run whose results ground the improvement."}},
@@ -82,8 +89,10 @@ module CompletionKit
82
89
  def self.update(args)
83
90
  prompt = Prompt.find(args["id"])
84
91
  attrs = args.except("id").slice("name", "description", "template", "llm_model")
85
- if prompt.runs.exists?
86
- new_prompt = prompt.clone_as_new_version(attrs)
92
+ if prompt.runs.exists? || (prompt.current? && prompt.behavioral_change?(attrs))
93
+ new_prompt = prompt.build_next_version(attrs)
94
+ return error_result(new_prompt.errors.full_messages.join(", ")) unless new_prompt.save
95
+
87
96
  new_prompt.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
88
97
  text_result(new_prompt.reload.as_json)
89
98
  elsif prompt.update(attrs)
@@ -5,17 +5,20 @@ module CompletionKit
5
5
 
6
6
  TOOLS = {
7
7
  "provider_credentials_list" => {
8
- description: "List all provider credentials (API keys are not exposed)",
8
+ description: "List the model provider connections saved in your CompletionKit account (OpenAI, Anthropic, Ollama or another OpenAI-compatible endpoint, OpenRouter, Azure AI Foundry). Returns each one's id, provider, custom endpoint, API version and timestamps. API keys are not exposed. Use it to see which providers are connected before you test a prompt, start a run or use an LLM judge. It does not show whether a key is still stored or whether the provider's models loaded.",
9
+ annotations: Base::READ_ONLY,
9
10
  inputSchema: {type: "object", properties: {}, required: []},
10
11
  handler: :list
11
12
  },
12
13
  "provider_credentials_get" => {
13
- description: "Get a provider credential by ID (API key is not exposed)",
14
+ description: "Get one saved model provider connection by its ID. Returns its provider, custom endpoint, API version and timestamps. The API key is not exposed. Use it to check how a provider is set up before you change it.",
15
+ annotations: Base::READ_ONLY,
14
16
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
15
17
  handler: :get
16
18
  },
17
19
  "provider_credentials_create" => {
18
- description: "Create a provider credential",
20
+ description: "Save an API key for a model provider (openai, anthropic, ollama for Ollama or any other OpenAI-compatible endpoint, openrouter or azure_foundry) so CompletionKit can run your prompts and LLM judges with it. You can save one credential per provider. azure_foundry requires api_endpoint, ollama needs api_endpoint for its models to load, and any api_endpoint must be an http(s) URL that resolves to a public address. The key is stored encrypted and is never returned. Saving starts a background job that lists the provider's models. For every provider except openrouter, the job then sends each model that has not been tested yet a short test prompt to check it can generate text and, if it can, a second one to check it can judge, and these requests use your key. Returns the saved credential without the key.",
21
+ annotations: Base::OPEN_WORLD_ADDITIVE,
19
22
  inputSchema: {
20
23
  type: "object",
21
24
  properties: {
@@ -29,7 +32,8 @@ module CompletionKit
29
32
  handler: :create
30
33
  },
31
34
  "provider_credentials_update" => {
32
- description: "Update a provider credential. Changing provider or api_endpoint on a credential with a stored key requires api_key in the same call, so a stored key never moves to a new provider or endpoint.",
35
+ description: "Change a saved model provider connection: its API key, provider, custom endpoint or API version. A new api_key replaces the stored key, and the old key cannot be recovered. An empty api_key clears the stored key. Changing provider or api_endpoint on a credential with a stored key requires api_key in the same call, so a stored key never moves to a new provider or endpoint. Each provider can have only one credential. Any api_endpoint must be an http(s) URL that resolves to a public address. Each successful save runs model discovery again in the background. It lists the provider's models and, for every provider except openrouter, sends each model that has not been tested yet a short test prompt to check it can generate text and, if it can, a second one to check it can judge, using the key. Models the provider no longer lists are marked retired, and prompts or judges set to a retired model can fail to run. Returns the credential without the key.",
36
+ annotations: Base::OPEN_WORLD_DESTRUCTIVE,
33
37
  inputSchema: {
34
38
  type: "object",
35
39
  properties: {
@@ -42,7 +46,8 @@ module CompletionKit
42
46
  handler: :update
43
47
  },
44
48
  "provider_credentials_delete" => {
45
- description: "Delete a provider credential",
49
+ description: "Permanently delete a saved model provider connection and its stored API key, and remove that provider's models from your model list. It is refused with an error while any current prompt version uses one of that provider's models, or while any run, including past runs, used one of them as its judge. This cannot be undone, so confirm with the user before calling it. Returns a short confirmation.",
50
+ annotations: Base::DESTRUCTIVE,
46
51
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
47
52
  handler: :delete
48
53
  }
@@ -10,11 +10,8 @@ module CompletionKit
10
10
 
11
11
  TOOLS = {
12
12
  "responses_list" => {
13
- description: "List responses for a run, in row order. Returns " \
14
- "{total, limit, offset, returned, responses}. Defaults to #{Base::DEFAULT_PAGE_LIMIT} rows " \
15
- "because full payloads are large: use \"fields\" to drop the bodies, \"min_score\"/\"max_score\" " \
16
- "to isolate low scorers, and sort \"score_asc\" to read the worst rows first. For per-metric " \
17
- "averages of the whole run use runs_get instead of aggregating here.",
13
+ description: "List the AI responses in a run with their scores, in row order, to see which test cases scored well or badly and why. Each row has the input, the response_text, the expected_output, an average judge score from 1 to 5 (5 is best), and each metric's review with judge feedback or a check's pass/fail. Returns {total, limit, offset, returned, responses}. Defaults to #{Base::DEFAULT_PAGE_LIMIT} rows because full payloads are large: use \"fields\" to drop the bodies, \"min_score\"/\"max_score\" to isolate low scorers, and sort \"score_asc\" to read the worst rows first. Rows with no judge score, such as rows graded only by checks, are left out by a score filter and listed last by a score sort. For per-metric averages of the whole run use runs_get instead of aggregating here.",
14
+ annotations: Base::READ_ONLY,
18
15
  inputSchema: {
19
16
  type: "object",
20
17
  properties: {
@@ -32,7 +29,8 @@ module CompletionKit
32
29
  handler: :list
33
30
  },
34
31
  "responses_get" => {
35
- description: "Get a specific response",
32
+ description: "Get one response from a run in full, by run_id and response id. The response must belong to that run. Returns the input, the complete response_text, the expected_output, the average judge score from 1 to 5 (5 is best), each metric's review with the judge's score and feedback or the check's pass/fail, the row's status and attempts, and any provider error. Use it to read a single test case closely after responses_list points you to it.",
33
+ annotations: Base::READ_ONLY,
36
34
  inputSchema: {
37
35
  type: "object",
38
36
  properties: {run_id: {type: "integer"}, id: {type: "integer"}},
@@ -22,19 +22,20 @@ module CompletionKit
22
22
 
23
23
  TOOLS = {
24
24
  "runs_list" => {
25
- description: "List all runs",
25
+ description: "List your test runs, newest first. A run tests one prompt version against a dataset, or grades existing outputs, and scores each response with your metrics. It returns each run's id, name, status (pending, running, completed or failed), prompt and dataset ids, judge model, average score (1 to 5, 5 is best), check pass rate, per-metric averages and progress. Runs older than the account's history window may be left out. Use it to find a run's id or see recent results, then call runs_get for one run.",
26
+ annotations: Base::READ_ONLY,
26
27
  inputSchema: {type: "object", properties: {}, required: []},
27
28
  handler: :list
28
29
  },
29
30
  "runs_get" => {
30
- description: "Get a run by ID, including \"metric_averages\": a per-metric breakdown with each metric's " \
31
- "average score (or pass rate for checks), how many rows it graded, and how many scored low. " \
32
- "Use this to find the metric dragging a prompt down without listing responses.",
31
+ description: "Get one test run's results by id: status, progress, average score (1 to 5, 5 is best), check pass rate, the ids of failed responses, any failure reason, and \"metric_averages\", a per-metric breakdown with each metric's average score (or pass rate for checks), how many rows it graded, and how many scored low. Use this to see how a prompt scored and to find the metric dragging a prompt down without listing responses. Poll it after runs_generate, runs_regrade or runs_rerun until status is completed or failed. The result includes a warning when no metrics are attached or when the judge's scores are not reproducible.",
32
+ annotations: Base::READ_ONLY,
33
33
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
34
34
  handler: :get
35
35
  },
36
36
  "runs_create" => {
37
- description: "Create a run. Omit prompt_id and provide output_column to score existing outputs by grading a pre-existing dataset column instead of generating new ones.",
37
+ description: "Set up a test run that scores a prompt's responses on a dataset. Give prompt_id and dataset_id to test a prompt: each dataset row fills the prompt's variables, the prompt's model answers it, and that answer gets scored. Omit prompt_id and provide output_column to score existing outputs by grading a pre-existing dataset column instead of generating new ones. Attach metrics with metric_ids or metric_group_id, and set judge_model, or LLM-judged metrics are skipped; with no metrics the run scores nothing. It fails if the dataset is missing a column the run needs, such as one the prompt uses, or if no configured provider serves judge_model. This only creates the run; call runs_generate to start it. It returns the new run, with a warning if no metrics are attached or if judge_temperature is above 0, which makes scores irreproducible.",
38
+ annotations: Base::ADDITIVE,
38
39
  inputSchema: {
39
40
  type: "object",
40
41
  properties: {
@@ -54,7 +55,8 @@ module CompletionKit
54
55
  handler: :create
55
56
  },
56
57
  "runs_update" => {
57
- description: "Update a run",
58
+ description: "Change a test run's settings: name, dataset, judge model, temperature, judge temperature, max tokens, output and expected columns, metrics or tags. The prompt cannot be changed. Passing metric_ids or metric_group_id replaces the run's whole metric list, and tag_names replaces all of its tags. It does not regenerate or rescore anything, and existing responses and scores stay as they are; call runs_regrade to rescore with the new metrics, or runs_rerun for a fresh run. It returns the updated run with the same warnings as runs_get.",
59
+ annotations: Base::DESTRUCTIVE,
58
60
  inputSchema: {
59
61
  type: "object",
60
62
  properties: {
@@ -74,27 +76,32 @@ module CompletionKit
74
76
  handler: :update
75
77
  },
76
78
  "runs_delete" => {
77
- description: "Delete a run",
79
+ description: "Permanently delete a test run and everything in it: its generated responses, their scores and judge feedback, the human agreement verdicts recorded on it (which also drops them from the judge's agreement stats), and any prompt suggestions made from it. This cannot be undone. The prompt, dataset and metrics are kept. Confirm with the user before calling. It returns a short confirmation message.",
80
+ annotations: Base::DESTRUCTIVE,
78
81
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
79
82
  handler: :delete
80
83
  },
81
84
  "runs_generate" => {
82
- description: "Start a run. Required for every run, including score-only runs (no prompt): generates responses with the prompt when there is one, otherwise copies the graded dataset column and grades it.",
85
+ description: "Start a test run so it produces and scores responses. It is required for every run, including score-only runs (no prompt): it generates responses with the prompt when there is one, and otherwise copies the graded dataset column and grades it. In a prompt run each dataset row goes to the prompt's model, and in either kind of run each LLM-judged metric then scores the result with the run's judge model. LLM-judged metrics are skipped when the run has no judge model. These model calls go to the providers you configured, which may charge for them; checks run without a provider call. The work runs in the background, so poll runs_get until status is completed or failed, and do not call this tool again to check progress. Only a pending or failed run can start, and any refused start marks the run as failed. Calling it on a running or completed run therefore does not start it but marks that run as failed, and a second call would then delete its results. Restarting a failed run first deletes its earlier responses, their scores and any human verdicts on them. For a completed run use runs_rerun, runs_retry_failures or runs_regrade instead. It returns the run with its progress, or the reason it could not start, such as a missing provider key or an empty dataset.",
86
+ annotations: Base::DESTRUCTIVE,
83
87
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
84
88
  handler: :generate
85
89
  },
86
90
  "runs_regrade" => {
87
- description: "Re-grade a run's existing responses with its currently attached metrics, without regenerating. Use after attaching or editing metrics on an already-generated run.",
91
+ description: "Re-score a test run's existing responses with its currently attached metrics, without regenerating. Use it after attaching or editing metrics on an already-generated run. It first clears every existing score and judge comment on the run's succeeded responses, including scores for metrics no longer attached, so the earlier scores are lost. If the run has checks but no judge model, its LLM-judged scores are cleared and not replaced. Human verdicts already recorded on the run are kept even though people gave them on the earlier scores, and agreement stats then pair them with the new scores. LLM-judged metrics call your judge model's provider; checks do not. The work runs in the background, so poll runs_get for progress. It returns the run, or an error when no configured provider serves the judge model, or when there is nothing to re-grade because the run has no succeeded responses, no metrics, or only LLM-judged metrics and no working judge model.",
92
+ annotations: Base::DESTRUCTIVE,
88
93
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
89
94
  handler: :regrade
90
95
  },
91
96
  "runs_rerun" => {
92
- description: "Create and start a fresh copy of a run with the same prompt, dataset, metrics, and settings. Use when the judge changed and you want a clean run instead of mixing versions.",
97
+ description: "Copy a test run and start the copy, with the same prompt version, dataset, metrics, judge model, settings and tags, under a new automatic name. Use it when the judge changed and you want a clean run instead of mixing versions, or to repeat a test. The original run and its scores stay untouched. The copy runs in the background and calls your model providers the same way runs_generate does. To test a different prompt version, create a new run with runs_create instead. It returns the new run; poll runs_get with its id for progress. If the prompt's model or the judge model has no configured provider, or the dataset no longer has a column the run needs, it returns an error and creates nothing. If the copy is created but cannot start, for example because the dataset is empty, it returns the reason and the copy stays as a failed run.",
98
+ annotations: Base::ADDITIVE,
93
99
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
94
100
  handler: :rerun
95
101
  },
96
102
  "runs_retry_failures" => {
97
- description: "Re-run only the failed responses of a run, optionally limited to specific response ids via \"only\".",
103
+ description: "Retry only the parts of a test run that failed, such as provider timeouts, rate limits or judge errors. Failed responses are generated again and failed scores are retried, while responses and scores that succeeded are left alone; optionally limit it to specific response ids via \"only\". It is refused when the judge has changed since the run executed, meaning a metric on the run now has a newer version than the one that scored it, because a retry would mix versions in the same run; use runs_rerun instead. The retries run in the background and call your model providers, except for checks. It returns the run with its progress, or the run unchanged when nothing failed.",
104
+ annotations: Base::ADDITIVE,
98
105
  inputSchema: {type: "object", properties: {id: {type: "integer"}, only: {type: "array", items: {type: "integer"}}}, required: ["id"]},
99
106
  handler: :retry_failures
100
107
  }
@@ -5,17 +5,20 @@ module CompletionKit
5
5
 
6
6
  TOOLS = {
7
7
  "tags_list" => {
8
- description: "List all tags",
8
+ description: "List every tag in CompletionKit, in name order. Returns each tag's id, name, color and timestamps. Tags label prompts, datasets, metrics, metric groups and runs. Use this to see which labels exist, or to find a tag's ID before renaming or deleting it.",
9
+ annotations: Base::READ_ONLY,
9
10
  inputSchema: {type: "object", properties: {}, required: []},
10
11
  handler: :list
11
12
  },
12
13
  "tags_get" => {
13
- description: "Get a tag by ID",
14
+ description: "Get one tag by ID. Returns its id, name, color and timestamps. Use it to check a tag's current name before you rename it or pass it in tag_names.",
15
+ annotations: Base::READ_ONLY,
14
16
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
15
17
  handler: :get
16
18
  },
17
19
  "tags_create" => {
18
- description: "Create a tag. Color is auto-assigned.",
20
+ description: "Create a tag for labelling prompts, datasets, metrics, metric groups and runs. The name is trimmed and lowercased. It may contain only the letters a to z, digits, spaces, hyphens and underscores (accented and non-Latin letters are rejected), can be up to 64 characters, and must not match an existing tag. Color is auto-assigned. Returns the new tag. You don't need this tool before tagging a record, because the create and update tools for those records take tag_names and create missing tags for you.",
21
+ annotations: Base::ADDITIVE,
19
22
  inputSchema: {
20
23
  type: "object",
21
24
  properties: {name: {type: "string"}},
@@ -24,7 +27,8 @@ module CompletionKit
24
27
  handler: :create
25
28
  },
26
29
  "tags_update" => {
27
- description: "Rename a tag.",
30
+ description: "Rename a tag. Pass the new name; leaving it out returns an error. The name follows the same rules as tags_create and can't match another tag, because tags are not merged. The new name shows at once on every prompt, dataset, metric, metric group and run that carries the tag. Returns the updated tag.",
31
+ annotations: Base::DESTRUCTIVE,
28
32
  inputSchema: {
29
33
  type: "object",
30
34
  properties: {id: {type: "integer"}, name: {type: "string"}},
@@ -33,7 +37,8 @@ module CompletionKit
33
37
  handler: :update
34
38
  },
35
39
  "tags_delete" => {
36
- description: "Delete a tag. Removes the tag from every linked metric, prompt, run, and dataset.",
40
+ description: "Delete a tag. Removes the tag from every linked metric, metric group, prompt, run, and dataset; those records themselves are kept. This cannot be undone. Use tags_update instead if you only want to rename it. Returns a confirmation message.",
41
+ annotations: Base::DESTRUCTIVE,
37
42
  inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
38
43
  handler: :delete
39
44
  }
@@ -103,7 +103,7 @@
103
103
  } %>
104
104
  <div class="ck-api-endpoint">
105
105
  <p class="ck-api-method"><span class="ck-chip ck-chip--soft">PATCH</span> /api/v1/prompts/:id</p>
106
- <p class="ck-meta-copy">Update a prompt. Accepts same params as create. If the prompt has no runs it is updated in place; if it already has runs, a new published version is created and returned (with a new id), leaving the original unchanged.</p>
106
+ <p class="ck-meta-copy">Update a prompt. Accepts same params as create. If the prompt has no runs it is updated in place; if it already has runs, a new version is created and returned (with a new id), leaving the original unchanged. The new version is published unless you send <code>"publish": false</code>, which saves it as an unpublished draft you can run and compare first, then publish with <code>POST /api/v1/prompts/:id/publish</code>. With <code>"publish": false</code>, a template or model change to the published version is saved as a draft even when it has no runs, so what your app fetches doesn't change. <code>publish</code> must be <code>true</code> or <code>false</code>; any other value returns 422.</p>
107
107
  <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X PATCH #{base_url}/api/v1/prompts/1 \\\n -H \"Authorization: Bearer #{token}\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"template\": \"Summarize concisely: {{text}}\"}'" %>
108
108
  </div>
109
109
  <div class="ck-api-endpoint">
@@ -372,11 +372,9 @@
372
372
  <div class="ck-api-endpoint" style="padding-top: 1rem;">
373
373
  <p class="ck-kicker" style="margin-bottom: 0.5rem;">MCP tools</p>
374
374
  <div class="ck-mcp-tools">
375
- <div class="ck-mcp-tool"><code class="ck-mcp-tool__name">tags_list</code><span class="ck-mcp-tool__desc">List all tags</span></div>
376
- <div class="ck-mcp-tool"><code class="ck-mcp-tool__name">tags_get</code><span class="ck-mcp-tool__desc">Get a tag by ID</span></div>
377
- <div class="ck-mcp-tool"><code class="ck-mcp-tool__name">tags_create</code><span class="ck-mcp-tool__desc">Create a tag (name required)</span></div>
378
- <div class="ck-mcp-tool"><code class="ck-mcp-tool__name">tags_update</code><span class="ck-mcp-tool__desc">Update a tag's name</span></div>
379
- <div class="ck-mcp-tool"><code class="ck-mcp-tool__name">tags_delete</code><span class="ck-mcp-tool__desc">Delete a tag and remove all its taggings</span></div>
375
+ <% CompletionKit::McpTools::Tags.definitions.each do |tool| %>
376
+ <div class="ck-mcp-tool"><code class="ck-mcp-tool__name"><%= tool[:name] %></code><span class="ck-mcp-tool__desc"><%= tool[:description] %></span></div>
377
+ <% end %>
380
378
  </div>
381
379
  <p class="ck-meta-copy" style="margin-top: 0.75rem;">The existing <code>metrics_create</code>, <code>metrics_update</code>, <code>prompts_create</code>, <code>prompts_update</code>, <code>runs_create</code>, <code>runs_update</code>, <code>datasets_create</code>, <code>datasets_create_from_url</code>, <code>datasets_update</code>, <code>metric_groups_create</code>, and <code>metric_groups_update</code> tools all accept a <code>tag_names</code> parameter with the same auto-create and replace semantics as the REST API.</p>
382
380
  </div>
@@ -2,7 +2,7 @@
2
2
  <% pc_delete_id = "ck_delete_provider_#{provider_credential.id}" %>
3
3
  <% pc_delete_confirm = "Delete the #{provider_credential.display_provider} provider and its discovered models? This can't be undone." %>
4
4
  <% end %>
5
- <%= form_with(model: provider_credential, local: true) do |form| %>
5
+ <%= form_with(model: provider_credential, url: local_assigns[:url], local: true) do |form| %>
6
6
  <% if provider_credential.errors.any? %>
7
7
  <div class="ck-flash ck-flash--alert" role="alert">
8
8
  <p class="ck-flash__title"><%= pluralize(provider_credential.errors.count, "problem") %> prevented this provider credential from being saved.</p>
@@ -66,7 +66,9 @@
66
66
  <% if provider_credential.persisted? %>
67
67
  <%= ck_delete_trigger(form_id: pc_delete_id, label: "Delete provider", confirm: pc_delete_confirm) %>
68
68
  <% end %>
69
- <%= link_to "Cancel", provider_credentials_path, class: ck_button_classes(:light, variant: :outline), tabindex: "0" %>
69
+ <% unless local_assigns[:cancel_path] == false %>
70
+ <%= link_to "Cancel", local_assigns[:cancel_path] || provider_credentials_path, class: ck_button_classes(:light, variant: :outline), tabindex: "0" %>
71
+ <% end %>
70
72
  <%= form.submit(provider_credential.persisted? ? "Save provider" : "Create provider", class: ck_button_classes(:dark)) %>
71
73
  </div>
72
74