@littlebigbrain/mcp 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/tools.js CHANGED
@@ -4,21 +4,65 @@ import { answerQuestion, } from "./question.js";
4
4
  import { commitAt, compareQuery, describeGraph, findNames, } from "./query-tools.js";
5
5
  import { registerRdfTool } from "./rdf-tool.js";
6
6
  import { registerFilesTool } from "./files-tool.js";
7
+ import { registerReviewsTools } from "./reviews-tool.js";
8
+ import { APPROVAL_TEXT, graphLabel, oneLine, ontologyRemovalMessage, retractMessage, shapesRejectMessage, toolHost, } from "./approval.js";
7
9
  import { queryTiming } from "./query-observer.js";
8
- import { DESTRUCTIVE, IDEMPOTENT_WRITE, MUTATING, READ_ONLY, configureInputSchema, configureWireSchema, detailSchema, graphScope, inspectInputSchema, inspectWireSchema, jsonObjectSchema, queryInputSchema, queryWireSchema, searchFeedbackSchema, yearMonthSchema, } from "./tool-contracts.js";
9
- import { analyze, assertCursorScope, contentHashKey, decodeQueryCursor, effectiveRowLimit, enrichError, errorResult, guide, normalizeDetail, normalizeLbbIris, ontologyDefineBody, queryCommitPin, queryEnvelope, queryToolResult, requireString, rowPageFrom, rowPageNext, run, scoped, stableJson, toolResult, } from "./tool-runtime.js";
10
+ import { DESTRUCTIVE, HARD_OUTPUT_CHARS, MUTATING, READ_ONLY, configureInputSchema, configureWireSchema, detailSchema, entityEmbeddingSchema, evalDatasetItemSchema, evalLabelItemSchema, evidenceSchema, graphScope, inspectInputSchema, inspectWireSchema, jsonObjectSchema, queryInputSchema, queryWireSchema, reviewAnswerSchema, searchFeedbackSchema, suggestedFactsBody, testInstallSchema, yearMonthSchema, } from "./tool-contracts.js";
11
+ import { analyze, assertCursorScope, contentHashKey, decodeQueryCursor, effectiveRowLimit, enrichError, errorResult, guide, normalizeDetail, normalizeLbbIris, ontologyDefineBody, pageLimit, queryCommitPin, queryEnvelope, queryToolResult, requireString, rowPageFrom, rowPageNext, run, runPage, scoped, stableJson, toolResult, ToolError, } from "./tool-runtime.js";
12
+ /** The most tests `GET /v1/workflows/named/tests` returns in one page. */
13
+ const WORKFLOW_TESTS_PAGE_MAX = 50;
14
+ /** The most items `GET /v1/workflows/named/queue` returns in one page. */
15
+ const WORKFLOW_QUEUE_PAGE_MAX = 100;
16
+ /** The most inputs `GET /v1/workflows/named/runs/results` returns in one page. */
17
+ const WORKFLOW_RUN_RESULTS_PAGE_MAX = 200;
18
+ /** The most datasets `GET /v1/evals/datasets` returns in one page. */
19
+ const EVAL_DATASETS_PAGE_MAX = 50;
10
20
  export function registerLbbTools(server, client, options = {}) {
11
- registerRdfTool(server, client);
12
- registerFilesTool(server, client);
13
- server.registerTool("lbb_inspect", {
14
- description: "Read graph context and exact graph facts. Actions: guide, graphs, publication, ontology, ontology_conformance, schema, ontology_search, metadata, entity, ontology_suggestions. graphs works before bootstrap; ontology_suggestions lists the ontology change suggestions that wait for review (or were decided); fact_count is how many graph facts (triplets and entity property rows) accepting one commits, and identity_count how many identities it links. publication reports whether writes are queryable. ontology and schema return complete entries with page_size, section and cursor; follow next until absent. schema reads active native ontology/SHACL metadata without running validation. Query asserted RDF/OWL axioms separately with lbb_query. ontology_conformance serves the durable report referenced by the pinned published root. entity returns one node's attributes and current relationships from the RDF read. For a node's past values, run SPARQL with as_of_commit_seq through lbb_query. Use lbb_query with SPARQL property paths for precise path selection.",
21
+ const host = toolHost(server, options);
22
+ registerRdfTool(host, client);
23
+ registerFilesTool(host, client);
24
+ registerReviewsTools(host, client);
25
+ host.register("lbb_inspect", {
26
+ description: "Read graph context and exact graph facts. Actions: guide, graphs, publication, ontology, ontology_conformance, schema, ontology_search, metadata, entity, ontology_suggestions. graphs works before bootstrap; ontology_suggestions lists the ontology change suggestions that wait for review (or were decided); fact_count is how many graph facts (triplets and entity property rows) accepting one commits, and identity_count how many identities it links. publication reports whether writes are queryable. ontology and schema return complete entries with page_size, section and cursor; follow next until absent. schema reads active native ontology/SHACL metadata without running validation. Query asserted RDF/OWL axioms separately with lbb_query. ontology_conformance serves the durable report referenced by the pinned published root. entity with iri, or version=records/1 and entity_id or entity_type+key, returns a complete exact-term record, separate assertions and snapshot from RDF. as_of_commit_seq pins retained history; schema-bound historical selection is currently unsupported. Oversized records fail rather than truncate. Omitting the new selectors preserves the native attributes/relationships view. Use lbb_query with SPARQL property paths for precise path selection.",
15
27
  inputSchema: inspectWireSchema,
16
28
  annotations: READ_ONLY,
17
- }, (rawArgs) => {
29
+ }, async (rawArgs) => {
18
30
  const parsed = inspectInputSchema.safeParse(rawArgs);
19
31
  if (!parsed.success)
20
32
  return errorResult(parsed.error);
21
33
  const args = parsed.data;
34
+ if (args.action === "entity" &&
35
+ (args.version ||
36
+ args.iri !== undefined ||
37
+ args.key !== undefined ||
38
+ args.schema_revision !== undefined ||
39
+ args.limits !== undefined)) {
40
+ try {
41
+ if (args.as_of !== undefined)
42
+ throw new ToolError("record_selector_invalid", "entity valid-time as_of is not supported; use as_of_commit_seq");
43
+ if (args.name !== undefined)
44
+ throw new ToolError("record_selector_invalid", "records/1 uses iri, native entity_id, or entity_type and key; name is not a stable selector");
45
+ const data = await scoped(client, args.graph).entities.retrieve({
46
+ iri: args.iri,
47
+ id: args.entity_id,
48
+ type: args.entity_type,
49
+ key: args.key,
50
+ schemaRevision: args.schema_revision,
51
+ consistency: args.consistency,
52
+ asOfCommitSeq: args.as_of_commit_seq,
53
+ limits: args.limits,
54
+ });
55
+ const result = { summary: "lbb_inspect.entity", data };
56
+ if (Buffer.byteLength(JSON.stringify(result, null, 2), "utf8") >
57
+ HARD_OUTPUT_CHARS) {
58
+ throw new ToolError("record_read_limit", "The complete record exceeds the MCP output budget. Use the HTTP entity retrieve API or a bounded SPARQL query; no record terms were returned.");
59
+ }
60
+ return toolResult(result);
61
+ }
62
+ catch (error) {
63
+ return errorResult(await enrichError(client, error));
64
+ }
65
+ }
22
66
  if (args.action === "ontology" || args.action === "schema") {
23
67
  return metadataPage(client, args)
24
68
  .then(toolResult)
@@ -61,12 +105,13 @@ export function registerLbbTools(server, client, options = {}) {
61
105
  name: requireString(args.name, "name"),
62
106
  }),
63
107
  asOfCommitSeq: args.as_of_commit_seq,
108
+ consistency: args.consistency,
64
109
  });
65
110
  }
66
111
  });
67
112
  });
68
- server.registerTool("lbb_query", {
69
- description: "Analytical and expert reads. Modes: question (a question in plain words), structured (SPARQL-subset JSON body), sparql (SPARQL text), search (instances by meaning over every searchable class; filter narrows by class and relationship; every hit checked against the graph), analyze, and four tools for writing your own queries: names, describe, commit_at, compare. names: before a query that names an entity (a person, a company), find its IRI with text=<the question or the names>; use the first candidate, not a CONTAINS match on the name. describe: before a query on classes or properties you have not seen, read them with question=<the question> (or classes/properties as IRIs): how many sampled instances hold each property (a filter on a rare one returns few rows), the values of small classes such as stages, examples and schema statements. commit_at: for a question about a date, find the commit of date=YYYY-MM-DD (or moment=RFC 3339), then run mode=sparql as_of_commit_seq=<it>. compare: for what changed between two points, run one SELECT at before and after (default the latest) with key=[the entity variable]; added, removed and changed come with totals and pages (next). Use mode=question when you have a question in plain words and no SPARQL query: the server picks the kind of question (route), runs queries in a bounded loop, reads their rows, and answers in plain words (about 8 s). The result holds the answer with citations and a chart hint (kind, x, y: columns of the rows), the steps, the route, and the query and rows the answer stands on. Continue or correct that query with mode=sparql. Each question uses model tokens and counts toward a daily limit of the stack. SPARQL is the query language; search finds what the words describe. SPARQL text can also search by meaning inside the query with ?x <https://littlebigbrain.com/search#similarTo> \"words\", so the other patterns filter and join the hits in one query; the result's search field reports the plan. To plan a search: lbb_embeddings action=list, then SPARQL for a class's relationships on a sample, then mode=search with explain=true to check the resolved filter before the real search (lbb_inspect action=guide has the queries). Relations are <https://littlebigbrain.com/r/NAME> and types <https://littlebigbrain.com/class/NAME> (both lowercased); entities are content-addressed, so anchor a named one by its rdfs:label rather than building its IRI. Structured and text queries pin one published watermark for the request.",
113
+ host.register("lbb_query", {
114
+ description: "Analytical and expert reads. Modes: question (a question in plain words), structured (SPARQL-subset JSON body), sparql (SPARQL text), search (instances by meaning over every searchable class; filter narrows by class and relationship; every hit checked against the graph), analyze, and four tools for writing your own queries: names, describe, commit_at, compare. names: before a query that names an entity (a person, a company), find its IRI with text=<the question or the names>; use the first candidate, not a CONTAINS match on the name. describe: before a query on classes or properties you have not seen, read them with question=<the question> (or classes/properties as IRIs): how many sampled instances hold each property (a filter on a rare one returns few rows), the values of small classes such as stages, examples and schema statements. commit_at: for a question about a date, find the commit of date=YYYY-MM-DD (or moment=RFC 3339), then run mode=sparql as_of_commit_seq=<it>. compare: for what changed between two points, run one SELECT at before and after (default the latest) with key=[the entity variable]; added, removed and changed come with totals and pages (next). Use mode=question when you have a question in plain words and no SPARQL query: the server picks the kind of question (route), runs queries in a bounded loop, reads their rows, and answers in plain words (about 8 s). The result holds the answer with citations and a chart hint (kind, x, y: columns of the rows), the steps, the route, and the query and rows the answer stands on. Continue or correct that query with mode=sparql. Each question uses model tokens and counts toward a daily limit of the stack. SPARQL is the query language; search finds what the words describe. SPARQL text can also search by meaning inside the query with ?x <https://littlebigbrain.com/search#similarTo> \"words\", so the other patterns filter and join the hits in one query; the result's search field reports the plan. To plan a search: lbb_embeddings action=list, then SPARQL for a class's relationships on a sample, then mode=search with explain=true to check the resolved filter before the real search (lbb_inspect action=guide has the queries). Use the exact IRIs returned by names, describe or RDF. Native records use content-addressed entity IRIs, <https://littlebigbrain.com/r/NAME> relations and <https://littlebigbrain.com/class/NAME> types (lowercased); imported RDF may use any vocabulary. Never build an entity IRI from a display name. Structured and text queries pin one published watermark for the request.",
70
115
  inputSchema: queryWireSchema,
71
116
  annotations: READ_ONLY,
72
117
  }, (rawArgs, extra) => {
@@ -128,9 +173,14 @@ export function registerLbbTools(server, client, options = {}) {
128
173
  "consistency",
129
174
  "min_indexed_seq",
130
175
  ]) {
176
+ const supplied = key === "entailment"
177
+ ? args.mode === "sparql"
178
+ ? args.entailment
179
+ : undefined
180
+ : args[key];
131
181
  if (cursor &&
132
- args[key] !== undefined &&
133
- args[key] !== cursor[key]) {
182
+ supplied !== undefined &&
183
+ supplied !== cursor[key]) {
134
184
  throw new Error(`cursor ${key} does not match the supplied ${key}`);
135
185
  }
136
186
  }
@@ -191,7 +241,9 @@ export function registerLbbTools(server, client, options = {}) {
191
241
  request.combinators.length > 0) {
192
242
  throw new Error("`combinators` (UNION/OPTIONAL/MINUS/EXISTS) is no longer accepted by structured mode; the analytics route was removed. Express the same query as SPARQL text with mode=sparql.");
193
243
  }
194
- const response = await timing.measureAsync("query_http_total", () => target.sparql(request, {
244
+ const response = await timing.measureAsync("query_http_total", () =>
245
+ // The body is the caller's JSON; the server checks it.
246
+ target.sparql(request, {
195
247
  consistency,
196
248
  minIndexedSeq,
197
249
  }));
@@ -357,7 +409,7 @@ export function registerLbbTools(server, client, options = {}) {
357
409
  });
358
410
  });
359
411
  });
360
- server.registerTool("lbb_models", {
412
+ host.register("lbb_models", {
361
413
  description: "Read model-training inputs or compare retrieval configurations over one pinned published snapshot, or read what the managed models did. shadow_eval takes the API ShadowEvalRequest body; dataset actions return bounded training examples at an optional signal split. activity reads one month of the stack's model use (all graphs): calls, items, estimated tokens and cost per feature (index, search, fit, judge, training, rerank, rewrite, trial, workflows) and model, by day and by graph, the workflows' calls per workflow and job (by_workflow), the months with activity, and the model each feature uses now.",
362
414
  inputSchema: {
363
415
  action: z.enum([
@@ -382,6 +434,7 @@ export function registerLbbTools(server, client, options = {}) {
382
434
  case "shadow_eval":
383
435
  if (!body)
384
436
  throw new Error("shadow_eval requires body");
437
+ // The body is the caller's JSON; the server checks it.
385
438
  return target.shadowEval(body);
386
439
  case "suggest_dataset":
387
440
  return target.suggestDataset({ limit, splitSeq: split_seq });
@@ -424,7 +477,7 @@ export function registerLbbTools(server, client, options = {}) {
424
477
  ...(args.dim ? { dim: args.dim } : {}),
425
478
  };
426
479
  };
427
- server.registerTool("lbb_embeddings", {
480
+ host.register("lbb_embeddings", {
428
481
  description: "Search setup, read only: embeddings declared on classes (an embedding on a class covers its subclasses; one model per graph). list shows each embedding's status (serving and building version, backfill progress, lag) and the graph's model; get shows one; preview shows every candidate fact of a class with its coverage and examples, and the exact text of sample instances, and stores nothing. Declare, refresh, or change the model with lbb_embeddings_manage; search with lbb_query mode=search. list is the first step of planning a search (lbb_inspect action=guide has the recipe).",
429
482
  inputSchema: {
430
483
  action: z.enum(["list", "get", "preview"]),
@@ -456,7 +509,7 @@ export function registerLbbTools(server, client, options = {}) {
456
509
  });
457
510
  }
458
511
  }));
459
- server.registerTool("lbb_embeddings_manage", {
512
+ host.register("lbb_embeddings_manage", {
460
513
  description: "Declare or refresh an embedding, or change the graph's model. Run lbb_embeddings action=preview first and show the user the text it will embed. declare sets the class, the fields (one-hop property paths such as label, description, calls/label; omit for an automatic choice); a new embedding takes the graph's model, and another model is refused. On an existing embedding it changes only what you name, and a changed recipe builds a new version while the old one serves. model moves every embedding of the graph to `model`: each builds a new version, and the graph switches when all are ready (a search never mixes two models). refresh runs one step of the embed job now (it also runs by itself after each published commit).",
461
514
  inputSchema: {
462
515
  action: z.enum(["declare", "refresh", "model"]),
@@ -487,21 +540,22 @@ export function registerLbbTools(server, client, options = {}) {
487
540
  });
488
541
  }
489
542
  }));
490
- server.registerTool("lbb_embeddings_delete", {
491
- description: "Delete an embedding: its name and its vectors (index-gc removes the stored runs). Its class stops being searchable until it is declared and built again. Ask the user first; confirm must repeat the name.",
543
+ host.register("lbb_embeddings_delete", {
544
+ description: `Delete an embedding: its name and its vectors (index-gc removes the stored runs). Its class stops being searchable until it is declared and built again. ${APPROVAL_TEXT}`,
492
545
  inputSchema: {
493
546
  name: z.string().describe("The embedding name."),
494
- confirm: z.string().describe("The same name again, to confirm."),
495
547
  detail: detailSchema,
496
548
  ...graphScope,
497
549
  },
498
550
  annotations: DESTRUCTIVE,
499
- }, ({ name, confirm, detail, graph }) => run(client, "lbb_embeddings_delete", detail, () => {
500
- if (confirm !== name) {
501
- throw new Error("confirm must repeat the embedding name");
502
- }
503
- return scoped(client, graph).embeddings.delete(name);
504
- }));
551
+ }, async ({ name, detail, graph }, extra) => {
552
+ const refused = await host.approve(extra, {
553
+ message: `Delete the embedding "${oneLine(name)}" of ${graphLabel(client, graph)} and its vectors? Its class stops being searchable until it is declared and built again.`,
554
+ });
555
+ if (refused)
556
+ return refused;
557
+ return run(client, "lbb_embeddings_delete", detail, () => scoped(client, graph).embeddings.delete(name));
558
+ });
505
559
  // Fit from text: fit sources declared on classes, like embeddings.
506
560
  const fitSetup = {
507
561
  class: z
@@ -532,7 +586,7 @@ export function registerLbbTools(server, client, options = {}) {
532
586
  ...(args.context ? { context: args.context } : {}),
533
587
  };
534
588
  };
535
- server.registerTool("lbb_fit_sources", {
589
+ host.register("lbb_fit_sources", {
536
590
  description: "Fit from text, read only: fit sources declared on classes. A fit source names the fields of a class that hold text (transcripts, documents); the server reads every instance, proposes ontology changes (classes, properties, relations) with verbatim quotes, checks them, and files them as ontology suggestions with origin id fit:<name>. list shows each source's status (progress, lag, instances read, proposals kept and dropped, suggestions per status, this month's spend, last error) and the stack's monthly budget; get shows one; preview shows the fields, every candidate field of the class and the text of sample instances, calls no model and stores nothing. Declare, refresh or dry-run with lbb_fit_sources_manage; list the suggestions with lbb_inspect action=ontology_suggestions.",
537
591
  inputSchema: {
538
592
  action: z.enum(["list", "get", "preview"]),
@@ -564,7 +618,7 @@ export function registerLbbTools(server, client, options = {}) {
564
618
  });
565
619
  }
566
620
  }));
567
- server.registerTool("lbb_fit_sources_manage", {
621
+ host.register("lbb_fit_sources_manage", {
568
622
  description: "Declare, dry-run or refresh a fit source (fit from text). Run lbb_fit_sources action=preview first and show the user the text the fit reads. dry_run runs the models on the first part of up to 3 instances (at most 60 s; truncated when one did not finish) and returns every proposal with its quote, its support score and what the checks made of it; it files nothing but spends model budget. declare sets the class, the fields and the context; the job then reads every instance and files suggestions (an existing source changes only what you name). refresh asks the job to run now and returns the status at once; the job also runs by itself after each published commit, so poll lbb_fit_sources action=get for progress. Nothing changes the ontology until a person accepts a suggestion.",
569
623
  inputSchema: {
570
624
  action: z.enum(["declare", "dry_run", "refresh"]),
@@ -590,21 +644,22 @@ export function registerLbbTools(server, client, options = {}) {
590
644
  return fit.refresh(args.name);
591
645
  }
592
646
  }));
593
- server.registerTool("lbb_fit_sources_delete", {
594
- description: "Delete a fit source: the fit stops reading its class. The suggestions it filed stay. Ask the user first; confirm must repeat the name.",
647
+ host.register("lbb_fit_sources_delete", {
648
+ description: `Delete a fit source: the fit stops reading its class. The suggestions it filed stay. ${APPROVAL_TEXT}`,
595
649
  inputSchema: {
596
650
  name: z.string().describe("The fit source name."),
597
- confirm: z.string().describe("The same name again, to confirm."),
598
651
  detail: detailSchema,
599
652
  ...graphScope,
600
653
  },
601
654
  annotations: DESTRUCTIVE,
602
- }, ({ name, confirm, detail, graph }) => run(client, "lbb_fit_sources_delete", detail, () => {
603
- if (confirm !== name) {
604
- throw new Error("confirm must repeat the fit source name");
605
- }
606
- return scoped(client, graph).ontology.fitSources.delete(name);
607
- }));
655
+ }, async ({ name, detail, graph }, extra) => {
656
+ const refused = await host.approve(extra, {
657
+ message: `Delete the fit source "${oneLine(name)}" of ${graphLabel(client, graph)}? The fit stops reading its class. The suggestions it filed stay.`,
658
+ });
659
+ if (refused)
660
+ return refused;
661
+ return run(client, "lbb_fit_sources_delete", detail, () => scoped(client, graph).ontology.fitSources.delete(name));
662
+ });
608
663
  // Starter workflows and triggered workflows: one API over everything that
609
664
  // runs when a class of the graph changes. search.embed and ontology.fit
610
665
  // are the embeddings and fit sources above, through the same stored
@@ -652,9 +707,9 @@ export function registerLbbTools(server, client, options = {}) {
652
707
  .number()
653
708
  .int()
654
709
  .min(1)
655
- .max(64)
710
+ .max(500)
656
711
  .optional()
657
- .describe("preview/use, developer workflow only: instances per message, 1 to 64 (default 16)."),
712
+ .describe("preview/use and define, developer workflow only: instances per message, 1 to 64 (default 16); for a stream, events per message, 1 to 500 (default 100). The server checks each kind's bound."),
658
713
  };
659
714
  const workflowRequest = (args) => {
660
715
  if (!args.starter)
@@ -662,6 +717,9 @@ export function registerLbbTools(server, client, options = {}) {
662
717
  if (!args.class && (args.fields || args.exclude)) {
663
718
  throw new Error("fields and exclude need class");
664
719
  }
720
+ if (args.batch !== undefined && args.batch > 64) {
721
+ throw new Error("batch is 1 to 64 instances per message");
722
+ }
665
723
  return {
666
724
  starter: args.starter,
667
725
  ...(args.name ? { name: args.name } : {}),
@@ -695,6 +753,11 @@ export function registerLbbTools(server, client, options = {}) {
695
753
  throw new Error("this action requires name");
696
754
  return args.name;
697
755
  };
756
+ const revisionId = (revision) => {
757
+ if (!revision)
758
+ throw new Error("this action requires revision");
759
+ return revision;
760
+ };
698
761
  // A workflow with triggers (`/v1/workflows/named`): your worker's
699
762
  // workflow type, one watch trigger, shards, routing and window.
700
763
  const namedSetup = {
@@ -719,33 +782,129 @@ export function registerLbbTools(server, client, options = {}) {
719
782
  trigger: z
720
783
  .string()
721
784
  .optional()
722
- .describe("define: the watch trigger's id (default watch). queue: the trigger whose queue to read."),
785
+ .describe("define: the trigger's id (default watch, content, schedule or stream). queue: the trigger whose queue to read."),
786
+ media_types: z
787
+ .array(z.string())
788
+ .min(1)
789
+ .max(32)
790
+ .optional()
791
+ .describe("define, in place of class: a content trigger on the uploads of these media types (application/pdf, image/*, */*), one content per message, without a wait for a publication."),
792
+ schedule: z
793
+ .object({
794
+ cron: z
795
+ .string()
796
+ .optional()
797
+ .describe("Five fields: minute, hour, day of month, month, day of week (0 3 * * 1-5), or @daily and the other macros."),
798
+ timezone: z
799
+ .string()
800
+ .optional()
801
+ .describe("With cron: an IANA time zone (Europe/Berlin). Default UTC."),
802
+ every: z
803
+ .string()
804
+ .optional()
805
+ .describe("In place of cron: a period in UTC, 5m to 366d (15m, 6h, 1d)."),
806
+ offset: z
807
+ .string()
808
+ .optional()
809
+ .describe("With every: moves each slot (every 1d, offset 3h runs at 03:00 UTC)."),
810
+ jitter: z
811
+ .string()
812
+ .optional()
813
+ .describe("Each slot runs up to this much later, the same delay each time; at most 1h."),
814
+ })
815
+ .optional()
816
+ .describe("define, in place of class: a schedule. One message per slot, at least 5 minutes apart, to one shard; a slot runs even when nothing changed (the message's changed is false)."),
817
+ stream: z
818
+ .object({
819
+ max_wait: z
820
+ .string()
821
+ .optional()
822
+ .describe("How long the first event of a partial batch waits, 1s to 15m (default 10s)."),
823
+ })
824
+ .optional()
825
+ .describe("define, in place of class: a stream of the events a client sends (POST /v1/workflows/named/events), in batches of batch events (1 to 500, default 100) or after max_wait; each event's key routes it to a shard, and shards stay as made."),
826
+ on_exhausted: z
827
+ .enum(["block", "skip"])
828
+ .optional()
829
+ .describe("define: what a turn that failed for good does to its shard: block (the default) holds the inbox until a person retries or skips it; skip keeps the failed turn in the history and goes on."),
830
+ fan_in: z
831
+ .string()
832
+ .optional()
833
+ .describe("define: the id of the workflow's fan-in identity (starter.<name>.<fan_in>) that the shards' turns send to, for example a catalogue. Default: none."),
723
834
  };
724
835
  const namedRequest = (args) => {
725
- if (!args.name || !args.class || !args.workflow) {
726
- throw new Error("define requires name, class and workflow");
836
+ if (!args.name || !args.workflow) {
837
+ throw new Error("define requires name and workflow");
838
+ }
839
+ const kinds = [
840
+ args.class,
841
+ args.media_types,
842
+ args.schedule,
843
+ args.stream,
844
+ ].filter((kind) => kind !== undefined).length;
845
+ if (kinds !== 1) {
846
+ throw new Error("define takes one trigger: class (a watch), media_types (a content trigger), schedule or stream");
727
847
  }
848
+ const trigger = args.stream
849
+ ? { stream: args.stream }
850
+ : args.schedule
851
+ ? { schedule: args.schedule }
852
+ : args.media_types
853
+ ? { content: { media_types: args.media_types } }
854
+ : {
855
+ watch: {
856
+ class: args.class,
857
+ ...(args.fields ? { fields: args.fields } : {}),
858
+ },
859
+ };
728
860
  return {
729
861
  name: args.name,
730
862
  workflow: args.workflow,
731
- triggers: [
732
- {
733
- ...(args.trigger ? { id: args.trigger } : {}),
734
- watch: {
735
- class: args.class,
736
- ...(args.fields ? { fields: args.fields } : {}),
737
- },
738
- },
739
- ],
863
+ triggers: [{ ...(args.trigger ? { id: args.trigger } : {}), ...trigger }],
740
864
  ...(args.owns ? { owns: args.owns } : {}),
741
865
  ...(args.batch !== undefined ? { batch: args.batch } : {}),
742
866
  ...(args.shards !== undefined ? { shards: args.shards } : {}),
743
867
  ...(args.route_by ? { route_by: args.route_by } : {}),
744
868
  ...(args.window !== undefined ? { window: args.window } : {}),
869
+ ...(args.budget_usd_per_month !== undefined
870
+ ? { budget_usd_per_month: args.budget_usd_per_month }
871
+ : {}),
872
+ ...(args.on_exhausted ? { on_exhausted: args.on_exhausted } : {}),
873
+ ...(args.fan_in ? { fan_in: args.fan_in } : {}),
874
+ };
875
+ };
876
+ // A workflow test (`client.workflows.tests`) runs a values.extract
877
+ // workflow on chosen inputs and writes nothing to the graph.
878
+ const testId = (args) => {
879
+ if (!args.id)
880
+ throw new Error("this action requires id");
881
+ return args.id;
882
+ };
883
+ // The one place that builds a test's request: a workflow of the graph by
884
+ // `name`, or a workflow to create (`install`).
885
+ const testModel = z.object({
886
+ provider: z.enum(["anthropic", "typesafe", "mock"]),
887
+ model: z.string(),
888
+ effort: z.enum(["low", "medium", "high", "xhigh", "max"]).optional(),
889
+ });
890
+ const testRequest = (args) => {
891
+ const inputs = {
892
+ ...(args.iris ? { iris: args.iris } : {}),
893
+ ...(args.sample !== undefined ? { sample: args.sample } : {}),
894
+ ...(args.models ? { models: args.models } : {}),
745
895
  };
896
+ if (args.install) {
897
+ if (args.name) {
898
+ throw new Error("tests_create takes name (a workflow of the graph) or install (a workflow to create), and not both");
899
+ }
900
+ return { install: args.install, ...inputs };
901
+ }
902
+ if (!args.name)
903
+ throw new Error("tests_create requires name or install");
904
+ return { name: args.name, ...inputs };
746
905
  };
747
- server.registerTool("lbb_workflows", {
748
- description: "Starter workflows and triggered workflows, read only. A starter workflow is a template; a triggered workflow is a workflow created from one on the graph. It watches a class: when a published commit adds or changes instances of the class (or changes the watched fields), the workflow reads them and does its work. search.embed keeps an embedding of the class (the same as lbb_embeddings); ontology.fit reads text fields and files ontology suggestions (the same as lbb_fit_sources); workflow (a developer workflow) sends the new and changed instances to a workflow of the developer's worker as trigger messages, and that code writes the facts the workflow owns. documents.parse reads uploaded PDF files into pages with line boxes, documents.link connects mentions of the same thing across the documents (it writes DocEntry and DocMention instances and links each mention to the entry that defines or describes it), documents.map proposes each link as a FactProposal in the graph's own ontology (fact_subject, fact_predicate, fact_object, fact_confidence, fact_evidence) for a review workflow, and values.extract fills chosen properties from text, each value with a quote; read the files and their pages with lbb_files. starters lists the starter workflows this server offers with their group, settings and whether each can run here. list shows every triggered workflow of the graph with its state (backfilling, ready, paused, failed, unavailable), lag, counters, this month's spend and last error; get shows one (starter and name) with its own status in details (for workflow: the messages sent, the instances waiting, the message in flight and whether it is paused). preview shows what creating one would do (the resolved watch, the instance count and the workflow's own preview; for workflow, the first messages it would send), calls no model and stores nothing. workflows lists the same workflows as workflows with triggers (/v1/workflows/named): each one's target workflow type (LBB's own are lbb.* types), its shards (engine identities starter.<name> and starter.<name>.<k>), its routing field, its window, its one watch trigger, its status and each shard's inbox; name for one. queue (name) shows what a workflow's watch has not sent yet: each waiting instance with its shard and its origin (a backfill, a commit range with the watched fields and the requests that changed it, a run now, a schema change), the pass in progress, and per shard its message in flight and its first unfinished turns with their origins; page with after and limit. Message workflows of the workflow engine are not listed here. Create, refresh, pause or resume with lbb_workflows_manage; delete with lbb_workflows_delete.",
906
+ host.register("lbb_workflows", {
907
+ description: "Starter workflows and triggered workflows, read only. A starter workflow is a template; a triggered workflow is a workflow created from one on the graph. It watches a class: when a published commit adds or changes instances of the class (or changes the watched fields), the workflow reads them and does its work. search.embed keeps an embedding of the class (the same as lbb_embeddings); ontology.fit reads text fields and files ontology suggestions (the same as lbb_fit_sources); workflow (a developer workflow) sends the new and changed instances to a workflow of the developer's worker as trigger messages, and that code writes the facts the workflow owns. documents.parse reads uploaded PDF files into pages with line boxes, documents.link connects mentions of the same thing across the documents (it writes DocEntry and DocMention instances and links each mention to the entry that defines or describes it), documents.map proposes each link as a FactProposal in the graph's own ontology (fact_subject, fact_predicate, fact_object, fact_confidence, fact_evidence) for a review workflow, and values.extract fills chosen properties from text, each value with a quote; read the files and their pages with lbb_files. starters lists the starter workflows this server offers with their group, settings and whether each can run here. list shows every triggered workflow of the graph with its state (backfilling, ready, paused, failed, unavailable), lag, counters, this month's spend and last error; get shows one (starter and name) with its own status in details (for workflow: the messages sent, the instances waiting, the message in flight and whether it is paused). preview shows what creating one would do (the resolved watch, the instance count and the workflow's own preview; for workflow, the first messages it would send), calls no model and stores nothing. workflows lists the same workflows as workflows with triggers (/v1/workflows/named): each one's target workflow type (LBB's own are lbb.* types), its shards (engine identities starter.<name> and starter.<name>.<k>), its routing field, its window, its one watch trigger, its status and each shard's inbox, one page in data (has_more and next_cursor page them with cursor and limit); name for one. queue (name) shows what a workflow's watch has not sent yet: each waiting instance with its shard and its origin (a backfill, a commit range with the watched fields and the requests that changed it, a run now, a schema change), the pass in progress, and per shard its message in flight and its first unfinished turns with their origins; the items are in data, and has_more and next_cursor page them with cursor and limit. run_results (name) shows what the workflow's runs did with each input, newest run first: the input's IRI and label, its outcome (written, review, no_value, failed, duplicate) and its values (the value, the quote that states it, its support, the conversion, and the review item it went to); run keeps one run (with run_counts, every input by outcome), outcome keeps some outcomes, iri keeps one input; data, has_more and next_cursor page them. A run records its results when it ends; they are kept 30 days. revisions (name) shows the history of a workflow's settings, newest change first: each item is a revision (settings, the request that makes it, and an id wrev_… that is the hash of the settings) with who changed it, when, how (create, change, activate, existing) and bound (the one in use); a row whose revision was deleted has available false and no settings; has_more and next_cursor page it with cursor and limit (1 to 100, default 20). revision (name, revision) reads one revision; compare two by their settings to say what changed. Each run summary names the revision it ran on. Message workflows of the workflow engine are not listed here. tests lists the graph's workflow tests, newest first, without texts and results (name: one workflow's tests), one page with next_cursor; tests_get reads one test (id) with its inputs' texts and, per input, each value read with its quote, its support and what the live workflow does with it (writes it, gives it to review, or drops it). A test goes queued, running, then done, failed or cancelled. Create, refresh, pause or resume with lbb_workflows_manage, which also starts, answers and cancels tests and goes back to an earlier revision with its action activate; delete with lbb_workflows_delete.",
749
908
  inputSchema: {
750
909
  action: z.enum([
751
910
  "starters",
@@ -754,26 +913,46 @@ export function registerLbbTools(server, client, options = {}) {
754
913
  "preview",
755
914
  "workflows",
756
915
  "queue",
916
+ "revisions",
917
+ "revision",
918
+ "run_results",
919
+ "tests",
920
+ "tests_get",
757
921
  ]),
758
922
  name: z
759
923
  .string()
760
924
  .optional()
761
- .describe("get: the triggered workflow's name. preview: the name a new workflow would get. workflows: one workflow's name (omit for all). queue: required."),
925
+ .describe("get: the triggered workflow's name. preview: the name a new workflow would get. workflows: one workflow's name (omit for all). queue, run_results, revisions and revision: required. tests: one workflow's tests (omit for all)."),
926
+ revision: z
927
+ .string()
928
+ .optional()
929
+ .describe("revision: the revision's id (wrev_ and 32 hex characters)."),
930
+ run: z
931
+ .string()
932
+ .optional()
933
+ .describe("run_results: one run's results (a run id), in input order."),
934
+ outcome: z
935
+ .array(z.enum(["written", "review", "no_value", "failed", "duplicate"]))
936
+ .optional()
937
+ .describe("run_results: only the inputs of these outcomes."),
938
+ iri: z
939
+ .string()
940
+ .optional()
941
+ .describe("run_results: only the results of this input."),
762
942
  ...workflowSetup,
763
943
  trigger: namedSetup.trigger,
764
- after: z
765
- .number()
766
- .int()
767
- .nonnegative()
944
+ cursor: z
945
+ .string()
768
946
  .optional()
769
- .describe("queue: the position to start from (next_after)."),
947
+ .describe("queue, tests, workflows, run_results and revisions: the next_cursor of the page before."),
770
948
  limit: z
771
949
  .number()
772
950
  .int()
773
951
  .min(1)
774
952
  .max(100)
775
953
  .optional()
776
- .describe("queue: items per page, 1 to 100 (default 50)."),
954
+ .describe("queue: items per page, 1 to 100 (default 50). run_results: inputs per page, at most 5, 20 or 100 by detail (compact, standard, full), and by default that many. workflows: workflows per page, 1 to 100 (the server's default 100). tests: tests per page, at most 5, 20 or 50 by detail (compact, standard, full) and by default that many, so the page comes back whole. revisions: revisions per page, 1 to 100 (default 20)."),
955
+ id: z.string().optional().describe("tests_get: the test's id."),
777
956
  sample: z
778
957
  .number()
779
958
  .int()
@@ -789,41 +968,86 @@ export function registerLbbTools(server, client, options = {}) {
789
968
  ...graphScope,
790
969
  },
791
970
  annotations: READ_ONLY,
792
- }, (args) => run(client, `lbb_workflows.${args.action}`, args.detail, () => {
971
+ }, (args) => {
793
972
  const workflows = scoped(client, args.graph).workflows;
794
- switch (args.action) {
795
- case "starters":
796
- return workflows.starters.list();
797
- case "list":
798
- return workflows.triggered.list();
799
- case "get":
800
- return workflows.triggered.get(workflowRef(args));
801
- case "preview":
802
- return workflows.triggered.preview({
803
- ...workflowRequest(args),
804
- ...(args.sample ? { sample: args.sample } : {}),
805
- ...(args.iris ? { iris: args.iris } : {}),
806
- });
807
- case "workflows":
808
- return args.name
809
- ? workflows.named.get(args.name)
810
- : workflows.named.list();
811
- case "queue":
812
- return workflows.named.queue(workflowName(args), {
813
- ...(args.trigger ? { trigger: args.trigger } : {}),
814
- ...(args.after !== undefined ? { after: args.after } : {}),
815
- ...(args.limit !== undefined ? { limit: args.limit } : {}),
816
- });
973
+ if (args.action === "queue") {
974
+ const name = workflowName(args);
975
+ return runPage(client, "lbb_workflows.queue", args.detail, pageLimit(args.detail, args.limit, WORKFLOW_QUEUE_PAGE_MAX), (limit) => workflows.named.queue(name, {
976
+ ...(args.trigger ? { trigger: args.trigger } : {}),
977
+ ...(args.cursor !== undefined ? { cursor: args.cursor } : {}),
978
+ limit,
979
+ }));
817
980
  }
818
- }));
819
- server.registerTool("lbb_workflows_manage", {
820
- description: "Create a workflow from a starter workflow, or refresh, pause or resume a triggered workflow. Run lbb_workflows action=preview first and show the user the watch and the instance count. use creates a workflow from a starter workflow of action=starters, under a name (default: the class's local name), with its watch (class, fields) and params; the workflow then reads every instance of the class and follows each published commit. A developer workflow (starter=workflow) also takes workflow (workflow_type and version of the developer's worker), owns and batch; it needs a server with the workflow engine. use again with the same starter and name changes only what you name, and the same request again changes nothing. A workflow that calls models (ontology.fit, documents.link, documents.map, values.extract) spends model budget as it reads. refresh asks the workflow to run now and returns at once with queued; poll lbb_workflows action=get for progress. pause stops a workflow (a developer workflow, documents.parse, documents.link, documents.map or values.extract) from reading and sending, and stops the claims of a developer workflow's shards; resume lets it catch up. Without starter, pause and resume name a workflow with triggers. search.embed and ontology.fit cannot be paused. define creates or changes a workflow with triggers of the developer's worker (/v1/workflows/named): name, workflow, class and fields (its watch trigger), owns, batch, and shards (engine identities, 1 to 16), route_by (a field that keeps work in order) and window (messages queued per shard, default 4); it needs a server with the workflow engine, and a change of shards waits for empty inboxes. run (name) runs a workflow with triggers now: its watch sends every instance again.",
981
+ if (args.action === "run_results") {
982
+ const name = workflowName(args);
983
+ return runPage(client, "lbb_workflows.run_results", args.detail, pageLimit(args.detail, args.limit, WORKFLOW_RUN_RESULTS_PAGE_MAX), (limit) => workflows.named.runResults(name, {
984
+ ...(args.run ? { run: args.run } : {}),
985
+ ...(args.outcome?.length ? { outcome: args.outcome } : {}),
986
+ ...(args.iri ? { iri: args.iri } : {}),
987
+ ...(args.cursor !== undefined ? { cursor: args.cursor } : {}),
988
+ limit,
989
+ }));
990
+ }
991
+ if (args.action === "tests") {
992
+ return runPage(client, "lbb_workflows.tests", args.detail, pageLimit(args.detail, args.limit, WORKFLOW_TESTS_PAGE_MAX), (limit) => workflows.tests.list({
993
+ name: args.name,
994
+ limit,
995
+ cursor: args.cursor,
996
+ }));
997
+ }
998
+ const action = args.action;
999
+ return run(client, `lbb_workflows.${action}`, args.detail, () => {
1000
+ switch (action) {
1001
+ case "starters":
1002
+ return workflows.starters.list();
1003
+ case "list":
1004
+ return workflows.triggered.list();
1005
+ case "get":
1006
+ return workflows.triggered.get(workflowRef(args));
1007
+ case "preview":
1008
+ return workflows.triggered.preview({
1009
+ ...workflowRequest(args),
1010
+ ...(args.sample ? { sample: args.sample } : {}),
1011
+ ...(args.iris ? { iris: args.iris } : {}),
1012
+ });
1013
+ case "workflows":
1014
+ return args.name
1015
+ ? workflows.named.get(args.name)
1016
+ : workflows.named.list({
1017
+ ...(args.cursor !== undefined ? { cursor: args.cursor } : {}),
1018
+ ...(args.limit !== undefined ? { limit: args.limit } : {}),
1019
+ });
1020
+ case "revisions":
1021
+ return workflows.named.revisions(workflowName(args), {
1022
+ ...(args.cursor !== undefined ? { cursor: args.cursor } : {}),
1023
+ ...(args.limit !== undefined ? { limit: args.limit } : {}),
1024
+ });
1025
+ case "revision":
1026
+ return workflows.named.getRevision(workflowName(args), revisionId(args.revision));
1027
+ case "tests_get":
1028
+ return workflows.tests.get(testId(args));
1029
+ }
1030
+ });
1031
+ });
1032
+ host.register("lbb_workflows_manage", {
1033
+ description: "Create a workflow from a starter workflow, or refresh, pause or resume a triggered workflow. Run lbb_workflows action=preview first and show the user the watch and the instance count. use creates a workflow from a starter workflow of action=starters, under a name (default: the class's local name), with its watch (class, fields) and params; the workflow then reads every instance of the class and follows each published commit. A developer workflow (starter=workflow) also takes workflow (workflow_type and version of the developer's worker), owns and batch; it needs a server with the workflow engine. use again with the same starter and name changes only what you name, and the same request again changes nothing. A workflow that calls models (ontology.fit, documents.link, documents.map, values.extract) spends model budget as it reads. refresh asks the workflow to run now and returns at once with queued; poll lbb_workflows action=get for progress. pause stops a workflow (a developer workflow, documents.parse, documents.link, documents.map or values.extract) from reading and sending, and stops the claims of a developer workflow's shards; resume lets it catch up. Without starter, pause and resume name a workflow with triggers. search.embed and ontology.fit cannot be paused. define creates or changes a workflow with triggers of the developer's worker (/v1/workflows/named): name, workflow, one trigger (class and fields for a watch, media_types for a content trigger on uploads, schedule for one run per slot of a timer, or stream for the events a client sends, in batches), owns, batch, and shards (engine identities, 1 to 16), route_by (a field that keeps work in order) and window (messages queued per shard, default 4), and optionally budget_usd_per_month (the workflow's own monthly model budget), on_exhausted (block or skip) and fan_in (the id of a fan-in identity the shards send to); it needs a server with the workflow engine, and a change of shards waits for empty inboxes. run (name) runs a workflow with triggers now: its watch sends every instance again, a content trigger reads its log again, a schedule sends one run outside its slots, a stream sends its partial batches. activate (name, revision) makes an earlier revision of a workflow's settings (from lbb_workflows action=revisions) the one in use: it goes through the same checks as a change, and answers the workflow with its active_revision. Show the user what changes first (compare the settings of the two revisions), and pass expected_revision, the revision you read as bound, so a change by someone else since fails with workflow_revision_conflict. A revision whose settings the graph now reads differently fails with workflow_revision_stale; then change the workflow instead. Activating changes what the workflow does from then on and deletes no data, so it is not a destructive write; a change of the watch, the parameters or the gate makes the workflow read its class again, and work that waits or runs keeps the revision it started with. tests_create tests a values.extract workflow on chosen inputs before it writes: name (a workflow of the graph; starter defaults to values.extract) or install (a workflow not created yet, as use takes it: starter, name, watch, params, gate), and iris (at most 20) or sample (1 to 20, default 6), and models to try other models before choosing one. A test reads each input as the workflow would and calls its models (the calls count in the monthly workflow budget); it writes nothing to the graph. Poll lbb_workflows action=tests_get. tests_answer records the person's answer to one value of a read input (id, iri, property, answer: accept, reject, correct with value, or cant_tell; null removes it); it writes nothing to the graph. Answer only what the user decided. tests_cancel stops a queued or running test (id). Save a test's answers as a dataset with lbb_evals action=datasets_create.",
821
1034
  inputSchema: {
822
- action: z.enum(["use", "refresh", "pause", "resume", "define", "run"]),
1035
+ action: z.enum([
1036
+ "use",
1037
+ "refresh",
1038
+ "pause",
1039
+ "resume",
1040
+ "define",
1041
+ "run",
1042
+ "activate",
1043
+ "tests_create",
1044
+ "tests_answer",
1045
+ "tests_cancel",
1046
+ ]),
823
1047
  name: z
824
1048
  .string()
825
1049
  .optional()
826
- .describe("The triggered workflow's name, [a-z0-9][a-z0-9-]{0,62} (refresh, pause, resume: required; use: omit for the class's local name)."),
1050
+ .describe("The triggered workflow's name, [a-z0-9][a-z0-9-]{0,62} (refresh, pause, resume: required; use: omit for the class's local name; tests_create: the workflow of the graph to test)."),
827
1051
  ...workflowSetup,
828
1052
  gate: z
829
1053
  .number()
@@ -835,8 +1059,54 @@ export function registerLbbTools(server, client, options = {}) {
835
1059
  .number()
836
1060
  .nonnegative()
837
1061
  .optional()
838
- .describe("use: a monthly model budget in USD for this workflow, below the stack's own."),
1062
+ .describe("use and define: a monthly model budget in USD for this workflow, below the stack's own. For define, its turns' activity calls stop at it (429 workflow_budget_exceeded) and the turn waits until the next month."),
839
1063
  ...namedSetup,
1064
+ revision: z
1065
+ .string()
1066
+ .optional()
1067
+ .describe("activate: the revision to use (wrev_ and 32 hex characters)."),
1068
+ expected_revision: z
1069
+ .string()
1070
+ .optional()
1071
+ .describe("activate: the revision the workflow had bound when you read it; another one fails with workflow_revision_conflict."),
1072
+ install: testInstallSchema
1073
+ .optional()
1074
+ .describe("tests_create: a workflow not created yet, as use takes it: starter (values.extract), name, watch ({class, fields, exclude}), params and gate ({below})."),
1075
+ iris: z
1076
+ .array(z.string())
1077
+ .max(20)
1078
+ .optional()
1079
+ .describe("tests_create: the instances to test, at most 20."),
1080
+ sample: z
1081
+ .number()
1082
+ .int()
1083
+ .min(1)
1084
+ .max(20)
1085
+ .optional()
1086
+ .describe("tests_create: the first instances of the watched class instead, 1 to 20 (default 6)."),
1087
+ models: z
1088
+ .object({
1089
+ extract: testModel.optional(),
1090
+ decide: testModel.optional(),
1091
+ })
1092
+ .optional()
1093
+ .describe("tests_create: other models to run the test on, per use (extract, decide), each one of the use's options in lbb_model_choice action=choices. The test's models name the ones it ran."),
1094
+ id: z
1095
+ .string()
1096
+ .optional()
1097
+ .describe("tests_answer, tests_cancel: the test's id."),
1098
+ iri: z
1099
+ .string()
1100
+ .optional()
1101
+ .describe("tests_answer: the input, by its instance IRI."),
1102
+ property: z
1103
+ .string()
1104
+ .optional()
1105
+ .describe("tests_answer: the property of the value."),
1106
+ answer: reviewAnswerSchema
1107
+ .nullable()
1108
+ .optional()
1109
+ .describe("tests_answer: { decision: accept | reject | correct | cant_tell, value (with correct), note }; null removes the answer. For a property without a value, accept says the text states none, and correct gives the value it states."),
840
1110
  detail: detailSchema,
841
1111
  ...graphScope,
842
1112
  },
@@ -860,35 +1130,64 @@ export function registerLbbTools(server, client, options = {}) {
860
1130
  return workflows.named.put(namedRequest(args));
861
1131
  case "run":
862
1132
  return workflows.named.run(workflowName(args));
1133
+ case "activate":
1134
+ return workflows.named.activateRevision(workflowName(args), revisionId(args.revision), args.expected_revision
1135
+ ? { expectedRevision: args.expected_revision }
1136
+ : {});
1137
+ case "tests_create":
1138
+ return workflows.tests.create(testRequest(args));
1139
+ case "tests_answer":
1140
+ if (!args.iri || !args.property || args.answer === undefined) {
1141
+ throw new Error("tests_answer requires id, iri, property and answer (null removes it)");
1142
+ }
1143
+ return workflows.tests.answer({
1144
+ id: testId(args),
1145
+ iri: args.iri,
1146
+ property: args.property,
1147
+ answer: args.answer,
1148
+ });
1149
+ case "tests_cancel":
1150
+ return workflows.tests.cancel(testId(args));
863
1151
  }
864
1152
  }));
865
- server.registerTool("lbb_workflows_delete", {
866
- description: "Delete a triggered workflow: it stops following its class, and what it wrote stays. Deleting search.embed deletes the embedding and its vectors, so its class stops being searchable; the suggestions an ontology.fit filed stay; a developer workflow leaves its workflow instance and the facts its code wrote. Without starter it deletes a workflow with triggers by its name, and its shards stay. Ask the user first; confirm must repeat the name.",
1153
+ host.register("lbb_workflows_delete", {
1154
+ description: `Delete a triggered workflow: it stops following its class, and what it wrote stays. Deleting search.embed deletes the embedding and its vectors, so its class stops being searchable; the suggestions an ontology.fit filed stay; a developer workflow leaves its workflow instance and the facts its code wrote. Without starter it deletes a workflow with triggers by its name, and its shards stay. ${APPROVAL_TEXT}`,
867
1155
  inputSchema: {
868
1156
  starter: z
869
1157
  .string()
870
1158
  .optional()
871
1159
  .describe("The starter workflow's catalog id, e.g. search.embed. Omit for a workflow with triggers."),
872
1160
  name: z.string().describe("The triggered workflow's name."),
873
- confirm: z.string().describe("The same name again, to confirm."),
874
1161
  detail: detailSchema,
875
1162
  ...graphScope,
876
1163
  },
877
1164
  annotations: DESTRUCTIVE,
878
- }, ({ starter, name, confirm, detail, graph }) => run(client, "lbb_workflows_delete", detail, () => {
879
- if (confirm !== name) {
880
- throw new Error("confirm must repeat the workflow name");
881
- }
882
- const workflows = scoped(client, graph).workflows;
883
- return starter
884
- ? workflows.triggered.delete({ starter, name })
885
- : workflows.named.delete(name);
886
- }));
1165
+ }, async ({ starter, name, detail, graph }, extra) => {
1166
+ const where = graphLabel(client, graph);
1167
+ const refused = await host.approve(extra, {
1168
+ message: !starter
1169
+ ? `Delete the workflow with triggers "${oneLine(name)}" of ${where}? Its shards stay.`
1170
+ : starter.startsWith("search.embed")
1171
+ ? `Delete the search.embed workflow "${oneLine(name)}" of ${where}, with its embedding and vectors? Its class stops being searchable.`
1172
+ : `Delete the ${oneLine(starter)} workflow "${oneLine(name)}" of ${where}? It stops following its class, and what it wrote stays.`,
1173
+ });
1174
+ if (refused)
1175
+ return refused;
1176
+ return run(client, "lbb_workflows_delete", detail, () => {
1177
+ const workflows = scoped(client, graph).workflows;
1178
+ return starter
1179
+ ? workflows.triggered.delete({ starter, name })
1180
+ : workflows.named.delete(name);
1181
+ });
1182
+ });
887
1183
  const modelUse = z
888
1184
  .enum(["ask", "route", "rerank", "fit", "label"])
889
1185
  .describe("A use of a model: ask = question answers, route = query routing, rerank = search rerank, fit = ontology fit, label = eval labels.");
890
- server.registerTool("lbb_model_choice", {
891
- description: "Model choice, read only: which model each use of a model on the graph runs, and trials that test other models against the graph's ground truth (the checked calls: a person's review, else the judge's verdict). options shows per use the model it runs now (current), LBB's model (default), a switch (switched), its checked calls, and the models a trial can test with their efforts, prices and whether the server holds their key; available=false with a reason when trials cannot run. trials lists the trials, newest first, each with its report: calls compared, both models' right share, mean score (0 to 1), cost per call and median time, the difference with its 95% interval (delta, ci_low, ci_high), outcome (too_few, better, same, worse) and qualifies (meets the bar: at least 20 calls, at most 5 points worse at the low end, no more failures, cheaper or better). trial reads one trial with its compared calls; trial_call reads one call with both answers and the ground truth. switches lists the switched uses. Start, stop, switch or revert with lbb_model_choice_manage.",
1186
+ const workflowUse = z
1187
+ .enum(["extract", "decide", "embed"])
1188
+ .describe("A use of a workflow's AI steps: extract = the reasoning model (values.extract reads values, the reason activity), decide = the decision model (the support of a value, the decide activity), embed = the embedding model (listed, takes no choice).");
1189
+ host.register("lbb_model_choice", {
1190
+ description: "Model choice, read only: which model each use of a model on the graph runs, and trials that test other models against the graph's ground truth (the checked calls: a person's review, else the judge's verdict). options shows per use the model it runs now (current), LBB's model (default), a switch (switched), its checked calls, and the models a trial can test with their efforts, prices and whether the server holds their key; available=false with a reason when trials cannot run. trials lists the trials, newest first, each with its report: calls compared, both models' right share, mean score (0 to 1), cost per call and median time, the difference with its 95% interval (delta, ci_low, ci_high), outcome (too_few, better, same, worse) and qualifies (meets the bar: at least 20 calls, at most 5 points worse at the low end, no more failures, cheaper or better). trial reads one trial with its compared calls; trial_call reads one call with both answers and the ground truth. switches lists the switched uses. choices lists the model choices of the workflow uses (extract, decide, embed): without workflow the environment's row of each use and each workflow's own choice; with workflow the row in force for each of its uses (its own, else the environment's, else the default). A row has scope (environment or workflow), model, source (person or default), since, by, the model before, the use's default and options (the models it can run, with efforts, list prices and whether the server holds their key). One page in data: has_more and next_cursor page it with cursor and limit. Start, stop, switch or revert, and set or remove a choice, with lbb_model_choice_manage.",
892
1191
  inputSchema: {
893
1192
  action: z.enum([
894
1193
  "options",
@@ -896,6 +1195,7 @@ export function registerLbbTools(server, client, options = {}) {
896
1195
  "trial",
897
1196
  "trial_call",
898
1197
  "switches",
1198
+ "choices",
899
1199
  ]),
900
1200
  trial_id: z
901
1201
  .string()
@@ -914,7 +1214,16 @@ export function registerLbbTools(server, client, options = {}) {
914
1214
  .positive()
915
1215
  .max(50)
916
1216
  .optional()
917
- .describe("trials: at most this many, newest first (default 20)."),
1217
+ .describe("trials: at most this many, newest first (default 20); choices: rows per page (default 50)."),
1218
+ workflow: z
1219
+ .string()
1220
+ .optional()
1221
+ .describe("choices: a named workflow: the rows in force for it."),
1222
+ use: workflowUse.optional().describe("choices: one use."),
1223
+ cursor: z
1224
+ .string()
1225
+ .optional()
1226
+ .describe("choices: the next_cursor of the page before."),
918
1227
  detail: detailSchema,
919
1228
  ...graphScope,
920
1229
  },
@@ -937,24 +1246,38 @@ export function registerLbbTools(server, client, options = {}) {
937
1246
  return models.trials.call(args.trial_id, args.call_id);
938
1247
  case "switches":
939
1248
  return models.switches.list();
1249
+ case "choices":
1250
+ return models.choices.list({
1251
+ workflow: args.workflow,
1252
+ use: args.use,
1253
+ limit: args.limit,
1254
+ cursor: args.cursor,
1255
+ });
940
1256
  }
941
1257
  }));
942
- server.registerTool("lbb_model_choice_manage", {
943
- description: "Model choice, writes. start tests a candidate model on the uses in jobs (without jobs, every use it can do that has checked calls): the server answers those checked calls again with the candidate and scores it and the model in use against the same ground truth. It changes no model and spends at most the trial budget ($3.00 by default) of model and judge cost; an open trial of the same use and candidate comes back as it is. Pick candidates from lbb_model_choice action=options (provider anthropic with an effort, or typesafe jev-latest). A trial runs by itself: poll lbb_model_choice action=trial until its status is no longer running. stop ends a trial; it keeps what it compared. switch makes the trial's use run its candidate on this graph from the next call on; only a trial that qualifies and compared with the model in use switches (ask to a Claude model, route to a Jev model). Show the user the trial's report (score, difference and interval, cost per call, time) and switch only what the user confirmed. revert puts the use back on LBB's model.",
1258
+ host.register("lbb_model_choice_manage", {
1259
+ description: "Model choice, writes. start tests a candidate model on the uses in jobs (without jobs, every use it can do that has checked calls): the server answers those checked calls again with the candidate and scores it and the model in use against the same ground truth. It changes no model and spends at most the trial budget ($3.00 by default) of model and judge cost; an open trial of the same use and candidate comes back as it is. Pick candidates from lbb_model_choice action=options (provider anthropic with an effort, or typesafe jev-latest). A trial runs by itself: poll lbb_model_choice action=trial until its status is no longer running. stop ends a trial; it keeps what it compared. switch makes the trial's use run its candidate on this graph from the next call on; only a trial that qualifies and compared with the model in use switches (ask to a Claude model, route to a Jev model). Show the user the trial's report (score, difference and interval, cost per call, time) and switch only what the user confirmed. revert puts the use back on LBB's model. put_choice chooses the model of a workflow use (use extract or decide, provider, model, effort) for the environment, or for one workflow with workflow: the next calls run it. Pick the model from the row's options in lbb_model_choice action=choices, show the user the price per million tokens of the model in use and of the new one, and set only what the user confirmed. delete_choice removes a choice: the use runs the environment's choice, else its default; it answers {id, object, deleted: true}. Both change only the model of future calls and remove no data, so neither asks for approval.",
944
1260
  inputSchema: {
945
- action: z.enum(["start", "stop", "switch", "revert"]),
1261
+ action: z.enum([
1262
+ "start",
1263
+ "stop",
1264
+ "switch",
1265
+ "revert",
1266
+ "put_choice",
1267
+ "delete_choice",
1268
+ ]),
946
1269
  provider: z
947
- .enum(["anthropic", "typesafe"])
1270
+ .enum(["anthropic", "typesafe", "mock"])
948
1271
  .optional()
949
- .describe("start: the candidate's provider."),
1272
+ .describe("start: the candidate's provider; put_choice: the model's provider (mock only on a local server)."),
950
1273
  model: z
951
1274
  .string()
952
1275
  .optional()
953
- .describe("start: the candidate model id, e.g. claude-haiku-5-5 or jev-latest."),
1276
+ .describe("start / put_choice: the model id, e.g. claude-haiku-5-5 or jev-latest."),
954
1277
  effort: z
955
1278
  .enum(["low", "medium", "high", "xhigh", "max"])
956
1279
  .optional()
957
- .describe("start: the effort of a Claude candidate."),
1280
+ .describe("start / put_choice: the effort of a Claude model."),
958
1281
  jobs: z
959
1282
  .array(modelUse)
960
1283
  .optional()
@@ -980,6 +1303,13 @@ export function registerLbbTools(server, client, options = {}) {
980
1303
  job: modelUse
981
1304
  .optional()
982
1305
  .describe("revert: the use to put back on LBB's model."),
1306
+ use: workflowUse
1307
+ .optional()
1308
+ .describe("put_choice / delete_choice: the workflow use."),
1309
+ workflow: z
1310
+ .string()
1311
+ .optional()
1312
+ .describe("put_choice / delete_choice: a named workflow of the graph; absent for the environment."),
983
1313
  detail: detailSchema,
984
1314
  ...graphScope,
985
1315
  },
@@ -1013,10 +1343,35 @@ export function registerLbbTools(server, client, options = {}) {
1013
1343
  if (!args.job)
1014
1344
  throw new Error("revert requires job");
1015
1345
  return models.switches.revert(args.job);
1346
+ case "put_choice":
1347
+ if (!args.use || !args.provider || !args.model) {
1348
+ throw new Error("put_choice requires use, provider and model");
1349
+ }
1350
+ return models.choices.put({
1351
+ use: args.use,
1352
+ model: {
1353
+ provider: args.provider,
1354
+ model: args.model,
1355
+ ...(args.effort ? { effort: args.effort } : {}),
1356
+ },
1357
+ ...(args.workflow ? { workflow: args.workflow } : {}),
1358
+ });
1359
+ case "delete_choice":
1360
+ if (!args.use)
1361
+ throw new Error("delete_choice requires use");
1362
+ return models.choices.delete({
1363
+ use: args.use,
1364
+ workflow: args.workflow,
1365
+ });
1016
1366
  }
1017
1367
  }));
1018
- server.registerTool("lbb_evals", {
1019
- description: "Managed evals: ground truth for the questions an app asks and the searches it runs. A query run with `request` (lbb_query, also mode=question) records a trace: the question, the query that answered it, the type of that query (`query_type`: sparql, hybrid = search by meaning with conditions, search = search by meaning alone) and one item per result (hit or row); trace reads it back with the item ids. label has two forms. On a question's trace, valid alone judges the whole answer: valid=true makes the trace's query the golden query of the question; valid=false with sparql gives the right query, which becomes the golden query; valid=false alone marks the answer wrong; route with valid=true (or valid=false with sparql) names the route the question expects, and valid=true alone takes the trace's route. item + valid (or items) judges one result, a citation of the answer: the ground truth of the hits of a search by meaning, and a result marked wrong must not come back for any type. golden freezes a stored query or a search (every result it returns now is relevant). run checks every golden at the current commit: a question is asked again through the query rewriter, then the type of the query it wrote is compared with the expected type, its rows with the rows of the golden query at the same commit (`query_check`), and its results with the judged results, and the router's pick with the golden's route (route_check, which does not change the verdict); a search is done again; new results open a review trace. run_answers checks your own app's answers instead: ask each question golden's request with your own loop, then send answers=[{golden, sparql, route}] (route = the route of lbb_query mode=question with route mode, sparql absent for an unanswerable question, error for a failure); the server runs each query and checks it the same way, and the runs form the app's own series (runner; results with runner reads them). judge lets the platform's judge model (the hosted frontier model) label unlabeled results. summary, traces, goldens, results, and settings read state. Model checks: a judge model checks a sample of the model calls LBB makes for the graph (rerank, route, ask = the answer of a question, rewrite, fit, propose, label). checks_summary reads a month per job and model (checks, score, right, partly, wrong, reviews) and the judge's agreement with people; checks lists the month's checks, newest first, with the judge's verdict, score and reason and the ground truth (`truth`). review_check records a person's review of one check (call_id): agree=true keeps the judge's verdict; agree=false with verdict (and an optional score, reference and note) corrects it. The review becomes the call's ground truth, so show the check to the user and review only what the user confirmed.",
1368
+ const datasetId = (id) => {
1369
+ if (!id)
1370
+ throw new Error("this action requires id");
1371
+ return id;
1372
+ };
1373
+ host.register("lbb_evals", {
1374
+ description: "Managed evals: ground truth for the questions an app asks and the searches it runs. A query run with `request` (lbb_query, also mode=question) records a trace: the question, the query that answered it, the type of that query (`query_type`: sparql, hybrid = search by meaning with conditions, search = search by meaning alone) and one item per result (hit or row); trace reads it back with the item ids. label has two forms. On a question's trace, valid alone judges the whole answer: valid=true makes the trace's query the golden query of the question; valid=false with sparql gives the right query, which becomes the golden query; valid=false alone marks the answer wrong; route with valid=true (or valid=false with sparql) names the route the question expects, and valid=true alone takes the trace's route. item + valid (or items) judges one result, a citation of the answer: the ground truth of the hits of a search by meaning, and a result marked wrong must not come back for any type. golden freezes a stored query or a search (every result it returns now is relevant). run checks every golden at the current commit: a question is asked again through the query rewriter, then the type of the query it wrote is compared with the expected type, its rows with the rows of the golden query at the same commit (`query_check`), and its results with the judged results, and the router's pick with the golden's route (route_check, which does not change the verdict); a search is done again; new results open a review trace. run_answers checks your own app's answers instead: ask each question golden's request with your own loop, then send answers=[{golden, sparql, route}] (route = the route of lbb_query mode=question with route mode, sparql absent for an unanswerable question, error for a failure); the server runs each query and checks it the same way, and the runs form the app's own series (runner; results with runner reads them). judge lets the platform's judge model (the hosted frontier model) label unlabeled results. summary, traces, goldens, results, and settings read state. Model checks: a judge model checks a sample of the model calls LBB makes for the graph (rerank, route, ask = the answer of a question, rewrite, fit, propose, label). checks_summary reads a month per job and model (checks, score, right, partly, wrong, reviews) and the judge's agreement with people; checks lists the month's checks, newest first, with the judge's verdict, score and reason and the ground truth (`truth`). review_check records a person's review of one check (call_id): agree=true keeps the judge's verdict; agree=false with verdict (and an optional score, reference and note) corrects it. The review becomes the call's ground truth, so show the check to the user and review only what the user confirmed. Datasets: named sets of examples with the outputs a correct run gives, which workflow tests and template evals share. The kind values holds per item an input text (name, text, iri), the property values a correct run writes (values) and those it must leave out (leave_out). datasets lists the graph's datasets, newest first, without their items (one page with next_cursor); datasets_get reads one (id) with its items; datasets_export returns it as a template's samples, records (records.json) and expected (expected.json). datasets_create makes one with name and items (at most 500), install (the settings it is graded with: each property's value type, units, context) and about, or from a workflow test's answers with source={kind: test, id}. datasets_items adds items to a dataset (id); an item with the name of one it holds replaces it. Delete a dataset with lbb_evals_delete, which asks the person first. A dataset ends with its graph.",
1020
1375
  inputSchema: {
1021
1376
  action: z.enum([
1022
1377
  "summary",
@@ -1035,6 +1390,11 @@ export function registerLbbTools(server, client, options = {}) {
1035
1390
  "checks_summary",
1036
1391
  "checks",
1037
1392
  "review_check",
1393
+ "datasets",
1394
+ "datasets_get",
1395
+ "datasets_export",
1396
+ "datasets_create",
1397
+ "datasets_items",
1038
1398
  ]),
1039
1399
  trace_id: z
1040
1400
  .string()
@@ -1049,15 +1409,9 @@ export function registerLbbTools(server, client, options = {}) {
1049
1409
  .optional()
1050
1410
  .describe("label: true = the result answers the request (thumbs up), false = it does not. Without item or items, on a question's trace: the whole answer is right (true) or wrong (false)."),
1051
1411
  items: z
1052
- .array(z
1053
- .object({
1054
- id: z.string(),
1055
- valid: z.boolean(),
1056
- note: z.string().optional(),
1057
- })
1058
- .strict())
1412
+ .array(z.union([evalLabelItemSchema, evalDatasetItemSchema]))
1059
1413
  .optional()
1060
- .describe("label: several results at once."),
1414
+ .describe("label: several results at once, each {id, valid, note}. datasets_create, datasets_items: the examples, at most 500, each {name, text, values, leave_out, iri, case}; values and leave_out hold {property, value, accept, stated, quote, reason}."),
1061
1415
  by: z
1062
1416
  .string()
1063
1417
  .optional()
@@ -1198,114 +1552,212 @@ export function registerLbbTools(server, client, options = {}) {
1198
1552
  reference: jsonObjectSchema
1199
1553
  .optional()
1200
1554
  .describe('review_check with agree=false: the right answer. A rerank or label check: {"grades": {"<hit id>": 0..3}}; a route or fit check: {"picks": {"<question id>": "<option>"}}.'),
1555
+ id: z
1556
+ .string()
1557
+ .optional()
1558
+ .describe("datasets_get, datasets_export, datasets_items: the dataset's id (ds_…)."),
1559
+ cursor: z
1560
+ .string()
1561
+ .optional()
1562
+ .describe("datasets: the next_cursor of the page before."),
1563
+ name: z
1564
+ .string()
1565
+ .optional()
1566
+ .describe("datasets_create: the dataset's name."),
1567
+ kind: z
1568
+ .enum(["values"])
1569
+ .optional()
1570
+ .describe("datasets_create: what the items hold: values (the default)."),
1571
+ install: z
1572
+ .object({
1573
+ properties: z
1574
+ .record(z.string(), z.string())
1575
+ .optional()
1576
+ .describe("Each property's value type: text, keyword, i64, f64 or bool"),
1577
+ units: z
1578
+ .record(z.string(), z.string())
1579
+ .optional()
1580
+ .describe("The unit of each number property"),
1581
+ context: z.string().optional(),
1582
+ })
1583
+ .strict()
1584
+ .optional()
1585
+ .describe("datasets_create: the settings it is graded with."),
1586
+ about: z
1587
+ .array(z.string())
1588
+ .optional()
1589
+ .describe("datasets_create: what the dataset is, in sentences."),
1590
+ source: z
1591
+ .object({ kind: z.literal("test"), id: z.string() })
1592
+ .strict()
1593
+ .optional()
1594
+ .describe('datasets_create: {"kind": "test", "id": <test id>}: the answers of a workflow test become the items, and its settings the install.'),
1201
1595
  detail: detailSchema,
1202
1596
  ...graphScope,
1203
1597
  },
1204
1598
  annotations: MUTATING,
1205
- }, ({ action, trace_id, item, valid, items, by, note, sparql, route, answers, runner, surface, embedding, top_k, request, golden_id, limit, unlabeled, consistency, month, job, verdict, reviewed, after, call_id, agree, score, reference, detail, graph, }) => run(client, `lbb_evals.${action}`, detail, () => {
1206
- const target = scoped(client, graph);
1207
- switch (action) {
1208
- case "summary":
1209
- return target.evals.summary();
1210
- case "traces":
1211
- return target.evals.traces({ limit, unlabeled });
1212
- case "trace":
1213
- if (!trace_id)
1214
- throw new Error("trace requires trace_id");
1215
- return target.evals.trace(trace_id);
1216
- case "label":
1217
- if (!trace_id)
1218
- throw new Error("label requires trace_id");
1219
- if (item && valid === undefined)
1220
- throw new Error("label requires valid with item");
1221
- if (!item && !items?.length && valid === undefined)
1222
- throw new Error("label requires item (with valid), items, or valid alone for the answer of a question");
1223
- if (sparql !== undefined &&
1224
- (item || items?.length || valid !== false))
1225
- throw new Error("label takes sparql (the right query) only with valid=false and no item");
1226
- if (route !== undefined &&
1227
- (item ||
1228
- items?.length ||
1229
- (valid !== true && sparql === undefined)))
1230
- throw new Error("label takes route only with valid=true, or valid=false with sparql, and no item");
1231
- return target.evals.label(trace_id, {
1232
- ...(item ? { item, valid } : {}),
1233
- ...(items?.length ? { items } : {}),
1234
- // The answer of a question as a whole.
1235
- ...(!item && !items?.length ? { valid } : {}),
1236
- ...(sparql !== undefined ? { sparql } : {}),
1237
- ...(route !== undefined ? { route } : {}),
1238
- by,
1239
- note,
1240
- });
1241
- case "judge":
1242
- return target.evals.judge({ traceId: trace_id, limit });
1243
- case "goldens":
1244
- return target.evals.goldens();
1245
- case "golden":
1246
- if (!sparql)
1247
- throw new Error("golden requires sparql");
1248
- if (surface === "search" && !embedding)
1249
- throw new Error("a search golden requires embedding");
1250
- return target.evals.createGolden({
1251
- sparql,
1252
- request,
1253
- ...(surface ? { surface } : {}),
1254
- ...(embedding ? { embedding } : {}),
1255
- ...(top_k ? { top_k } : {}),
1256
- });
1257
- case "accept":
1258
- if (!golden_id)
1259
- throw new Error("accept requires golden_id");
1260
- return target.evals.acceptGolden(golden_id, { consistency });
1261
- case "delete":
1262
- if (!golden_id)
1263
- throw new Error("delete requires golden_id");
1264
- return target.evals.deleteGolden(golden_id);
1265
- case "run":
1266
- return target.evals.run({ consistency });
1267
- case "run_answers":
1268
- if (!answers?.length)
1269
- throw new Error("run_answers requires answers");
1270
- return target.evals.runAnswers({ answers, ...(runner ? { runner } : {}) }, { consistency });
1271
- case "results":
1272
- return target.evals.results({ limit, runner });
1273
- case "settings":
1274
- return target.evals.settings();
1275
- case "checks_summary":
1276
- return target.checks.summary({ month });
1277
- case "checks":
1278
- return target.checks.list({
1279
- job,
1280
- month,
1281
- verdict,
1282
- reviewed,
1283
- after,
1284
- limit,
1285
- });
1286
- case "review_check":
1287
- if (!call_id)
1288
- throw new Error("review_check requires call_id");
1289
- if (agree === undefined)
1290
- throw new Error("review_check requires agree");
1291
- if (agree &&
1292
- (verdict !== undefined ||
1293
- score !== undefined ||
1294
- reference !== undefined))
1295
- throw new Error("review_check with agree=true takes no verdict, score or reference");
1296
- if (!agree && !verdict)
1297
- throw new Error("review_check with agree=false requires verdict");
1298
- return target.checks.review(call_id, {
1299
- agree,
1300
- ...(verdict ? { verdict } : {}),
1301
- ...(score !== undefined ? { score } : {}),
1302
- ...(reference ? { reference } : {}),
1303
- ...(note ? { note } : {}),
1304
- });
1599
+ }, ({ action, trace_id, item, valid, items, by, note, sparql, route, answers, runner, surface, embedding, top_k, request, golden_id, limit, unlabeled, consistency, month, job, verdict, reviewed, after, call_id, agree, score, reference, id, cursor, name, kind, install, about, source, detail, graph, }) => {
1600
+ if (action === "datasets") {
1601
+ const datasets = scoped(client, graph).evals.datasets;
1602
+ return runPage(client, "lbb_evals.datasets", detail, pageLimit(detail, limit, EVAL_DATASETS_PAGE_MAX), (pageSize) => datasets.list({ limit: pageSize, cursor }));
1305
1603
  }
1306
- }));
1307
- server.registerTool("lbb_commit", {
1308
- description: "Write graph facts, retract them, or label ranked results. mode=facts writes triplets/embeddings/properties; mode=retract removes a wrongly-added fact (by edge or by entity) without a full reset; mode=search_feedback stores query/result relevance labels (Feedback grades: 3=ideal/good, 1=partial, 0=bad; include query, search_id when available, target, rank, score). Explicit idempotency_key wins; when omitted, MCP derives a stable content hash so content-identical retries dedupe. Facts mode defaults edge_idempotency to append; pass skip_unchanged for re-runnable backfills.",
1604
+ return run(client, `lbb_evals.${action}`, detail, () => {
1605
+ const target = scoped(client, graph);
1606
+ switch (action) {
1607
+ case "summary":
1608
+ return target.evals.summary();
1609
+ case "traces":
1610
+ return target.evals.traces({ limit, unlabeled });
1611
+ case "trace":
1612
+ if (!trace_id)
1613
+ throw new Error("trace requires trace_id");
1614
+ return target.evals.trace(trace_id);
1615
+ case "label": {
1616
+ if (!trace_id)
1617
+ throw new Error("label requires trace_id");
1618
+ const labels = items && z.array(evalLabelItemSchema).parse(items);
1619
+ if (item && valid === undefined)
1620
+ throw new Error("label requires valid with item");
1621
+ if (!item && !labels?.length && valid === undefined)
1622
+ throw new Error("label requires item (with valid), items, or valid alone for the answer of a question");
1623
+ if (sparql !== undefined &&
1624
+ (item || labels?.length || valid !== false))
1625
+ throw new Error("label takes sparql (the right query) only with valid=false and no item");
1626
+ if (route !== undefined &&
1627
+ (item ||
1628
+ labels?.length ||
1629
+ (valid !== true && sparql === undefined)))
1630
+ throw new Error("label takes route only with valid=true, or valid=false with sparql, and no item");
1631
+ return target.evals.label(trace_id, {
1632
+ ...(item ? { item, valid } : {}),
1633
+ ...(labels?.length ? { items: labels } : {}),
1634
+ // The answer of a question as a whole.
1635
+ ...(!item && !labels?.length ? { valid } : {}),
1636
+ ...(sparql !== undefined ? { sparql } : {}),
1637
+ ...(route !== undefined ? { route } : {}),
1638
+ by,
1639
+ note,
1640
+ });
1641
+ }
1642
+ case "judge":
1643
+ return target.evals.judge({ traceId: trace_id, limit });
1644
+ case "goldens":
1645
+ return target.evals.goldens();
1646
+ case "golden":
1647
+ if (!sparql)
1648
+ throw new Error("golden requires sparql");
1649
+ if (surface === "search" && !embedding)
1650
+ throw new Error("a search golden requires embedding");
1651
+ return target.evals.createGolden({
1652
+ sparql,
1653
+ request,
1654
+ ...(surface ? { surface } : {}),
1655
+ ...(embedding ? { embedding } : {}),
1656
+ ...(top_k ? { top_k } : {}),
1657
+ });
1658
+ case "accept":
1659
+ if (!golden_id)
1660
+ throw new Error("accept requires golden_id");
1661
+ return target.evals.acceptGolden(golden_id, { consistency });
1662
+ case "delete":
1663
+ if (!golden_id)
1664
+ throw new Error("delete requires golden_id");
1665
+ return target.evals.deleteGolden(golden_id);
1666
+ case "run":
1667
+ return target.evals.run({ consistency });
1668
+ case "run_answers":
1669
+ if (!answers?.length)
1670
+ throw new Error("run_answers requires answers");
1671
+ return target.evals.runAnswers({ answers, ...(runner ? { runner } : {}) }, { consistency });
1672
+ case "results":
1673
+ return target.evals.results({ limit, runner });
1674
+ case "settings":
1675
+ return target.evals.settings();
1676
+ case "checks_summary":
1677
+ return target.checks.summary({ month });
1678
+ case "checks":
1679
+ return target.checks.list({
1680
+ job,
1681
+ month,
1682
+ verdict,
1683
+ reviewed,
1684
+ after,
1685
+ limit,
1686
+ });
1687
+ case "review_check":
1688
+ if (!call_id)
1689
+ throw new Error("review_check requires call_id");
1690
+ if (agree === undefined)
1691
+ throw new Error("review_check requires agree");
1692
+ if (agree &&
1693
+ (verdict !== undefined ||
1694
+ score !== undefined ||
1695
+ reference !== undefined))
1696
+ throw new Error("review_check with agree=true takes no verdict, score or reference");
1697
+ if (!agree && !verdict)
1698
+ throw new Error("review_check with agree=false requires verdict");
1699
+ return target.checks.review(call_id, {
1700
+ agree,
1701
+ ...(verdict ? { verdict } : {}),
1702
+ ...(score !== undefined ? { score } : {}),
1703
+ ...(reference ? { reference } : {}),
1704
+ ...(note ? { note } : {}),
1705
+ });
1706
+ case "datasets_get":
1707
+ return target.evals.datasets.get(datasetId(id));
1708
+ case "datasets_export":
1709
+ return target.evals.datasets.export(datasetId(id));
1710
+ case "datasets_create": {
1711
+ if (!name)
1712
+ throw new Error("datasets_create requires name");
1713
+ const examples = items && z.array(evalDatasetItemSchema).parse(items);
1714
+ if (!source && !examples?.length)
1715
+ throw new Error("datasets_create requires items, or a source (a workflow test)");
1716
+ return target.evals.datasets.create({
1717
+ name,
1718
+ kind: kind ?? "values",
1719
+ ...(examples ? { items: examples } : {}),
1720
+ ...(install ? { install } : {}),
1721
+ ...(about ? { about } : {}),
1722
+ ...(source ? { source } : {}),
1723
+ });
1724
+ }
1725
+ case "datasets_items": {
1726
+ const examples = items && z.array(evalDatasetItemSchema).parse(items);
1727
+ if (!examples?.length)
1728
+ throw new Error("datasets_items requires items");
1729
+ return target.evals.datasets.addItems(datasetId(id), examples);
1730
+ }
1731
+ }
1732
+ });
1733
+ });
1734
+ host.register("lbb_evals_delete", {
1735
+ description: `Delete an Evals dataset by id, with its items. A later datasets_get or datasets_export of it answers 404, and the next save of its workflow test makes a new dataset. The server reads the dataset first, so the question names it and its item count. ${APPROVAL_TEXT}`,
1736
+ inputSchema: {
1737
+ id: z.string().describe("The dataset's id (ds_…)."),
1738
+ detail: detailSchema,
1739
+ ...graphScope,
1740
+ },
1741
+ annotations: DESTRUCTIVE,
1742
+ }, async ({ id, detail, graph }, extra) => {
1743
+ const datasets = scoped(client, graph).evals.datasets;
1744
+ const refused = await host.approve(extra, {
1745
+ message: async () => {
1746
+ const dataset = await datasets
1747
+ .get(id)
1748
+ .catch(async (error) => {
1749
+ throw await enrichError(client, error);
1750
+ });
1751
+ const items = `${dataset.item_count} item${dataset.item_count === 1 ? "" : "s"}`;
1752
+ return `Delete the dataset "${oneLine(dataset.name)}" (${oneLine(id)}) of ${graphLabel(client, graph)} with its ${items}? The next save of its workflow test makes a new dataset.`;
1753
+ },
1754
+ });
1755
+ if (refused)
1756
+ return refused;
1757
+ return run(client, "lbb_evals_delete", detail, () => datasets.delete(id));
1758
+ });
1759
+ host.register("lbb_commit", {
1760
+ description: `Write graph facts, retract them, or label ranked results. mode=facts writes triplets/embeddings/properties; mode=retract removes a wrongly-added fact (by edge or by entity) without a full reset; mode=search_feedback stores query/result relevance labels (Feedback grades: 3=ideal/good, 1=partial, 0=bad; include query, search_id when available, target, rank, score). Explicit idempotency_key wins; when omitted, MCP derives a stable content hash so content-identical retries dedupe. Facts mode defaults edge_idempotency to append; pass skip_unchanged for re-runnable backfills. A retract asks the person to approve the links and records it removes first. ${APPROVAL_TEXT}`,
1309
1761
  inputSchema: {
1310
1762
  idempotency_key: z.string().optional(),
1311
1763
  mode: z.enum(["facts", "retract", "search_feedback"]).optional(),
@@ -1315,7 +1767,7 @@ export function registerLbbTools(server, client, options = {}) {
1315
1767
  relation: z.string(),
1316
1768
  target: z.object({ type: z.string(), name: z.string() }),
1317
1769
  confidence: z.number().min(0).max(1).optional(),
1318
- evidence: z.unknown().optional(),
1770
+ evidence: evidenceSchema.nullish(),
1319
1771
  valid_time: z
1320
1772
  .object({
1321
1773
  start: z.string().optional(),
@@ -1328,9 +1780,7 @@ export function registerLbbTools(server, client, options = {}) {
1328
1780
  .optional(),
1329
1781
  }))
1330
1782
  .optional(),
1331
- entity_embeddings: z
1332
- .array(z.record(z.string(), z.unknown()))
1333
- .optional(),
1783
+ entity_embeddings: z.array(entityEmbeddingSchema).optional(),
1334
1784
  entity_properties: z
1335
1785
  .array(z.record(z.string(), z.unknown()))
1336
1786
  .optional()
@@ -1369,66 +1819,104 @@ export function registerLbbTools(server, client, options = {}) {
1369
1819
  .describe("mode=retract: entities whose every current edge is removed (a current-state tombstone; the record and its history are kept for as_of reads)."),
1370
1820
  ...graphScope,
1371
1821
  },
1372
- annotations: IDEMPOTENT_WRITE,
1373
- }, ({ idempotency_key, mode, triplets, entity_embeddings, entity_properties, search_feedback, dry_run, observed_at, edge_idempotency, retract_edges, retract_entities, graph, }) => run(client, "lbb_commit", "standard", () => {
1822
+ // A retract removes facts, so the tool may destroy.
1823
+ annotations: DESTRUCTIVE,
1824
+ }, async ({ idempotency_key, mode, triplets, entity_embeddings, entity_properties, search_feedback, dry_run, observed_at, edge_idempotency, retract_edges, retract_entities, graph, }, extra) => {
1374
1825
  const commitMode = mode ??
1375
1826
  (search_feedback
1376
1827
  ? "search_feedback"
1377
1828
  : retract_edges || retract_entities
1378
1829
  ? "retract"
1379
1830
  : "facts");
1380
- if (dry_run && commitMode !== "facts")
1381
- throw new Error("dry_run is supported only for lbb_commit mode=facts");
1382
- if (commitMode === "retract") {
1383
- const edges = retract_edges ?? [];
1384
- const entities = retract_entities ?? [];
1385
- if (edges.length === 0 && entities.length === 0) {
1386
- throw new Error("lbb_commit mode=retract requires retract_edges or retract_entities");
1831
+ if (commitMode === "retract" &&
1832
+ !dry_run &&
1833
+ (retract_edges?.length || retract_entities?.length)) {
1834
+ const refused = await host.approve(extra, {
1835
+ message: retractMessage(graphLabel(client, graph), retract_edges ?? [], retract_entities ?? []),
1836
+ });
1837
+ if (refused)
1838
+ return refused;
1839
+ }
1840
+ return run(client, "lbb_commit", "standard", () => {
1841
+ if (dry_run && commitMode !== "facts")
1842
+ throw new Error("dry_run is supported only for lbb_commit mode=facts");
1843
+ if (commitMode === "retract") {
1844
+ const edges = retract_edges ?? [];
1845
+ const entities = retract_entities ?? [];
1846
+ if (edges.length === 0 && entities.length === 0) {
1847
+ throw new Error("lbb_commit mode=retract requires retract_edges or retract_entities");
1848
+ }
1849
+ const key = idempotency_key ??
1850
+ contentHashKey({ graph }, { mode: "retract", edges, entities });
1851
+ return scoped(client, graph).retract({ edges, entities }, { idempotencyKey: key });
1852
+ }
1853
+ if (commitMode === "search_feedback") {
1854
+ if (!search_feedback)
1855
+ throw new Error("lbb_commit mode=search_feedback requires search_feedback");
1856
+ const key = idempotency_key ??
1857
+ contentHashKey({ graph }, { mode: "search_feedback", search_feedback });
1858
+ return scoped(client, graph).searchFeedback({
1859
+ ...search_feedback,
1860
+ // A search's snapshot goes back as the server returned it.
1861
+ snapshot: search_feedback.snapshot,
1862
+ }, { idempotencyKey: key });
1863
+ }
1864
+ if (search_feedback) {
1865
+ throw new Error("lbb_commit facts mode cannot include search_feedback");
1866
+ }
1867
+ const payload = {
1868
+ triplets: triplets ?? [],
1869
+ entity_embeddings: entity_embeddings ?? [],
1870
+ // A flat map or the verbose list per record; the server coerces
1871
+ // each value to its field's declared type.
1872
+ entity_properties: (entity_properties ??
1873
+ []),
1874
+ ...(observed_at ? { observed_at } : {}),
1875
+ edge_idempotency: edge_idempotency ?? "append",
1876
+ };
1877
+ if (!triplets?.length &&
1878
+ !entity_embeddings?.length &&
1879
+ !entity_properties?.length) {
1880
+ throw new Error("lbb_commit requires at least one triplet, entity embedding, or entity property");
1387
1881
  }
1388
- const key = idempotency_key ??
1389
- contentHashKey({ graph }, { mode: "retract", edges, entities });
1390
- return scoped(client, graph).retract({ edges, entities }, {
1882
+ const key = idempotency_key ?? contentHashKey({ graph }, payload);
1883
+ if (dry_run)
1884
+ return scoped(client, graph).commitDryRun(payload);
1885
+ return scoped(client, graph).commit(payload, {
1391
1886
  idempotencyKey: key,
1392
1887
  });
1393
- }
1394
- if (commitMode === "search_feedback") {
1395
- if (!search_feedback)
1396
- throw new Error("lbb_commit mode=search_feedback requires search_feedback");
1397
- const key = idempotency_key ??
1398
- contentHashKey({ graph }, { mode: "search_feedback", search_feedback });
1399
- return scoped(client, graph).searchFeedback(search_feedback, { idempotencyKey: key });
1400
- }
1401
- if (search_feedback) {
1402
- throw new Error("lbb_commit facts mode cannot include search_feedback");
1403
- }
1404
- const payload = {
1405
- triplets: triplets ?? [],
1406
- entity_embeddings: entity_embeddings ?? [],
1407
- entity_properties: entity_properties ?? [],
1408
- ...(observed_at ? { observed_at } : {}),
1409
- edge_idempotency: edge_idempotency ?? "append",
1410
- };
1411
- if (payload.triplets.length === 0 &&
1412
- payload.entity_embeddings.length === 0 &&
1413
- payload.entity_properties.length === 0) {
1414
- throw new Error("lbb_commit requires at least one triplet, entity embedding, or entity property");
1415
- }
1416
- const key = idempotency_key ?? contentHashKey({ graph }, payload);
1417
- if (dry_run)
1418
- return scoped(client, graph).commitDryRun(payload);
1419
- return scoped(client, graph).commit(payload, {
1420
- idempotencyKey: key,
1421
1888
  });
1422
- }));
1423
- server.registerTool("lbb_configure", {
1424
- description: "Manage native schema metadata. Actions: define_ontology (friendly spec with super_types), evolve_ontology (ordered edits including add_super_types), list_starters (the base ontologies crm, documents and work, each with its status on the graph: absent, partial or applied, what applying adds, and conflicts), apply_starter (add what the graph lacks of a starter in one ontology version; a relation the graph has is widened; refused with starter_conflict when the graph holds a term differently; dry_run previews), publish_schema (SHACL activation), suggest_ontology_change (file a change for a person to review instead of applying it; prefer it when the graph's owner reviews ontology changes, and list the result with lbb_inspect action=ontology_suggestions; facts files graph facts for review, which accepting commits after the change, and change may be empty when facts is set), get_rewrite_profile and set_rewrite_profile (the graph's notes and up to 20 worked question-to-SPARQL examples that lbb_query mode=question reads for every question; pass the version you read as expected_version). define, evolve, publish and set_rewrite_profile support dry_run previews. Definition/import here extracts native metadata; it does NOT store the complete RDF/OWL document as queryable graph facts. Use lbb_rdf import for full OWL and lbb_rdf update for additive INSERT DATA revisions; RDF deletions are unsupported. Publish_schema accepts unchanged ontology plus shapes; use define/evolve for native ontology changes. Publication enqueues durable conformance; a preview does not validate the whole graph.",
1889
+ });
1890
+ host.register("lbb_configure", {
1891
+ description: `Manage native schema metadata. Actions: define_ontology (friendly spec with super_types), evolve_ontology (ordered edits including add_super_types), list_starters (the base ontologies crm, documents and work, each with its status on the graph: absent, partial or applied, what applying adds, and conflicts), apply_starter (add what the graph lacks of a starter in one ontology version; a relation the graph has is widened; refused with starter_conflict when the graph holds a term differently; dry_run previews), publish_schema (SHACL activation), suggest_ontology_change (file a change for a person to review instead of applying it; prefer it when the graph's owner reviews ontology changes, and list the result with lbb_inspect action=ontology_suggestions; facts files graph facts for review, which accepting commits after the change, and change may be empty when facts is set), get_rewrite_profile and set_rewrite_profile (the graph's notes and up to 20 worked question-to-SPARQL examples that lbb_query mode=question reads for every question; pass the version you read as expected_version). define, evolve, publish and set_rewrite_profile support dry_run previews. Definition/import here extracts native metadata; it does NOT store the complete RDF/OWL document as queryable graph facts. Use lbb_rdf import for full OWL and lbb_rdf update for additive INSERT DATA revisions; RDF deletions are unsupported. Publish_schema accepts unchanged ontology plus shapes; use define/evolve for native ontology changes. Publication enqueues durable conformance; a preview does not validate the whole graph. An evolve_ontology that removes (remove_entity_type, remove_relation, or narrow_relation with remove_domain or remove_range) and a publish_schema with desired_mode=reject ask the person to approve first; a dry_run does not ask. ${APPROVAL_TEXT}`,
1425
1892
  inputSchema: configureWireSchema,
1426
- annotations: MUTATING,
1427
- }, (rawArgs) => {
1893
+ // evolve_ontology can remove types, relations and their source and
1894
+ // target types, and a reject-mode publish refuses writes, so the
1895
+ // tool may destroy.
1896
+ annotations: { ...MUTATING, destructiveHint: true },
1897
+ }, async (rawArgs, extra) => {
1428
1898
  const parsed = configureInputSchema.safeParse(rawArgs);
1429
1899
  if (!parsed.success)
1430
1900
  return errorResult(parsed.error);
1431
1901
  const args = parsed.data;
1902
+ if (args.action === "evolve_ontology" && !args.dry_run) {
1903
+ const message = ontologyRemovalMessage(graphLabel(client, args.graph), args.ops);
1904
+ if (message) {
1905
+ const refused = await host.approve(extra, { message });
1906
+ if (refused)
1907
+ return refused;
1908
+ }
1909
+ }
1910
+ if (args.action === "publish_schema" &&
1911
+ !args.dry_run &&
1912
+ args.desired_mode === "reject" &&
1913
+ args.shapes) {
1914
+ const refused = await host.approve(extra, {
1915
+ message: shapesRejectMessage(graphLabel(client, args.graph), args.shapes),
1916
+ });
1917
+ if (refused)
1918
+ return refused;
1919
+ }
1432
1920
  return run(client, `lbb_configure.${args.action}`, "standard", () => {
1433
1921
  if (args.action === "define_ontology") {
1434
1922
  return client.withScope({ graph: args.graph }).ontologyDefine(ontologyDefineBody({
@@ -1449,7 +1937,7 @@ export function registerLbbTools(server, client, options = {}) {
1449
1937
  title: args.title,
1450
1938
  rationale: args.rationale ?? "",
1451
1939
  change,
1452
- ...(args.facts ? { facts: args.facts } : {}),
1940
+ ...(args.facts ? { facts: suggestedFactsBody(args.facts) } : {}),
1453
1941
  origin: { kind: "agent", id: agent, label: agent },
1454
1942
  ...(args.anchor ? { anchor: args.anchor } : {}),
1455
1943
  ...(args.key ? { key: args.key } : {}),