@littlebigbrain/mcp 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -5
- package/dist/approval.d.ts +139 -0
- package/dist/approval.js +426 -0
- package/dist/files-tool.d.ts +4 -2
- package/dist/files-tool.js +9 -4
- package/dist/index.d.ts +1 -1
- package/dist/query-observer.d.ts +37 -0
- package/dist/rdf-tool.d.ts +2 -2
- package/dist/rdf-tool.js +18 -4
- package/dist/records-write.d.ts +101 -0
- package/dist/records-write.js +150 -0
- package/dist/reviews-tool.d.ts +3 -0
- package/dist/reviews-tool.js +420 -0
- package/dist/sparql-update.d.ts +15 -0
- package/dist/sparql-update.js +146 -0
- package/dist/tool-contracts.js +263 -16
- package/dist/tool-runtime.js +116 -18
- package/dist/tools.js +887 -274
- package/package.json +3 -3
package/dist/tools.js
CHANGED
|
@@ -3,22 +3,68 @@ import { metadataPage } from "./metadata-pages.js";
|
|
|
3
3
|
import { answerQuestion, } from "./question.js";
|
|
4
4
|
import { commitAt, compareQuery, describeGraph, findNames, } from "./query-tools.js";
|
|
5
5
|
import { registerRdfTool } from "./rdf-tool.js";
|
|
6
|
+
import { commitRecordsTool } from "./records-write.js";
|
|
6
7
|
import { registerFilesTool } from "./files-tool.js";
|
|
8
|
+
import { registerReviewsTools } from "./reviews-tool.js";
|
|
9
|
+
import { APPROVAL_TEXT, graphLabel, oneLine, ontologyRemovalMessage, retractMessage, shapesRejectMessage, toolHost, } from "./approval.js";
|
|
7
10
|
import { queryTiming } from "./query-observer.js";
|
|
8
|
-
import { DESTRUCTIVE,
|
|
9
|
-
import { analyze, assertCursorScope, contentHashKey, decodeQueryCursor, effectiveRowLimit, enrichError, errorResult, guide, normalizeDetail, normalizeLbbIris, ontologyDefineBody, queryCommitPin, queryEnvelope, queryToolResult, requireString, rowPageFrom, rowPageNext, run, scoped, stableJson, toolResult, } from "./tool-runtime.js";
|
|
11
|
+
import { DESTRUCTIVE, HARD_OUTPUT_CHARS, MUTATING, READ_ONLY, configureInputSchema, configureWireSchema, detailSchema, entityEmbeddingSchema, evalDatasetItemSchema, experimentArmSchema, evalLabelItemSchema, evidenceSchema, graphScope, inspectInputSchema, inspectWireSchema, jsonObjectSchema, queryInputSchema, queryWireSchema, reviewAnswerSchema, searchFeedbackSchema, suggestedFactsBody, testInstallSchema, yearMonthSchema, } from "./tool-contracts.js";
|
|
12
|
+
import { analyze, assertCursorScope, contentHashKey, decodeQueryCursor, effectiveRowLimit, enrichError, errorResult, guide, normalizeDetail, normalizeLbbIris, ontologyDefineBody, pageLimit, queryCommitPin, queryEnvelope, queryToolResult, requireString, rowPageFrom, rowPageNext, run, runPage, scoped, stableJson, toolResult, ToolError, } from "./tool-runtime.js";
|
|
13
|
+
/** The most tests `GET /v1/workflows/named/tests` returns in one page. */
|
|
14
|
+
const WORKFLOW_TESTS_PAGE_MAX = 50;
|
|
15
|
+
/** The most items `GET /v1/workflows/named/queue` returns in one page. */
|
|
16
|
+
const WORKFLOW_QUEUE_PAGE_MAX = 100;
|
|
17
|
+
/** The most inputs `GET /v1/workflows/named/runs/results` returns in one page. */
|
|
18
|
+
const WORKFLOW_RUN_RESULTS_PAGE_MAX = 200;
|
|
19
|
+
/** The most datasets `GET /v1/evals/datasets` returns in one page. */
|
|
20
|
+
const EVAL_DATASETS_PAGE_MAX = 50;
|
|
21
|
+
const EVAL_EXPERIMENTS_PAGE_MAX = 50;
|
|
10
22
|
export function registerLbbTools(server, client, options = {}) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
23
|
+
const host = toolHost(server, options);
|
|
24
|
+
registerRdfTool(host, client);
|
|
25
|
+
registerFilesTool(host, client);
|
|
26
|
+
registerReviewsTools(host, client);
|
|
27
|
+
host.register("lbb_inspect", {
|
|
28
|
+
description: "Read graph context and exact graph facts. Actions: guide, graphs, publication, ontology, ontology_conformance, schema, ontology_search, metadata, entity, ontology_suggestions. graphs works before bootstrap; ontology_suggestions lists the ontology change suggestions that wait for review (or were decided); fact_count is how many graph facts (triplets and entity property rows) accepting one commits, and identity_count how many identities it links. publication reports whether writes are queryable. ontology and schema return complete entries with page_size, section and cursor; follow next until absent. schema reads active native ontology/SHACL metadata without running validation. Query asserted RDF/OWL axioms separately with lbb_query. ontology_conformance serves the durable report referenced by the pinned published root. entity with iri, or version=records/1 and entity_id or entity_type+key, returns a complete exact-term record, separate assertions and snapshot from RDF. as_of_commit_seq pins retained history; schema-bound historical selection is currently unsupported. Oversized records fail rather than truncate. Omitting the new selectors preserves the native attributes/relationships view. Use lbb_query with SPARQL property paths for precise path selection.",
|
|
15
29
|
inputSchema: inspectWireSchema,
|
|
16
30
|
annotations: READ_ONLY,
|
|
17
|
-
}, (rawArgs) => {
|
|
31
|
+
}, async (rawArgs) => {
|
|
18
32
|
const parsed = inspectInputSchema.safeParse(rawArgs);
|
|
19
33
|
if (!parsed.success)
|
|
20
34
|
return errorResult(parsed.error);
|
|
21
35
|
const args = parsed.data;
|
|
36
|
+
if (args.action === "entity" &&
|
|
37
|
+
(args.version ||
|
|
38
|
+
args.iri !== undefined ||
|
|
39
|
+
args.key !== undefined ||
|
|
40
|
+
args.schema_revision !== undefined ||
|
|
41
|
+
args.limits !== undefined)) {
|
|
42
|
+
try {
|
|
43
|
+
if (args.as_of !== undefined)
|
|
44
|
+
throw new ToolError("record_selector_invalid", "entity valid-time as_of is not supported; use as_of_commit_seq");
|
|
45
|
+
if (args.name !== undefined)
|
|
46
|
+
throw new ToolError("record_selector_invalid", "records/1 uses iri, native entity_id, or entity_type and key; name is not a stable selector");
|
|
47
|
+
const data = await scoped(client, args.graph).entities.retrieve({
|
|
48
|
+
iri: args.iri,
|
|
49
|
+
id: args.entity_id,
|
|
50
|
+
type: args.entity_type,
|
|
51
|
+
key: args.key,
|
|
52
|
+
schemaRevision: args.schema_revision,
|
|
53
|
+
consistency: args.consistency,
|
|
54
|
+
asOfCommitSeq: args.as_of_commit_seq,
|
|
55
|
+
limits: args.limits,
|
|
56
|
+
});
|
|
57
|
+
const result = { summary: "lbb_inspect.entity", data };
|
|
58
|
+
if (Buffer.byteLength(JSON.stringify(result, null, 2), "utf8") >
|
|
59
|
+
HARD_OUTPUT_CHARS) {
|
|
60
|
+
throw new ToolError("record_read_limit", "The complete record exceeds the MCP output budget. Use the HTTP entity retrieve API or a bounded SPARQL query; no record terms were returned.");
|
|
61
|
+
}
|
|
62
|
+
return toolResult(result);
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
return errorResult(await enrichError(client, error));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
22
68
|
if (args.action === "ontology" || args.action === "schema") {
|
|
23
69
|
return metadataPage(client, args)
|
|
24
70
|
.then(toolResult)
|
|
@@ -61,12 +107,13 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
61
107
|
name: requireString(args.name, "name"),
|
|
62
108
|
}),
|
|
63
109
|
asOfCommitSeq: args.as_of_commit_seq,
|
|
110
|
+
consistency: args.consistency,
|
|
64
111
|
});
|
|
65
112
|
}
|
|
66
113
|
});
|
|
67
114
|
});
|
|
68
|
-
|
|
69
|
-
description: "Analytical and expert reads. Modes: question (a question in plain words), structured (SPARQL-subset JSON body), sparql (SPARQL text), search (instances by meaning over every searchable class; filter narrows by class and relationship; every hit checked against the graph), analyze, and four tools for writing your own queries: names, describe, commit_at, compare. names: before a query that names an entity (a person, a company), find its IRI with text=<the question or the names>; use the first candidate, not a CONTAINS match on the name. describe: before a query on classes or properties you have not seen, read them with question=<the question> (or classes/properties as IRIs): how many sampled instances hold each property (a filter on a rare one returns few rows), the values of small classes such as stages, examples and schema statements. commit_at: for a question about a date, find the commit of date=YYYY-MM-DD (or moment=RFC 3339), then run mode=sparql as_of_commit_seq=<it>. compare: for what changed between two points, run one SELECT at before and after (default the latest) with key=[the entity variable]; added, removed and changed come with totals and pages (next). Use mode=question when you have a question in plain words and no SPARQL query: the server picks the kind of question (route), runs queries in a bounded loop, reads their rows, and answers in plain words (about 8 s). The result holds the answer with citations and a chart hint (kind, x, y: columns of the rows), the steps, the route, and the query and rows the answer stands on. Continue or correct that query with mode=sparql. Each question uses model tokens and counts toward a daily limit of the stack. SPARQL is the query language; search finds what the words describe. SPARQL text can also search by meaning inside the query with ?x <https://littlebigbrain.com/search#similarTo> \"words\", so the other patterns filter and join the hits in one query; the result's search field reports the plan. To plan a search: lbb_embeddings action=list, then SPARQL for a class's relationships on a sample, then mode=search with explain=true to check the resolved filter before the real search (lbb_inspect action=guide has the queries).
|
|
115
|
+
host.register("lbb_query", {
|
|
116
|
+
description: "Analytical and expert reads. Modes: question (a question in plain words), structured (SPARQL-subset JSON body), sparql (SPARQL text), search (instances by meaning over every searchable class; filter narrows by class and relationship; every hit checked against the graph), analyze, and four tools for writing your own queries: names, describe, commit_at, compare. names: before a query that names an entity (a person, a company), find its IRI with text=<the question or the names>; use the first candidate, not a CONTAINS match on the name. describe: before a query on classes or properties you have not seen, read them with question=<the question> (or classes/properties as IRIs): how many sampled instances hold each property (a filter on a rare one returns few rows), the values of small classes such as stages, examples and schema statements. commit_at: for a question about a date, find the commit of date=YYYY-MM-DD (or moment=RFC 3339), then run mode=sparql as_of_commit_seq=<it>. compare: for what changed between two points, run one SELECT at before and after (default the latest) with key=[the entity variable]; added, removed and changed come with totals and pages (next). Use mode=question when you have a question in plain words and no SPARQL query: the server picks the kind of question (route), runs queries in a bounded loop, reads their rows, and answers in plain words (about 8 s). The result holds the answer with citations and a chart hint (kind, x, y: columns of the rows), the steps, the route, and the query and rows the answer stands on. Continue or correct that query with mode=sparql. Each question uses model tokens and counts toward a daily limit of the stack. SPARQL is the query language; search finds what the words describe. SPARQL text can also search by meaning inside the query with ?x <https://littlebigbrain.com/search#similarTo> \"words\", so the other patterns filter and join the hits in one query; the result's search field reports the plan. To plan a search: lbb_embeddings action=list, then SPARQL for a class's relationships on a sample, then mode=search with explain=true to check the resolved filter before the real search (lbb_inspect action=guide has the queries). Use the exact IRIs returned by names, describe or RDF. Native records use content-addressed entity IRIs, <https://littlebigbrain.com/r/NAME> relations and <https://littlebigbrain.com/class/NAME> types (lowercased); imported RDF may use any vocabulary. Never build an entity IRI from a display name. Structured and text queries pin one published watermark for the request.",
|
|
70
117
|
inputSchema: queryWireSchema,
|
|
71
118
|
annotations: READ_ONLY,
|
|
72
119
|
}, (rawArgs, extra) => {
|
|
@@ -128,9 +175,14 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
128
175
|
"consistency",
|
|
129
176
|
"min_indexed_seq",
|
|
130
177
|
]) {
|
|
178
|
+
const supplied = key === "entailment"
|
|
179
|
+
? args.mode === "sparql"
|
|
180
|
+
? args.entailment
|
|
181
|
+
: undefined
|
|
182
|
+
: args[key];
|
|
131
183
|
if (cursor &&
|
|
132
|
-
|
|
133
|
-
|
|
184
|
+
supplied !== undefined &&
|
|
185
|
+
supplied !== cursor[key]) {
|
|
134
186
|
throw new Error(`cursor ${key} does not match the supplied ${key}`);
|
|
135
187
|
}
|
|
136
188
|
}
|
|
@@ -191,7 +243,9 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
191
243
|
request.combinators.length > 0) {
|
|
192
244
|
throw new Error("`combinators` (UNION/OPTIONAL/MINUS/EXISTS) is no longer accepted by structured mode; the analytics route was removed. Express the same query as SPARQL text with mode=sparql.");
|
|
193
245
|
}
|
|
194
|
-
const response = await timing.measureAsync("query_http_total", () =>
|
|
246
|
+
const response = await timing.measureAsync("query_http_total", () =>
|
|
247
|
+
// The body is the caller's JSON; the server checks it.
|
|
248
|
+
target.sparql(request, {
|
|
195
249
|
consistency,
|
|
196
250
|
minIndexedSeq,
|
|
197
251
|
}));
|
|
@@ -357,7 +411,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
357
411
|
});
|
|
358
412
|
});
|
|
359
413
|
});
|
|
360
|
-
|
|
414
|
+
host.register("lbb_models", {
|
|
361
415
|
description: "Read model-training inputs or compare retrieval configurations over one pinned published snapshot, or read what the managed models did. shadow_eval takes the API ShadowEvalRequest body; dataset actions return bounded training examples at an optional signal split. activity reads one month of the stack's model use (all graphs): calls, items, estimated tokens and cost per feature (index, search, fit, judge, training, rerank, rewrite, trial, workflows) and model, by day and by graph, the workflows' calls per workflow and job (by_workflow), the months with activity, and the model each feature uses now.",
|
|
362
416
|
inputSchema: {
|
|
363
417
|
action: z.enum([
|
|
@@ -382,6 +436,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
382
436
|
case "shadow_eval":
|
|
383
437
|
if (!body)
|
|
384
438
|
throw new Error("shadow_eval requires body");
|
|
439
|
+
// The body is the caller's JSON; the server checks it.
|
|
385
440
|
return target.shadowEval(body);
|
|
386
441
|
case "suggest_dataset":
|
|
387
442
|
return target.suggestDataset({ limit, splitSeq: split_seq });
|
|
@@ -424,7 +479,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
424
479
|
...(args.dim ? { dim: args.dim } : {}),
|
|
425
480
|
};
|
|
426
481
|
};
|
|
427
|
-
|
|
482
|
+
host.register("lbb_embeddings", {
|
|
428
483
|
description: "Search setup, read only: embeddings declared on classes (an embedding on a class covers its subclasses; one model per graph). list shows each embedding's status (serving and building version, backfill progress, lag) and the graph's model; get shows one; preview shows every candidate fact of a class with its coverage and examples, and the exact text of sample instances, and stores nothing. Declare, refresh, or change the model with lbb_embeddings_manage; search with lbb_query mode=search. list is the first step of planning a search (lbb_inspect action=guide has the recipe).",
|
|
429
484
|
inputSchema: {
|
|
430
485
|
action: z.enum(["list", "get", "preview"]),
|
|
@@ -456,7 +511,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
456
511
|
});
|
|
457
512
|
}
|
|
458
513
|
}));
|
|
459
|
-
|
|
514
|
+
host.register("lbb_embeddings_manage", {
|
|
460
515
|
description: "Declare or refresh an embedding, or change the graph's model. Run lbb_embeddings action=preview first and show the user the text it will embed. declare sets the class, the fields (one-hop property paths such as label, description, calls/label; omit for an automatic choice); a new embedding takes the graph's model, and another model is refused. On an existing embedding it changes only what you name, and a changed recipe builds a new version while the old one serves. model moves every embedding of the graph to `model`: each builds a new version, and the graph switches when all are ready (a search never mixes two models). refresh runs one step of the embed job now (it also runs by itself after each published commit).",
|
|
461
516
|
inputSchema: {
|
|
462
517
|
action: z.enum(["declare", "refresh", "model"]),
|
|
@@ -487,21 +542,22 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
487
542
|
});
|
|
488
543
|
}
|
|
489
544
|
}));
|
|
490
|
-
|
|
491
|
-
description:
|
|
545
|
+
host.register("lbb_embeddings_delete", {
|
|
546
|
+
description: `Delete an embedding: its name and its vectors (index-gc removes the stored runs). Its class stops being searchable until it is declared and built again. ${APPROVAL_TEXT}`,
|
|
492
547
|
inputSchema: {
|
|
493
548
|
name: z.string().describe("The embedding name."),
|
|
494
|
-
confirm: z.string().describe("The same name again, to confirm."),
|
|
495
549
|
detail: detailSchema,
|
|
496
550
|
...graphScope,
|
|
497
551
|
},
|
|
498
552
|
annotations: DESTRUCTIVE,
|
|
499
|
-
}, ({ name,
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
|
|
553
|
+
}, async ({ name, detail, graph }, extra) => {
|
|
554
|
+
const refused = await host.approve(extra, {
|
|
555
|
+
message: `Delete the embedding "${oneLine(name)}" of ${graphLabel(client, graph)} and its vectors? Its class stops being searchable until it is declared and built again.`,
|
|
556
|
+
});
|
|
557
|
+
if (refused)
|
|
558
|
+
return refused;
|
|
559
|
+
return run(client, "lbb_embeddings_delete", detail, () => scoped(client, graph).embeddings.delete(name));
|
|
560
|
+
});
|
|
505
561
|
// Fit from text: fit sources declared on classes, like embeddings.
|
|
506
562
|
const fitSetup = {
|
|
507
563
|
class: z
|
|
@@ -532,7 +588,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
532
588
|
...(args.context ? { context: args.context } : {}),
|
|
533
589
|
};
|
|
534
590
|
};
|
|
535
|
-
|
|
591
|
+
host.register("lbb_fit_sources", {
|
|
536
592
|
description: "Fit from text, read only: fit sources declared on classes. A fit source names the fields of a class that hold text (transcripts, documents); the server reads every instance, proposes ontology changes (classes, properties, relations) with verbatim quotes, checks them, and files them as ontology suggestions with origin id fit:<name>. list shows each source's status (progress, lag, instances read, proposals kept and dropped, suggestions per status, this month's spend, last error) and the stack's monthly budget; get shows one; preview shows the fields, every candidate field of the class and the text of sample instances, calls no model and stores nothing. Declare, refresh or dry-run with lbb_fit_sources_manage; list the suggestions with lbb_inspect action=ontology_suggestions.",
|
|
537
593
|
inputSchema: {
|
|
538
594
|
action: z.enum(["list", "get", "preview"]),
|
|
@@ -564,7 +620,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
564
620
|
});
|
|
565
621
|
}
|
|
566
622
|
}));
|
|
567
|
-
|
|
623
|
+
host.register("lbb_fit_sources_manage", {
|
|
568
624
|
description: "Declare, dry-run or refresh a fit source (fit from text). Run lbb_fit_sources action=preview first and show the user the text the fit reads. dry_run runs the models on the first part of up to 3 instances (at most 60 s; truncated when one did not finish) and returns every proposal with its quote, its support score and what the checks made of it; it files nothing but spends model budget. declare sets the class, the fields and the context; the job then reads every instance and files suggestions (an existing source changes only what you name). refresh asks the job to run now and returns the status at once; the job also runs by itself after each published commit, so poll lbb_fit_sources action=get for progress. Nothing changes the ontology until a person accepts a suggestion.",
|
|
569
625
|
inputSchema: {
|
|
570
626
|
action: z.enum(["declare", "dry_run", "refresh"]),
|
|
@@ -590,21 +646,22 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
590
646
|
return fit.refresh(args.name);
|
|
591
647
|
}
|
|
592
648
|
}));
|
|
593
|
-
|
|
594
|
-
description:
|
|
649
|
+
host.register("lbb_fit_sources_delete", {
|
|
650
|
+
description: `Delete a fit source: the fit stops reading its class. The suggestions it filed stay. ${APPROVAL_TEXT}`,
|
|
595
651
|
inputSchema: {
|
|
596
652
|
name: z.string().describe("The fit source name."),
|
|
597
|
-
confirm: z.string().describe("The same name again, to confirm."),
|
|
598
653
|
detail: detailSchema,
|
|
599
654
|
...graphScope,
|
|
600
655
|
},
|
|
601
656
|
annotations: DESTRUCTIVE,
|
|
602
|
-
}, ({ name,
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
|
|
657
|
+
}, async ({ name, detail, graph }, extra) => {
|
|
658
|
+
const refused = await host.approve(extra, {
|
|
659
|
+
message: `Delete the fit source "${oneLine(name)}" of ${graphLabel(client, graph)}? The fit stops reading its class. The suggestions it filed stay.`,
|
|
660
|
+
});
|
|
661
|
+
if (refused)
|
|
662
|
+
return refused;
|
|
663
|
+
return run(client, "lbb_fit_sources_delete", detail, () => scoped(client, graph).ontology.fitSources.delete(name));
|
|
664
|
+
});
|
|
608
665
|
// Starter workflows and triggered workflows: one API over everything that
|
|
609
666
|
// runs when a class of the graph changes. search.embed and ontology.fit
|
|
610
667
|
// are the embeddings and fit sources above, through the same stored
|
|
@@ -652,9 +709,9 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
652
709
|
.number()
|
|
653
710
|
.int()
|
|
654
711
|
.min(1)
|
|
655
|
-
.max(
|
|
712
|
+
.max(500)
|
|
656
713
|
.optional()
|
|
657
|
-
.describe("preview/use, developer workflow only: instances per message, 1 to 64 (default 16)."),
|
|
714
|
+
.describe("preview/use and define, developer workflow only: instances per message, 1 to 64 (default 16); for a stream, events per message, 1 to 500 (default 100). The server checks each kind's bound."),
|
|
658
715
|
};
|
|
659
716
|
const workflowRequest = (args) => {
|
|
660
717
|
if (!args.starter)
|
|
@@ -662,6 +719,9 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
662
719
|
if (!args.class && (args.fields || args.exclude)) {
|
|
663
720
|
throw new Error("fields and exclude need class");
|
|
664
721
|
}
|
|
722
|
+
if (args.batch !== undefined && args.batch > 64) {
|
|
723
|
+
throw new Error("batch is 1 to 64 instances per message");
|
|
724
|
+
}
|
|
665
725
|
return {
|
|
666
726
|
starter: args.starter,
|
|
667
727
|
...(args.name ? { name: args.name } : {}),
|
|
@@ -695,6 +755,11 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
695
755
|
throw new Error("this action requires name");
|
|
696
756
|
return args.name;
|
|
697
757
|
};
|
|
758
|
+
const revisionId = (revision) => {
|
|
759
|
+
if (!revision)
|
|
760
|
+
throw new Error("this action requires revision");
|
|
761
|
+
return revision;
|
|
762
|
+
};
|
|
698
763
|
// A workflow with triggers (`/v1/workflows/named`): your worker's
|
|
699
764
|
// workflow type, one watch trigger, shards, routing and window.
|
|
700
765
|
const namedSetup = {
|
|
@@ -719,33 +784,129 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
719
784
|
trigger: z
|
|
720
785
|
.string()
|
|
721
786
|
.optional()
|
|
722
|
-
.describe("define: the
|
|
787
|
+
.describe("define: the trigger's id (default watch, content, schedule or stream). queue: the trigger whose queue to read."),
|
|
788
|
+
media_types: z
|
|
789
|
+
.array(z.string())
|
|
790
|
+
.min(1)
|
|
791
|
+
.max(32)
|
|
792
|
+
.optional()
|
|
793
|
+
.describe("define, in place of class: a content trigger on the uploads of these media types (application/pdf, image/*, */*), one content per message, without a wait for a publication."),
|
|
794
|
+
schedule: z
|
|
795
|
+
.object({
|
|
796
|
+
cron: z
|
|
797
|
+
.string()
|
|
798
|
+
.optional()
|
|
799
|
+
.describe("Five fields: minute, hour, day of month, month, day of week (0 3 * * 1-5), or @daily and the other macros."),
|
|
800
|
+
timezone: z
|
|
801
|
+
.string()
|
|
802
|
+
.optional()
|
|
803
|
+
.describe("With cron: an IANA time zone (Europe/Berlin). Default UTC."),
|
|
804
|
+
every: z
|
|
805
|
+
.string()
|
|
806
|
+
.optional()
|
|
807
|
+
.describe("In place of cron: a period in UTC, 5m to 366d (15m, 6h, 1d)."),
|
|
808
|
+
offset: z
|
|
809
|
+
.string()
|
|
810
|
+
.optional()
|
|
811
|
+
.describe("With every: moves each slot (every 1d, offset 3h runs at 03:00 UTC)."),
|
|
812
|
+
jitter: z
|
|
813
|
+
.string()
|
|
814
|
+
.optional()
|
|
815
|
+
.describe("Each slot runs up to this much later, the same delay each time; at most 1h."),
|
|
816
|
+
})
|
|
817
|
+
.optional()
|
|
818
|
+
.describe("define, in place of class: a schedule. One message per slot, at least 5 minutes apart, to one shard; a slot runs even when nothing changed (the message's changed is false)."),
|
|
819
|
+
stream: z
|
|
820
|
+
.object({
|
|
821
|
+
max_wait: z
|
|
822
|
+
.string()
|
|
823
|
+
.optional()
|
|
824
|
+
.describe("How long the first event of a partial batch waits, 1s to 15m (default 10s)."),
|
|
825
|
+
})
|
|
826
|
+
.optional()
|
|
827
|
+
.describe("define, in place of class: a stream of the events a client sends (POST /v1/workflows/named/events), in batches of batch events (1 to 500, default 100) or after max_wait; each event's key routes it to a shard, and shards stay as made."),
|
|
828
|
+
on_exhausted: z
|
|
829
|
+
.enum(["block", "skip"])
|
|
830
|
+
.optional()
|
|
831
|
+
.describe("define: what a turn that failed for good does to its shard: block (the default) holds the inbox until a person retries or skips it; skip keeps the failed turn in the history and goes on."),
|
|
832
|
+
fan_in: z
|
|
833
|
+
.string()
|
|
834
|
+
.optional()
|
|
835
|
+
.describe("define: the id of the workflow's fan-in identity (starter.<name>.<fan_in>) that the shards' turns send to, for example a catalogue. Default: none."),
|
|
723
836
|
};
|
|
724
837
|
const namedRequest = (args) => {
|
|
725
|
-
if (!args.name || !args.
|
|
726
|
-
throw new Error("define requires name
|
|
838
|
+
if (!args.name || !args.workflow) {
|
|
839
|
+
throw new Error("define requires name and workflow");
|
|
840
|
+
}
|
|
841
|
+
const kinds = [
|
|
842
|
+
args.class,
|
|
843
|
+
args.media_types,
|
|
844
|
+
args.schedule,
|
|
845
|
+
args.stream,
|
|
846
|
+
].filter((kind) => kind !== undefined).length;
|
|
847
|
+
if (kinds !== 1) {
|
|
848
|
+
throw new Error("define takes one trigger: class (a watch), media_types (a content trigger), schedule or stream");
|
|
727
849
|
}
|
|
850
|
+
const trigger = args.stream
|
|
851
|
+
? { stream: args.stream }
|
|
852
|
+
: args.schedule
|
|
853
|
+
? { schedule: args.schedule }
|
|
854
|
+
: args.media_types
|
|
855
|
+
? { content: { media_types: args.media_types } }
|
|
856
|
+
: {
|
|
857
|
+
watch: {
|
|
858
|
+
class: args.class,
|
|
859
|
+
...(args.fields ? { fields: args.fields } : {}),
|
|
860
|
+
},
|
|
861
|
+
};
|
|
728
862
|
return {
|
|
729
863
|
name: args.name,
|
|
730
864
|
workflow: args.workflow,
|
|
731
|
-
triggers: [
|
|
732
|
-
{
|
|
733
|
-
...(args.trigger ? { id: args.trigger } : {}),
|
|
734
|
-
watch: {
|
|
735
|
-
class: args.class,
|
|
736
|
-
...(args.fields ? { fields: args.fields } : {}),
|
|
737
|
-
},
|
|
738
|
-
},
|
|
739
|
-
],
|
|
865
|
+
triggers: [{ ...(args.trigger ? { id: args.trigger } : {}), ...trigger }],
|
|
740
866
|
...(args.owns ? { owns: args.owns } : {}),
|
|
741
867
|
...(args.batch !== undefined ? { batch: args.batch } : {}),
|
|
742
868
|
...(args.shards !== undefined ? { shards: args.shards } : {}),
|
|
743
869
|
...(args.route_by ? { route_by: args.route_by } : {}),
|
|
744
870
|
...(args.window !== undefined ? { window: args.window } : {}),
|
|
871
|
+
...(args.budget_usd_per_month !== undefined
|
|
872
|
+
? { budget_usd_per_month: args.budget_usd_per_month }
|
|
873
|
+
: {}),
|
|
874
|
+
...(args.on_exhausted ? { on_exhausted: args.on_exhausted } : {}),
|
|
875
|
+
...(args.fan_in ? { fan_in: args.fan_in } : {}),
|
|
876
|
+
};
|
|
877
|
+
};
|
|
878
|
+
// A workflow test (`client.workflows.tests`) runs a values.extract
|
|
879
|
+
// workflow on chosen inputs and writes nothing to the graph.
|
|
880
|
+
const testId = (args) => {
|
|
881
|
+
if (!args.id)
|
|
882
|
+
throw new Error("this action requires id");
|
|
883
|
+
return args.id;
|
|
884
|
+
};
|
|
885
|
+
// The one place that builds a test's request: a workflow of the graph by
|
|
886
|
+
// `name`, or a workflow to create (`install`).
|
|
887
|
+
const testModel = z.object({
|
|
888
|
+
provider: z.enum(["anthropic", "typesafe", "mock"]),
|
|
889
|
+
model: z.string(),
|
|
890
|
+
effort: z.enum(["low", "medium", "high", "xhigh", "max"]).optional(),
|
|
891
|
+
});
|
|
892
|
+
const testRequest = (args) => {
|
|
893
|
+
const inputs = {
|
|
894
|
+
...(args.iris ? { iris: args.iris } : {}),
|
|
895
|
+
...(args.sample !== undefined ? { sample: args.sample } : {}),
|
|
896
|
+
...(args.models ? { models: args.models } : {}),
|
|
745
897
|
};
|
|
898
|
+
if (args.install) {
|
|
899
|
+
if (args.name) {
|
|
900
|
+
throw new Error("tests_create takes name (a workflow of the graph) or install (a workflow to create), and not both");
|
|
901
|
+
}
|
|
902
|
+
return { install: args.install, ...inputs };
|
|
903
|
+
}
|
|
904
|
+
if (!args.name)
|
|
905
|
+
throw new Error("tests_create requires name or install");
|
|
906
|
+
return { name: args.name, ...inputs };
|
|
746
907
|
};
|
|
747
|
-
|
|
748
|
-
description: "Starter workflows and triggered workflows, read only. A starter workflow is a template; a triggered workflow is a workflow created from one on the graph. It watches a class: when a published commit adds or changes instances of the class (or changes the watched fields), the workflow reads them and does its work. search.embed keeps an embedding of the class (the same as lbb_embeddings); ontology.fit reads text fields and files ontology suggestions (the same as lbb_fit_sources); workflow (a developer workflow) sends the new and changed instances to a workflow of the developer's worker as trigger messages, and that code writes the facts the workflow owns. documents.parse reads uploaded PDF files into pages with line boxes, documents.link connects mentions of the same thing across the documents (it writes DocEntry and DocMention instances and links each mention to the entry that defines or describes it), documents.map proposes each link as a FactProposal in the graph's own ontology (fact_subject, fact_predicate, fact_object, fact_confidence, fact_evidence) for a review workflow, and values.extract fills chosen properties from text, each value with a quote; read the files and their pages with lbb_files. starters lists the starter workflows this server offers with their group, settings and whether each can run here. list shows every triggered workflow of the graph with its state (backfilling, ready, paused, failed, unavailable), lag, counters, this month's spend and last error; get shows one (starter and name) with its own status in details (for workflow: the messages sent, the instances waiting, the message in flight and whether it is paused). preview shows what creating one would do (the resolved watch, the instance count and the workflow's own preview; for workflow, the first messages it would send), calls no model and stores nothing. workflows lists the same workflows as workflows with triggers (/v1/workflows/named): each one's target workflow type (LBB's own are lbb.* types), its shards (engine identities starter.<name> and starter.<name>.<k>), its routing field, its window, its one watch trigger, its status and each shard's inbox; name for one. queue (name) shows what a workflow's watch has not sent yet: each waiting instance with its shard and its origin (a backfill, a commit range with the watched fields and the requests that changed it, a run now, a schema change), the pass in progress, and per shard its message in flight and its first unfinished turns with their origins; page with
|
|
908
|
+
host.register("lbb_workflows", {
|
|
909
|
+
description: "Starter workflows and triggered workflows, read only. A starter workflow is a template; a triggered workflow is a workflow created from one on the graph. It watches a class: when a published commit adds or changes instances of the class (or changes the watched fields), the workflow reads them and does its work. search.embed keeps an embedding of the class (the same as lbb_embeddings); ontology.fit reads text fields and files ontology suggestions (the same as lbb_fit_sources); workflow (a developer workflow) sends the new and changed instances to a workflow of the developer's worker as trigger messages, and that code writes the facts the workflow owns. documents.parse reads uploaded PDF files into pages with line boxes, documents.link connects mentions of the same thing across the documents (it writes DocEntry and DocMention instances and links each mention to the entry that defines or describes it), documents.map proposes each link as a FactProposal in the graph's own ontology (fact_subject, fact_predicate, fact_object, fact_confidence, fact_evidence) for a review workflow, and values.extract fills chosen properties from text, each value with a quote; read the files and their pages with lbb_files. starters lists the starter workflows this server offers with their group, settings and whether each can run here. list shows every triggered workflow of the graph with its state (backfilling, ready, paused, failed, unavailable), lag, counters, this month's spend and last error; get shows one (starter and name) with its own status in details (for workflow: the messages sent, the instances waiting, the message in flight and whether it is paused). preview shows what creating one would do (the resolved watch, the instance count and the workflow's own preview; for workflow, the first messages it would send), calls no model and stores nothing. workflows lists the same workflows as workflows with triggers (/v1/workflows/named): each one's target workflow type (LBB's own are lbb.* types), its shards (engine identities starter.<name> and starter.<name>.<k>), its routing field, its window, its one watch trigger, its status and each shard's inbox, one page in data (has_more and next_cursor page them with cursor and limit); name for one. queue (name) shows what a workflow's watch has not sent yet: each waiting instance with its shard and its origin (a backfill, a commit range with the watched fields and the requests that changed it, a run now, a schema change), the pass in progress, and per shard its message in flight and its first unfinished turns with their origins; the items are in data, and has_more and next_cursor page them with cursor and limit. run_results (name) shows what the workflow's runs did with each input, newest run first: the input's IRI and label, its outcome (written, review, no_value, failed, duplicate) and its values (the value, the quote that states it, its support, the conversion, and the review item it went to); run keeps one run (with run_counts, every input by outcome), outcome keeps some outcomes, iri keeps one input; data, has_more and next_cursor page them. A run records its results when it ends; they are kept 30 days. revisions (name) shows the history of a workflow's settings, newest change first: each item is a revision (settings, the request that makes it, and an id wrev_… that is the hash of the settings) with who changed it, when, how (create, change, activate, existing) and bound (the one in use); a row whose revision was deleted has available false and no settings; has_more and next_cursor page it with cursor and limit (1 to 100, default 20). last_tested on a revision of a workflow that has tests (values.extract) says what the workflow's tests read with those exact settings: an object (test, finished, counts, and experiment when the test is an arm of an experiment) names the newest test that read inputs with them, null means no test is recorded for those settings, and no key means the workflow has no tests or the server does not say; it shows what happened and gives no verdict. revision (name, revision) reads one revision; compare two by their settings to say what changed. Each run summary names the revision it ran on. Message workflows of the workflow engine are not listed here. tests lists the graph's workflow tests, newest first, without texts and results (name: one workflow's tests), one page with next_cursor; tests_get reads one test (id) with its inputs' texts and, per input, each value read with its quote, its support and what the live workflow does with it (writes it, gives it to review, or drops it). A test goes queued, running, then done, failed or cancelled. Create, refresh, pause or resume with lbb_workflows_manage, which also starts, answers and cancels tests and goes back to an earlier revision with its action activate; delete with lbb_workflows_delete.",
|
|
749
910
|
inputSchema: {
|
|
750
911
|
action: z.enum([
|
|
751
912
|
"starters",
|
|
@@ -754,26 +915,46 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
754
915
|
"preview",
|
|
755
916
|
"workflows",
|
|
756
917
|
"queue",
|
|
918
|
+
"revisions",
|
|
919
|
+
"revision",
|
|
920
|
+
"run_results",
|
|
921
|
+
"tests",
|
|
922
|
+
"tests_get",
|
|
757
923
|
]),
|
|
758
924
|
name: z
|
|
759
925
|
.string()
|
|
760
926
|
.optional()
|
|
761
|
-
.describe("get: the triggered workflow's name. preview: the name a new workflow would get. workflows: one workflow's name (omit for all). queue: required."),
|
|
927
|
+
.describe("get: the triggered workflow's name. preview: the name a new workflow would get. workflows: one workflow's name (omit for all). queue, run_results, revisions and revision: required. tests: one workflow's tests (omit for all)."),
|
|
928
|
+
revision: z
|
|
929
|
+
.string()
|
|
930
|
+
.optional()
|
|
931
|
+
.describe("revision: the revision's id (wrev_ and 32 hex characters)."),
|
|
932
|
+
run: z
|
|
933
|
+
.string()
|
|
934
|
+
.optional()
|
|
935
|
+
.describe("run_results: one run's results (a run id), in input order."),
|
|
936
|
+
outcome: z
|
|
937
|
+
.array(z.enum(["written", "review", "no_value", "failed", "duplicate"]))
|
|
938
|
+
.optional()
|
|
939
|
+
.describe("run_results: only the inputs of these outcomes."),
|
|
940
|
+
iri: z
|
|
941
|
+
.string()
|
|
942
|
+
.optional()
|
|
943
|
+
.describe("run_results: only the results of this input."),
|
|
762
944
|
...workflowSetup,
|
|
763
945
|
trigger: namedSetup.trigger,
|
|
764
|
-
|
|
765
|
-
.
|
|
766
|
-
.int()
|
|
767
|
-
.nonnegative()
|
|
946
|
+
cursor: z
|
|
947
|
+
.string()
|
|
768
948
|
.optional()
|
|
769
|
-
.describe("queue: the
|
|
949
|
+
.describe("queue, tests, workflows, run_results and revisions: the next_cursor of the page before."),
|
|
770
950
|
limit: z
|
|
771
951
|
.number()
|
|
772
952
|
.int()
|
|
773
953
|
.min(1)
|
|
774
954
|
.max(100)
|
|
775
955
|
.optional()
|
|
776
|
-
.describe("queue: items per page, 1 to 100 (default 50)."),
|
|
956
|
+
.describe("queue: items per page, 1 to 100 (default 50). run_results: inputs per page, at most 5, 20 or 100 by detail (compact, standard, full), and by default that many. workflows: workflows per page, 1 to 100 (the server's default 100). tests: tests per page, at most 5, 20 or 50 by detail (compact, standard, full) and by default that many, so the page comes back whole. revisions: revisions per page, 1 to 100 (default 20)."),
|
|
957
|
+
id: z.string().optional().describe("tests_get: the test's id."),
|
|
777
958
|
sample: z
|
|
778
959
|
.number()
|
|
779
960
|
.int()
|
|
@@ -789,41 +970,86 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
789
970
|
...graphScope,
|
|
790
971
|
},
|
|
791
972
|
annotations: READ_ONLY,
|
|
792
|
-
}, (args) =>
|
|
973
|
+
}, (args) => {
|
|
793
974
|
const workflows = scoped(client, args.graph).workflows;
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
case "preview":
|
|
802
|
-
return workflows.triggered.preview({
|
|
803
|
-
...workflowRequest(args),
|
|
804
|
-
...(args.sample ? { sample: args.sample } : {}),
|
|
805
|
-
...(args.iris ? { iris: args.iris } : {}),
|
|
806
|
-
});
|
|
807
|
-
case "workflows":
|
|
808
|
-
return args.name
|
|
809
|
-
? workflows.named.get(args.name)
|
|
810
|
-
: workflows.named.list();
|
|
811
|
-
case "queue":
|
|
812
|
-
return workflows.named.queue(workflowName(args), {
|
|
813
|
-
...(args.trigger ? { trigger: args.trigger } : {}),
|
|
814
|
-
...(args.after !== undefined ? { after: args.after } : {}),
|
|
815
|
-
...(args.limit !== undefined ? { limit: args.limit } : {}),
|
|
816
|
-
});
|
|
975
|
+
if (args.action === "queue") {
|
|
976
|
+
const name = workflowName(args);
|
|
977
|
+
return runPage(client, "lbb_workflows.queue", args.detail, pageLimit(args.detail, args.limit, WORKFLOW_QUEUE_PAGE_MAX), (limit) => workflows.named.queue(name, {
|
|
978
|
+
...(args.trigger ? { trigger: args.trigger } : {}),
|
|
979
|
+
...(args.cursor !== undefined ? { cursor: args.cursor } : {}),
|
|
980
|
+
limit,
|
|
981
|
+
}));
|
|
817
982
|
}
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
983
|
+
if (args.action === "run_results") {
|
|
984
|
+
const name = workflowName(args);
|
|
985
|
+
return runPage(client, "lbb_workflows.run_results", args.detail, pageLimit(args.detail, args.limit, WORKFLOW_RUN_RESULTS_PAGE_MAX), (limit) => workflows.named.runResults(name, {
|
|
986
|
+
...(args.run ? { run: args.run } : {}),
|
|
987
|
+
...(args.outcome?.length ? { outcome: args.outcome } : {}),
|
|
988
|
+
...(args.iri ? { iri: args.iri } : {}),
|
|
989
|
+
...(args.cursor !== undefined ? { cursor: args.cursor } : {}),
|
|
990
|
+
limit,
|
|
991
|
+
}));
|
|
992
|
+
}
|
|
993
|
+
if (args.action === "tests") {
|
|
994
|
+
return runPage(client, "lbb_workflows.tests", args.detail, pageLimit(args.detail, args.limit, WORKFLOW_TESTS_PAGE_MAX), (limit) => workflows.tests.list({
|
|
995
|
+
name: args.name,
|
|
996
|
+
limit,
|
|
997
|
+
cursor: args.cursor,
|
|
998
|
+
}));
|
|
999
|
+
}
|
|
1000
|
+
const action = args.action;
|
|
1001
|
+
return run(client, `lbb_workflows.${action}`, args.detail, () => {
|
|
1002
|
+
switch (action) {
|
|
1003
|
+
case "starters":
|
|
1004
|
+
return workflows.starters.list();
|
|
1005
|
+
case "list":
|
|
1006
|
+
return workflows.triggered.list();
|
|
1007
|
+
case "get":
|
|
1008
|
+
return workflows.triggered.get(workflowRef(args));
|
|
1009
|
+
case "preview":
|
|
1010
|
+
return workflows.triggered.preview({
|
|
1011
|
+
...workflowRequest(args),
|
|
1012
|
+
...(args.sample ? { sample: args.sample } : {}),
|
|
1013
|
+
...(args.iris ? { iris: args.iris } : {}),
|
|
1014
|
+
});
|
|
1015
|
+
case "workflows":
|
|
1016
|
+
return args.name
|
|
1017
|
+
? workflows.named.get(args.name)
|
|
1018
|
+
: workflows.named.list({
|
|
1019
|
+
...(args.cursor !== undefined ? { cursor: args.cursor } : {}),
|
|
1020
|
+
...(args.limit !== undefined ? { limit: args.limit } : {}),
|
|
1021
|
+
});
|
|
1022
|
+
case "revisions":
|
|
1023
|
+
return workflows.named.revisions(workflowName(args), {
|
|
1024
|
+
...(args.cursor !== undefined ? { cursor: args.cursor } : {}),
|
|
1025
|
+
...(args.limit !== undefined ? { limit: args.limit } : {}),
|
|
1026
|
+
});
|
|
1027
|
+
case "revision":
|
|
1028
|
+
return workflows.named.getRevision(workflowName(args), revisionId(args.revision));
|
|
1029
|
+
case "tests_get":
|
|
1030
|
+
return workflows.tests.get(testId(args));
|
|
1031
|
+
}
|
|
1032
|
+
});
|
|
1033
|
+
});
|
|
1034
|
+
host.register("lbb_workflows_manage", {
|
|
1035
|
+
description: "Create a workflow from a starter workflow, or refresh, pause or resume a triggered workflow. Run lbb_workflows action=preview first and show the user the watch and the instance count. use creates a workflow from a starter workflow of action=starters, under a name (default: the class's local name), with its watch (class, fields) and params; the workflow then reads every instance of the class and follows each published commit. A developer workflow (starter=workflow) also takes workflow (workflow_type and version of the developer's worker), owns and batch; it needs a server with the workflow engine. use again with the same starter and name changes only what you name, and the same request again changes nothing. A workflow that calls models (ontology.fit, documents.link, documents.map, values.extract) spends model budget as it reads. refresh asks the workflow to run now and returns at once with queued; poll lbb_workflows action=get for progress. pause stops a workflow (a developer workflow, documents.parse, documents.link, documents.map or values.extract) from reading and sending, and stops the claims of a developer workflow's shards; resume lets it catch up. Without starter, pause and resume name a workflow with triggers. search.embed and ontology.fit cannot be paused. define creates or changes a workflow with triggers of the developer's worker (/v1/workflows/named): name, workflow, one trigger (class and fields for a watch, media_types for a content trigger on uploads, schedule for one run per slot of a timer, or stream for the events a client sends, in batches), owns, batch, and shards (engine identities, 1 to 16), route_by (a field that keeps work in order) and window (messages queued per shard, default 4), and optionally budget_usd_per_month (the workflow's own monthly model budget), on_exhausted (block or skip) and fan_in (the id of a fan-in identity the shards send to); it needs a server with the workflow engine, and a change of shards waits for empty inboxes. run (name) runs a workflow with triggers now: its watch sends every instance again, a content trigger reads its log again, a schedule sends one run outside its slots, a stream sends its partial batches. activate (name, revision) makes an earlier revision of a workflow's settings (from lbb_workflows action=revisions) the one in use: it goes through the same checks as a change, and answers the workflow with its active_revision. Show the user what changes first (compare the settings of the two revisions), and when the revision's last_tested is null say that no test is recorded for those settings and ask before you activate (activation stays allowed), and pass expected_revision, the revision you read as bound, so a change by someone else since fails with workflow_revision_conflict. A revision whose settings the graph now reads differently fails with workflow_revision_stale; then change the workflow instead. Activating changes what the workflow does from then on and deletes no data, so it is not a destructive write; a change of the watch, the parameters or the gate makes the workflow read its class again, and work that waits or runs keeps the revision it started with. tests_create tests a values.extract workflow on chosen inputs before it writes: name (a workflow of the graph; starter defaults to values.extract) or install (a workflow not created yet, as use takes it: starter, name, watch, params, gate), and iris (at most 20) or sample (1 to 20, default 6), and models to try other models before choosing one. A test reads each input as the workflow would and calls its models (the calls count in the monthly workflow budget); it writes nothing to the graph. Poll lbb_workflows action=tests_get. tests_answer records the person's answer to one value of a read input (id, iri, property, answer: accept, reject, correct with value, or cant_tell; null removes it); it writes nothing to the graph. Answer only what the user decided. tests_cancel stops a queued or running test (id). Save a test's answers as a dataset with lbb_evals action=datasets_create.",
|
|
821
1036
|
inputSchema: {
|
|
822
|
-
action: z.enum([
|
|
1037
|
+
action: z.enum([
|
|
1038
|
+
"use",
|
|
1039
|
+
"refresh",
|
|
1040
|
+
"pause",
|
|
1041
|
+
"resume",
|
|
1042
|
+
"define",
|
|
1043
|
+
"run",
|
|
1044
|
+
"activate",
|
|
1045
|
+
"tests_create",
|
|
1046
|
+
"tests_answer",
|
|
1047
|
+
"tests_cancel",
|
|
1048
|
+
]),
|
|
823
1049
|
name: z
|
|
824
1050
|
.string()
|
|
825
1051
|
.optional()
|
|
826
|
-
.describe("The triggered workflow's name, [a-z0-9][a-z0-9-]{0,62} (refresh, pause, resume: required; use: omit for the class's local name)."),
|
|
1052
|
+
.describe("The triggered workflow's name, [a-z0-9][a-z0-9-]{0,62} (refresh, pause, resume: required; use: omit for the class's local name; tests_create: the workflow of the graph to test)."),
|
|
827
1053
|
...workflowSetup,
|
|
828
1054
|
gate: z
|
|
829
1055
|
.number()
|
|
@@ -835,8 +1061,58 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
835
1061
|
.number()
|
|
836
1062
|
.nonnegative()
|
|
837
1063
|
.optional()
|
|
838
|
-
.describe("use: a monthly model budget in USD for this workflow, below the stack's own."),
|
|
1064
|
+
.describe("use and define: a monthly model budget in USD for this workflow, below the stack's own. For define, its turns' activity calls stop at it (429 workflow_budget_exceeded) and the turn waits until the next month."),
|
|
839
1065
|
...namedSetup,
|
|
1066
|
+
revision: z
|
|
1067
|
+
.string()
|
|
1068
|
+
.optional()
|
|
1069
|
+
.describe("activate: the revision to use (wrev_ and 32 hex characters)."),
|
|
1070
|
+
expected_revision: z
|
|
1071
|
+
.string()
|
|
1072
|
+
.optional()
|
|
1073
|
+
.describe("activate: the revision the workflow had bound when you read it; another one fails with workflow_revision_conflict."),
|
|
1074
|
+
experiment: z
|
|
1075
|
+
.string()
|
|
1076
|
+
.optional()
|
|
1077
|
+
.describe("activate: an experiment (exp_…) whose B this activation applies; revision is B's revision. The workflow must still run the experiment's A, else experiment_outdated; when B's revision is active already, nothing changes."),
|
|
1078
|
+
install: testInstallSchema
|
|
1079
|
+
.optional()
|
|
1080
|
+
.describe("tests_create: a workflow not created yet, as use takes it: starter (values.extract), name, watch ({class, fields, exclude}), params and gate ({below})."),
|
|
1081
|
+
iris: z
|
|
1082
|
+
.array(z.string())
|
|
1083
|
+
.max(20)
|
|
1084
|
+
.optional()
|
|
1085
|
+
.describe("tests_create: the instances to test, at most 20."),
|
|
1086
|
+
sample: z
|
|
1087
|
+
.number()
|
|
1088
|
+
.int()
|
|
1089
|
+
.min(1)
|
|
1090
|
+
.max(20)
|
|
1091
|
+
.optional()
|
|
1092
|
+
.describe("tests_create: the first instances of the watched class instead, 1 to 20 (default 6)."),
|
|
1093
|
+
models: z
|
|
1094
|
+
.object({
|
|
1095
|
+
extract: testModel.optional(),
|
|
1096
|
+
decide: testModel.optional(),
|
|
1097
|
+
})
|
|
1098
|
+
.optional()
|
|
1099
|
+
.describe("tests_create: other models to run the test on, per use (extract, decide), each one of the use's options in lbb_model_choice action=choices. The test's models name the ones it ran."),
|
|
1100
|
+
id: z
|
|
1101
|
+
.string()
|
|
1102
|
+
.optional()
|
|
1103
|
+
.describe("tests_answer, tests_cancel: the test's id."),
|
|
1104
|
+
iri: z
|
|
1105
|
+
.string()
|
|
1106
|
+
.optional()
|
|
1107
|
+
.describe("tests_answer: the input, by its instance IRI."),
|
|
1108
|
+
property: z
|
|
1109
|
+
.string()
|
|
1110
|
+
.optional()
|
|
1111
|
+
.describe("tests_answer: the property of the value."),
|
|
1112
|
+
answer: reviewAnswerSchema
|
|
1113
|
+
.nullable()
|
|
1114
|
+
.optional()
|
|
1115
|
+
.describe("tests_answer: { decision: accept | reject | correct | cant_tell, value (with correct), note }; null removes the answer. For a property without a value, accept says the text states none, and correct gives the value it states."),
|
|
840
1116
|
detail: detailSchema,
|
|
841
1117
|
...graphScope,
|
|
842
1118
|
},
|
|
@@ -860,35 +1136,67 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
860
1136
|
return workflows.named.put(namedRequest(args));
|
|
861
1137
|
case "run":
|
|
862
1138
|
return workflows.named.run(workflowName(args));
|
|
1139
|
+
case "activate":
|
|
1140
|
+
return workflows.named.activateRevision(workflowName(args), revisionId(args.revision), {
|
|
1141
|
+
...(args.expected_revision
|
|
1142
|
+
? { expectedRevision: args.expected_revision }
|
|
1143
|
+
: {}),
|
|
1144
|
+
...(args.experiment ? { experiment: args.experiment } : {}),
|
|
1145
|
+
});
|
|
1146
|
+
case "tests_create":
|
|
1147
|
+
return workflows.tests.create(testRequest(args));
|
|
1148
|
+
case "tests_answer":
|
|
1149
|
+
if (!args.iri || !args.property || args.answer === undefined) {
|
|
1150
|
+
throw new Error("tests_answer requires id, iri, property and answer (null removes it)");
|
|
1151
|
+
}
|
|
1152
|
+
return workflows.tests.answer({
|
|
1153
|
+
id: testId(args),
|
|
1154
|
+
iri: args.iri,
|
|
1155
|
+
property: args.property,
|
|
1156
|
+
answer: args.answer,
|
|
1157
|
+
});
|
|
1158
|
+
case "tests_cancel":
|
|
1159
|
+
return workflows.tests.cancel(testId(args));
|
|
863
1160
|
}
|
|
864
1161
|
}));
|
|
865
|
-
|
|
866
|
-
description:
|
|
1162
|
+
host.register("lbb_workflows_delete", {
|
|
1163
|
+
description: `Delete a triggered workflow: it stops following its class, and what it wrote stays. Deleting search.embed deletes the embedding and its vectors, so its class stops being searchable; the suggestions an ontology.fit filed stay; a developer workflow leaves its workflow instance and the facts its code wrote. Without starter it deletes a workflow with triggers by its name, and its shards stay. ${APPROVAL_TEXT}`,
|
|
867
1164
|
inputSchema: {
|
|
868
1165
|
starter: z
|
|
869
1166
|
.string()
|
|
870
1167
|
.optional()
|
|
871
1168
|
.describe("The starter workflow's catalog id, e.g. search.embed. Omit for a workflow with triggers."),
|
|
872
1169
|
name: z.string().describe("The triggered workflow's name."),
|
|
873
|
-
confirm: z.string().describe("The same name again, to confirm."),
|
|
874
1170
|
detail: detailSchema,
|
|
875
1171
|
...graphScope,
|
|
876
1172
|
},
|
|
877
1173
|
annotations: DESTRUCTIVE,
|
|
878
|
-
}, ({ starter, name,
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
1174
|
+
}, async ({ starter, name, detail, graph }, extra) => {
|
|
1175
|
+
const where = graphLabel(client, graph);
|
|
1176
|
+
const refused = await host.approve(extra, {
|
|
1177
|
+
message: !starter
|
|
1178
|
+
? `Delete the workflow with triggers "${oneLine(name)}" of ${where}? Its shards stay.`
|
|
1179
|
+
: starter.startsWith("search.embed")
|
|
1180
|
+
? `Delete the search.embed workflow "${oneLine(name)}" of ${where}, with its embedding and vectors? Its class stops being searchable.`
|
|
1181
|
+
: `Delete the ${oneLine(starter)} workflow "${oneLine(name)}" of ${where}? It stops following its class, and what it wrote stays.`,
|
|
1182
|
+
});
|
|
1183
|
+
if (refused)
|
|
1184
|
+
return refused;
|
|
1185
|
+
return run(client, "lbb_workflows_delete", detail, () => {
|
|
1186
|
+
const workflows = scoped(client, graph).workflows;
|
|
1187
|
+
return starter
|
|
1188
|
+
? workflows.triggered.delete({ starter, name })
|
|
1189
|
+
: workflows.named.delete(name);
|
|
1190
|
+
});
|
|
1191
|
+
});
|
|
887
1192
|
const modelUse = z
|
|
888
1193
|
.enum(["ask", "route", "rerank", "fit", "label"])
|
|
889
1194
|
.describe("A use of a model: ask = question answers, route = query routing, rerank = search rerank, fit = ontology fit, label = eval labels.");
|
|
890
|
-
|
|
891
|
-
|
|
1195
|
+
const workflowUse = z
|
|
1196
|
+
.enum(["extract", "decide", "embed"])
|
|
1197
|
+
.describe("A use of a workflow's AI steps: extract = the reasoning model (values.extract reads values, the reason activity), decide = the decision model (the support of a value, the decide activity), embed = the embedding model (listed, takes no choice).");
|
|
1198
|
+
host.register("lbb_model_choice", {
|
|
1199
|
+
description: "Model choice, read only: which model each use of a model on the graph runs, and trials that test other models against the graph's ground truth (the checked calls: a person's review, else the judge's verdict). options shows per use the model it runs now (current), LBB's model (default), a switch (switched), its checked calls, and the models a trial can test with their efforts, prices and whether the server holds their key; available=false with a reason when trials cannot run. trials lists the trials, newest first, each with its report: calls compared, both models' right share, mean score (0 to 1), cost per call and median time, the difference with its 95% interval (delta, ci_low, ci_high), outcome (too_few, better, same, worse) and qualifies (meets the bar: at least 20 calls, at most 5 points worse at the low end, no more failures, cheaper or better). trial reads one trial with its compared calls; trial_call reads one call with both answers and the ground truth. switches lists the switched uses. choices lists the model choices of the workflow uses (extract, decide, embed): without workflow the environment's row of each use and each workflow's own choice; with workflow the row in force for each of its uses (its own, else the environment's, else the default). A row has scope (environment or workflow), model, source (person or default), since, by, the model before, the use's default and options (the models it can run, with efforts, list prices and whether the server holds their key). One page in data: has_more and next_cursor page it with cursor and limit. Start, stop, switch or revert, and set or remove a choice, with lbb_model_choice_manage.",
|
|
892
1200
|
inputSchema: {
|
|
893
1201
|
action: z.enum([
|
|
894
1202
|
"options",
|
|
@@ -896,6 +1204,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
896
1204
|
"trial",
|
|
897
1205
|
"trial_call",
|
|
898
1206
|
"switches",
|
|
1207
|
+
"choices",
|
|
899
1208
|
]),
|
|
900
1209
|
trial_id: z
|
|
901
1210
|
.string()
|
|
@@ -914,7 +1223,16 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
914
1223
|
.positive()
|
|
915
1224
|
.max(50)
|
|
916
1225
|
.optional()
|
|
917
|
-
.describe("trials: at most this many, newest first (default 20)."),
|
|
1226
|
+
.describe("trials: at most this many, newest first (default 20); choices: rows per page (default 50)."),
|
|
1227
|
+
workflow: z
|
|
1228
|
+
.string()
|
|
1229
|
+
.optional()
|
|
1230
|
+
.describe("choices: a named workflow: the rows in force for it."),
|
|
1231
|
+
use: workflowUse.optional().describe("choices: one use."),
|
|
1232
|
+
cursor: z
|
|
1233
|
+
.string()
|
|
1234
|
+
.optional()
|
|
1235
|
+
.describe("choices: the next_cursor of the page before."),
|
|
918
1236
|
detail: detailSchema,
|
|
919
1237
|
...graphScope,
|
|
920
1238
|
},
|
|
@@ -937,24 +1255,38 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
937
1255
|
return models.trials.call(args.trial_id, args.call_id);
|
|
938
1256
|
case "switches":
|
|
939
1257
|
return models.switches.list();
|
|
1258
|
+
case "choices":
|
|
1259
|
+
return models.choices.list({
|
|
1260
|
+
workflow: args.workflow,
|
|
1261
|
+
use: args.use,
|
|
1262
|
+
limit: args.limit,
|
|
1263
|
+
cursor: args.cursor,
|
|
1264
|
+
});
|
|
940
1265
|
}
|
|
941
1266
|
}));
|
|
942
|
-
|
|
943
|
-
description: "Model choice, writes. start tests a candidate model on the uses in jobs (without jobs, every use it can do that has checked calls): the server answers those checked calls again with the candidate and scores it and the model in use against the same ground truth. It changes no model and spends at most the trial budget ($3.00 by default) of model and judge cost; an open trial of the same use and candidate comes back as it is. Pick candidates from lbb_model_choice action=options (provider anthropic with an effort, or typesafe jev-latest). A trial runs by itself: poll lbb_model_choice action=trial until its status is no longer running. stop ends a trial; it keeps what it compared. switch makes the trial's use run its candidate on this graph from the next call on; only a trial that qualifies and compared with the model in use switches (ask to a Claude model, route to a Jev model). Show the user the trial's report (score, difference and interval, cost per call, time) and switch only what the user confirmed. revert puts the use back on LBB's model.",
|
|
1267
|
+
host.register("lbb_model_choice_manage", {
|
|
1268
|
+
description: "Model choice, writes. start tests a candidate model on the uses in jobs (without jobs, every use it can do that has checked calls): the server answers those checked calls again with the candidate and scores it and the model in use against the same ground truth. It changes no model and spends at most the trial budget ($3.00 by default) of model and judge cost; an open trial of the same use and candidate comes back as it is. Pick candidates from lbb_model_choice action=options (provider anthropic with an effort, or typesafe jev-latest). A trial runs by itself: poll lbb_model_choice action=trial until its status is no longer running. stop ends a trial; it keeps what it compared. switch makes the trial's use run its candidate on this graph from the next call on; only a trial that qualifies and compared with the model in use switches (ask to a Claude model, route to a Jev model). Show the user the trial's report (score, difference and interval, cost per call, time) and switch only what the user confirmed. revert puts the use back on LBB's model. put_choice chooses the model of a workflow use (use extract or decide, provider, model, effort) for the environment, or for one workflow with workflow: the next calls run it. Pick the model from the row's options in lbb_model_choice action=choices, show the user the price per million tokens of the model in use and of the new one, and set only what the user confirmed. delete_choice removes a choice: the use runs the environment's choice, else its default; it answers {id, object, deleted: true}. Both change only the model of future calls and remove no data, so neither asks for approval.",
|
|
944
1269
|
inputSchema: {
|
|
945
|
-
action: z.enum([
|
|
1270
|
+
action: z.enum([
|
|
1271
|
+
"start",
|
|
1272
|
+
"stop",
|
|
1273
|
+
"switch",
|
|
1274
|
+
"revert",
|
|
1275
|
+
"put_choice",
|
|
1276
|
+
"delete_choice",
|
|
1277
|
+
]),
|
|
946
1278
|
provider: z
|
|
947
|
-
.enum(["anthropic", "typesafe"])
|
|
1279
|
+
.enum(["anthropic", "typesafe", "mock"])
|
|
948
1280
|
.optional()
|
|
949
|
-
.describe("start: the candidate's provider."),
|
|
1281
|
+
.describe("start: the candidate's provider; put_choice: the model's provider (mock only on a local server)."),
|
|
950
1282
|
model: z
|
|
951
1283
|
.string()
|
|
952
1284
|
.optional()
|
|
953
|
-
.describe("start: the
|
|
1285
|
+
.describe("start / put_choice: the model id, e.g. claude-haiku-5-5 or jev-latest."),
|
|
954
1286
|
effort: z
|
|
955
1287
|
.enum(["low", "medium", "high", "xhigh", "max"])
|
|
956
1288
|
.optional()
|
|
957
|
-
.describe("start: the effort of a Claude
|
|
1289
|
+
.describe("start / put_choice: the effort of a Claude model."),
|
|
958
1290
|
jobs: z
|
|
959
1291
|
.array(modelUse)
|
|
960
1292
|
.optional()
|
|
@@ -980,6 +1312,17 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
980
1312
|
job: modelUse
|
|
981
1313
|
.optional()
|
|
982
1314
|
.describe("revert: the use to put back on LBB's model."),
|
|
1315
|
+
use: workflowUse
|
|
1316
|
+
.optional()
|
|
1317
|
+
.describe("put_choice / delete_choice: the workflow use."),
|
|
1318
|
+
workflow: z
|
|
1319
|
+
.string()
|
|
1320
|
+
.optional()
|
|
1321
|
+
.describe("put_choice / delete_choice: a named workflow of the graph; absent for the environment."),
|
|
1322
|
+
experiment: z
|
|
1323
|
+
.string()
|
|
1324
|
+
.optional()
|
|
1325
|
+
.describe("put_choice: an experiment (exp_…) whose B this choice applies, with workflow; use and the model are B's change. The workflow must still run the experiment's A, else experiment_outdated; when it runs B already, nothing changes."),
|
|
983
1326
|
detail: detailSchema,
|
|
984
1327
|
...graphScope,
|
|
985
1328
|
},
|
|
@@ -1013,10 +1356,52 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1013
1356
|
if (!args.job)
|
|
1014
1357
|
throw new Error("revert requires job");
|
|
1015
1358
|
return models.switches.revert(args.job);
|
|
1359
|
+
case "put_choice":
|
|
1360
|
+
if (!args.use || !args.provider || !args.model) {
|
|
1361
|
+
throw new Error("put_choice requires use, provider and model");
|
|
1362
|
+
}
|
|
1363
|
+
return models.choices.put({
|
|
1364
|
+
use: args.use,
|
|
1365
|
+
model: {
|
|
1366
|
+
provider: args.provider,
|
|
1367
|
+
model: args.model,
|
|
1368
|
+
...(args.effort ? { effort: args.effort } : {}),
|
|
1369
|
+
},
|
|
1370
|
+
...(args.workflow ? { workflow: args.workflow } : {}),
|
|
1371
|
+
...(args.experiment ? { experiment: args.experiment } : {}),
|
|
1372
|
+
});
|
|
1373
|
+
case "delete_choice":
|
|
1374
|
+
if (!args.use)
|
|
1375
|
+
throw new Error("delete_choice requires use");
|
|
1376
|
+
return models.choices.delete({
|
|
1377
|
+
use: args.use,
|
|
1378
|
+
workflow: args.workflow,
|
|
1379
|
+
});
|
|
1016
1380
|
}
|
|
1017
1381
|
}));
|
|
1018
|
-
|
|
1019
|
-
|
|
1382
|
+
const datasetId = (id) => {
|
|
1383
|
+
if (!id)
|
|
1384
|
+
throw new Error("this action requires id");
|
|
1385
|
+
return id;
|
|
1386
|
+
};
|
|
1387
|
+
const experimentId = (id) => {
|
|
1388
|
+
if (!id)
|
|
1389
|
+
throw new Error("this action requires id (the experiment's)");
|
|
1390
|
+
return id;
|
|
1391
|
+
};
|
|
1392
|
+
const experimentRequest = (args) => {
|
|
1393
|
+
if (!args.workflow || !args.dataset || !args.b)
|
|
1394
|
+
throw new Error("this action requires workflow, dataset and b");
|
|
1395
|
+
return {
|
|
1396
|
+
workflow: args.workflow,
|
|
1397
|
+
dataset: args.dataset,
|
|
1398
|
+
b: args.b,
|
|
1399
|
+
...(args.max_items ? { max_items: args.max_items } : {}),
|
|
1400
|
+
...(args.max_cost_usd ? { max_cost_usd: args.max_cost_usd } : {}),
|
|
1401
|
+
};
|
|
1402
|
+
};
|
|
1403
|
+
host.register("lbb_evals", {
|
|
1404
|
+
description: "Managed evals: ground truth for the questions an app asks and the searches it runs. A query run with `request` (lbb_query, also mode=question) records a trace: the question, the query that answered it, the type of that query (`query_type`: sparql, hybrid = search by meaning with conditions, search = search by meaning alone) and one item per result (hit or row); trace reads it back with the item ids. label has two forms. On a question's trace, valid alone judges the whole answer: valid=true makes the trace's query the golden query of the question; valid=false with sparql gives the right query, which becomes the golden query; valid=false alone marks the answer wrong; route with valid=true (or valid=false with sparql) names the route the question expects, and valid=true alone takes the trace's route. item + valid (or items) judges one result, a citation of the answer: the ground truth of the hits of a search by meaning, and a result marked wrong must not come back for any type. golden freezes a stored query or a search (every result it returns now is relevant). run checks every golden at the current commit: a question is asked again through the query rewriter, then the type of the query it wrote is compared with the expected type, its rows with the rows of the golden query at the same commit (`query_check`), and its results with the judged results, and the router's pick with the golden's route (route_check, which does not change the verdict); a search is done again; new results open a review trace. run_answers checks your own app's answers instead: ask each question golden's request with your own loop, then send answers=[{golden, sparql, route}] (route = the route of lbb_query mode=question with route mode, sparql absent for an unanswerable question, error for a failure); the server runs each query and checks it the same way, and the runs form the app's own series (runner; results with runner reads them). judge lets the platform's judge model (the hosted frontier model) label unlabeled results. summary, traces, goldens, results, and settings read state. Model checks: a judge model checks a sample of the model calls LBB makes for the graph (rerank, route, ask = the answer of a question, rewrite, fit, propose, label). checks_summary reads a month per job and model (checks, score, right, partly, wrong, reviews) and the judge's agreement with people; checks lists the month's checks, newest first, with the judge's verdict, score and reason and the ground truth (`truth`). review_check records a person's review of one check (call_id): agree=true keeps the judge's verdict; agree=false with verdict (and an optional score, reference and note) corrects it. The review becomes the call's ground truth, so show the check to the user and review only what the user confirmed. Datasets: named sets of examples with the outputs a correct run gives, which workflow tests and template evals share. The kind values holds per item an input text (name, text, iri), the property values a correct run writes (values) and those it must leave out (leave_out). datasets lists the graph's datasets, newest first, without their items (one page with next_cursor); datasets_get reads one (id) with its items; datasets_export returns it as a template's samples, records (records.json) and expected (expected.json). datasets_create makes one with name and items (at most 500), install (the settings it is graded with: each property's value type, units, context) and about, or from a workflow test's answers with source={kind: test, id}. datasets_items adds items to a dataset (id); an item with the name of one it holds replaces it. Delete a dataset with lbb_evals_delete, which asks the person first. A dataset ends with its graph. Experiments: two configurations of a workflow, A and B, on the items of a dataset, with the difference of B against A and its 95% interval. A is the workflow as it runs now; b changes one part: {model: {use: extract or decide, model: {provider, model, effort}}} (a model from lbb_model_choice action=choices options) or {revision: wrev_…} (a kept settings revision), not both. Each arm runs as a workflow test that writes nothing to the graph and spends model budget; each item is graded against the dataset's labels. experiments_estimate shows the cost first (items, the cost of both arms, the limit, the budget left): call it, show the user the figures, and start only what the user confirmed. experiments_create starts one (workflow, dataset, b, optionally max_items up to 200 and max_cost_usd up to 100, default 5); poll experiment (id) until its status is done, failed or cancelled, and read report.delta with ci_low and ci_high (outcome too_few under 10 items, else better, worse or same) and the item rows. experiments lists them (workflow, dataset, status, one page with next_cursor); experiments_cancel stops one, keeping what it compared. None of them changes the workflow or the graph. To use B, make the change with the experiment's id: lbb_model_choice_manage action=put_choice (or lbb_workflows_manage action=activate) with experiment, after the user confirmed; it fails with experiment_outdated when the workflow moved since.",
|
|
1020
1405
|
inputSchema: {
|
|
1021
1406
|
action: z.enum([
|
|
1022
1407
|
"summary",
|
|
@@ -1035,6 +1420,16 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1035
1420
|
"checks_summary",
|
|
1036
1421
|
"checks",
|
|
1037
1422
|
"review_check",
|
|
1423
|
+
"datasets",
|
|
1424
|
+
"datasets_get",
|
|
1425
|
+
"datasets_export",
|
|
1426
|
+
"datasets_create",
|
|
1427
|
+
"datasets_items",
|
|
1428
|
+
"experiments",
|
|
1429
|
+
"experiment",
|
|
1430
|
+
"experiments_estimate",
|
|
1431
|
+
"experiments_create",
|
|
1432
|
+
"experiments_cancel",
|
|
1038
1433
|
]),
|
|
1039
1434
|
trace_id: z
|
|
1040
1435
|
.string()
|
|
@@ -1049,15 +1444,9 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1049
1444
|
.optional()
|
|
1050
1445
|
.describe("label: true = the result answers the request (thumbs up), false = it does not. Without item or items, on a question's trace: the whole answer is right (true) or wrong (false)."),
|
|
1051
1446
|
items: z
|
|
1052
|
-
.array(z
|
|
1053
|
-
.object({
|
|
1054
|
-
id: z.string(),
|
|
1055
|
-
valid: z.boolean(),
|
|
1056
|
-
note: z.string().optional(),
|
|
1057
|
-
})
|
|
1058
|
-
.strict())
|
|
1447
|
+
.array(z.union([evalLabelItemSchema, evalDatasetItemSchema]))
|
|
1059
1448
|
.optional()
|
|
1060
|
-
.describe("label: several results at once."),
|
|
1449
|
+
.describe("label: several results at once, each {id, valid, note}. datasets_create, datasets_items: the examples, at most 500, each {name, text, values, leave_out, iri, case}; values and leave_out hold {property, value, accept, stated, quote, reason, source}; source says which person's answer gave a label (kind test or review, id, at): send it back as read, or leave it out."),
|
|
1061
1450
|
by: z
|
|
1062
1451
|
.string()
|
|
1063
1452
|
.optional()
|
|
@@ -1198,124 +1587,289 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1198
1587
|
reference: jsonObjectSchema
|
|
1199
1588
|
.optional()
|
|
1200
1589
|
.describe('review_check with agree=false: the right answer. A rerank or label check: {"grades": {"<hit id>": 0..3}}; a route or fit check: {"picks": {"<question id>": "<option>"}}.'),
|
|
1590
|
+
id: z
|
|
1591
|
+
.string()
|
|
1592
|
+
.optional()
|
|
1593
|
+
.describe("datasets_get, datasets_export, datasets_items: the dataset's id (ds_…). experiment, experiments_cancel: the experiment's id (exp_…)."),
|
|
1594
|
+
cursor: z
|
|
1595
|
+
.string()
|
|
1596
|
+
.optional()
|
|
1597
|
+
.describe("datasets, experiments: the next_cursor of the page before."),
|
|
1598
|
+
workflow: z
|
|
1599
|
+
.string()
|
|
1600
|
+
.optional()
|
|
1601
|
+
.describe("experiments_estimate, experiments_create: the workflow to compare. experiments: only the experiments of this workflow."),
|
|
1602
|
+
dataset: z
|
|
1603
|
+
.string()
|
|
1604
|
+
.optional()
|
|
1605
|
+
.describe("experiments_estimate, experiments_create: the dataset (ds_…) whose items both arms run. experiments: only the experiments on this dataset."),
|
|
1606
|
+
b: experimentArmSchema
|
|
1607
|
+
.optional()
|
|
1608
|
+
.describe("experiments_estimate, experiments_create: what B changes in A: {model: {use, model: {provider, model, effort}}} or {revision}, not both."),
|
|
1609
|
+
max_items: z
|
|
1610
|
+
.number()
|
|
1611
|
+
.int()
|
|
1612
|
+
.min(1)
|
|
1613
|
+
.max(200)
|
|
1614
|
+
.optional()
|
|
1615
|
+
.describe("experiments_estimate, experiments_create: run the dataset's first items only (default 200)."),
|
|
1616
|
+
max_cost_usd: z
|
|
1617
|
+
.number()
|
|
1618
|
+
.positive()
|
|
1619
|
+
.max(100)
|
|
1620
|
+
.optional()
|
|
1621
|
+
.describe("experiments_estimate, experiments_create: the cost limit of both arms in US dollars (default 5)."),
|
|
1622
|
+
status: z
|
|
1623
|
+
.enum(["queued", "running", "done", "failed", "cancelled"])
|
|
1624
|
+
.optional()
|
|
1625
|
+
.describe("experiments: only the experiments with this status."),
|
|
1626
|
+
name: z
|
|
1627
|
+
.string()
|
|
1628
|
+
.optional()
|
|
1629
|
+
.describe("datasets_create: the dataset's name."),
|
|
1630
|
+
kind: z
|
|
1631
|
+
.enum(["values"])
|
|
1632
|
+
.optional()
|
|
1633
|
+
.describe("datasets_create: what the items hold: values (the default)."),
|
|
1634
|
+
install: z
|
|
1635
|
+
.object({
|
|
1636
|
+
properties: z
|
|
1637
|
+
.record(z.string(), z.string())
|
|
1638
|
+
.optional()
|
|
1639
|
+
.describe("Each property's value type: text, keyword, i64, f64 or bool"),
|
|
1640
|
+
units: z
|
|
1641
|
+
.record(z.string(), z.string())
|
|
1642
|
+
.optional()
|
|
1643
|
+
.describe("The unit of each number property"),
|
|
1644
|
+
context: z.string().optional(),
|
|
1645
|
+
})
|
|
1646
|
+
.strict()
|
|
1647
|
+
.optional()
|
|
1648
|
+
.describe("datasets_create: the settings it is graded with."),
|
|
1649
|
+
about: z
|
|
1650
|
+
.array(z.string())
|
|
1651
|
+
.optional()
|
|
1652
|
+
.describe("datasets_create: what the dataset is, in sentences."),
|
|
1653
|
+
source: z
|
|
1654
|
+
.object({ kind: z.literal("test"), id: z.string() })
|
|
1655
|
+
.strict()
|
|
1656
|
+
.optional()
|
|
1657
|
+
.describe('datasets_create: {"kind": "test", "id": <test id>}: the answers of a workflow test become the items, and its settings the install.'),
|
|
1201
1658
|
detail: detailSchema,
|
|
1202
1659
|
...graphScope,
|
|
1203
1660
|
},
|
|
1204
1661
|
annotations: MUTATING,
|
|
1205
|
-
}, ({ action, trace_id, item, valid, items, by, note, sparql, route, answers, runner, surface, embedding, top_k, request, golden_id, limit, unlabeled, consistency, month, job, verdict, reviewed, after, call_id, agree, score, reference,
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
return target.evals.trace(trace_id);
|
|
1216
|
-
case "label":
|
|
1217
|
-
if (!trace_id)
|
|
1218
|
-
throw new Error("label requires trace_id");
|
|
1219
|
-
if (item && valid === undefined)
|
|
1220
|
-
throw new Error("label requires valid with item");
|
|
1221
|
-
if (!item && !items?.length && valid === undefined)
|
|
1222
|
-
throw new Error("label requires item (with valid), items, or valid alone for the answer of a question");
|
|
1223
|
-
if (sparql !== undefined &&
|
|
1224
|
-
(item || items?.length || valid !== false))
|
|
1225
|
-
throw new Error("label takes sparql (the right query) only with valid=false and no item");
|
|
1226
|
-
if (route !== undefined &&
|
|
1227
|
-
(item ||
|
|
1228
|
-
items?.length ||
|
|
1229
|
-
(valid !== true && sparql === undefined)))
|
|
1230
|
-
throw new Error("label takes route only with valid=true, or valid=false with sparql, and no item");
|
|
1231
|
-
return target.evals.label(trace_id, {
|
|
1232
|
-
...(item ? { item, valid } : {}),
|
|
1233
|
-
...(items?.length ? { items } : {}),
|
|
1234
|
-
// The answer of a question as a whole.
|
|
1235
|
-
...(!item && !items?.length ? { valid } : {}),
|
|
1236
|
-
...(sparql !== undefined ? { sparql } : {}),
|
|
1237
|
-
...(route !== undefined ? { route } : {}),
|
|
1238
|
-
by,
|
|
1239
|
-
note,
|
|
1240
|
-
});
|
|
1241
|
-
case "judge":
|
|
1242
|
-
return target.evals.judge({ traceId: trace_id, limit });
|
|
1243
|
-
case "goldens":
|
|
1244
|
-
return target.evals.goldens();
|
|
1245
|
-
case "golden":
|
|
1246
|
-
if (!sparql)
|
|
1247
|
-
throw new Error("golden requires sparql");
|
|
1248
|
-
if (surface === "search" && !embedding)
|
|
1249
|
-
throw new Error("a search golden requires embedding");
|
|
1250
|
-
return target.evals.createGolden({
|
|
1251
|
-
sparql,
|
|
1252
|
-
request,
|
|
1253
|
-
...(surface ? { surface } : {}),
|
|
1254
|
-
...(embedding ? { embedding } : {}),
|
|
1255
|
-
...(top_k ? { top_k } : {}),
|
|
1256
|
-
});
|
|
1257
|
-
case "accept":
|
|
1258
|
-
if (!golden_id)
|
|
1259
|
-
throw new Error("accept requires golden_id");
|
|
1260
|
-
return target.evals.acceptGolden(golden_id, { consistency });
|
|
1261
|
-
case "delete":
|
|
1262
|
-
if (!golden_id)
|
|
1263
|
-
throw new Error("delete requires golden_id");
|
|
1264
|
-
return target.evals.deleteGolden(golden_id);
|
|
1265
|
-
case "run":
|
|
1266
|
-
return target.evals.run({ consistency });
|
|
1267
|
-
case "run_answers":
|
|
1268
|
-
if (!answers?.length)
|
|
1269
|
-
throw new Error("run_answers requires answers");
|
|
1270
|
-
return target.evals.runAnswers({ answers, ...(runner ? { runner } : {}) }, { consistency });
|
|
1271
|
-
case "results":
|
|
1272
|
-
return target.evals.results({ limit, runner });
|
|
1273
|
-
case "settings":
|
|
1274
|
-
return target.evals.settings();
|
|
1275
|
-
case "checks_summary":
|
|
1276
|
-
return target.checks.summary({ month });
|
|
1277
|
-
case "checks":
|
|
1278
|
-
return target.checks.list({
|
|
1279
|
-
job,
|
|
1280
|
-
month,
|
|
1281
|
-
verdict,
|
|
1282
|
-
reviewed,
|
|
1283
|
-
after,
|
|
1284
|
-
limit,
|
|
1285
|
-
});
|
|
1286
|
-
case "review_check":
|
|
1287
|
-
if (!call_id)
|
|
1288
|
-
throw new Error("review_check requires call_id");
|
|
1289
|
-
if (agree === undefined)
|
|
1290
|
-
throw new Error("review_check requires agree");
|
|
1291
|
-
if (agree &&
|
|
1292
|
-
(verdict !== undefined ||
|
|
1293
|
-
score !== undefined ||
|
|
1294
|
-
reference !== undefined))
|
|
1295
|
-
throw new Error("review_check with agree=true takes no verdict, score or reference");
|
|
1296
|
-
if (!agree && !verdict)
|
|
1297
|
-
throw new Error("review_check with agree=false requires verdict");
|
|
1298
|
-
return target.checks.review(call_id, {
|
|
1299
|
-
agree,
|
|
1300
|
-
...(verdict ? { verdict } : {}),
|
|
1301
|
-
...(score !== undefined ? { score } : {}),
|
|
1302
|
-
...(reference ? { reference } : {}),
|
|
1303
|
-
...(note ? { note } : {}),
|
|
1304
|
-
});
|
|
1662
|
+
}, ({ action, trace_id, item, valid, items, by, note, sparql, route, answers, runner, surface, embedding, top_k, request, golden_id, limit, unlabeled, consistency, month, job, verdict, reviewed, after, call_id, agree, score, reference, id, cursor, workflow, dataset, b, max_items, max_cost_usd, status, name, kind, install, about, source, detail, graph, }) => {
|
|
1663
|
+
if (action === "experiments") {
|
|
1664
|
+
const experiments = scoped(client, graph).evals.experiments;
|
|
1665
|
+
return runPage(client, "lbb_evals.experiments", detail, pageLimit(detail, limit, EVAL_EXPERIMENTS_PAGE_MAX), (pageSize) => experiments.list({
|
|
1666
|
+
workflow,
|
|
1667
|
+
dataset,
|
|
1668
|
+
status,
|
|
1669
|
+
limit: pageSize,
|
|
1670
|
+
cursor,
|
|
1671
|
+
}));
|
|
1305
1672
|
}
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1673
|
+
if (action === "datasets") {
|
|
1674
|
+
const datasets = scoped(client, graph).evals.datasets;
|
|
1675
|
+
return runPage(client, "lbb_evals.datasets", detail, pageLimit(detail, limit, EVAL_DATASETS_PAGE_MAX), (pageSize) => datasets.list({ limit: pageSize, cursor }));
|
|
1676
|
+
}
|
|
1677
|
+
return run(client, `lbb_evals.${action}`, detail, () => {
|
|
1678
|
+
const target = scoped(client, graph);
|
|
1679
|
+
switch (action) {
|
|
1680
|
+
case "summary":
|
|
1681
|
+
return target.evals.summary();
|
|
1682
|
+
case "traces":
|
|
1683
|
+
return target.evals.traces({ limit, unlabeled });
|
|
1684
|
+
case "trace":
|
|
1685
|
+
if (!trace_id)
|
|
1686
|
+
throw new Error("trace requires trace_id");
|
|
1687
|
+
return target.evals.trace(trace_id);
|
|
1688
|
+
case "label": {
|
|
1689
|
+
if (!trace_id)
|
|
1690
|
+
throw new Error("label requires trace_id");
|
|
1691
|
+
const labels = items && z.array(evalLabelItemSchema).parse(items);
|
|
1692
|
+
if (item && valid === undefined)
|
|
1693
|
+
throw new Error("label requires valid with item");
|
|
1694
|
+
if (!item && !labels?.length && valid === undefined)
|
|
1695
|
+
throw new Error("label requires item (with valid), items, or valid alone for the answer of a question");
|
|
1696
|
+
if (sparql !== undefined &&
|
|
1697
|
+
(item || labels?.length || valid !== false))
|
|
1698
|
+
throw new Error("label takes sparql (the right query) only with valid=false and no item");
|
|
1699
|
+
if (route !== undefined &&
|
|
1700
|
+
(item ||
|
|
1701
|
+
labels?.length ||
|
|
1702
|
+
(valid !== true && sparql === undefined)))
|
|
1703
|
+
throw new Error("label takes route only with valid=true, or valid=false with sparql, and no item");
|
|
1704
|
+
return target.evals.label(trace_id, {
|
|
1705
|
+
...(item ? { item, valid } : {}),
|
|
1706
|
+
...(labels?.length ? { items: labels } : {}),
|
|
1707
|
+
// The answer of a question as a whole.
|
|
1708
|
+
...(!item && !labels?.length ? { valid } : {}),
|
|
1709
|
+
...(sparql !== undefined ? { sparql } : {}),
|
|
1710
|
+
...(route !== undefined ? { route } : {}),
|
|
1711
|
+
by,
|
|
1712
|
+
note,
|
|
1713
|
+
});
|
|
1714
|
+
}
|
|
1715
|
+
case "judge":
|
|
1716
|
+
return target.evals.judge({ traceId: trace_id, limit });
|
|
1717
|
+
case "goldens":
|
|
1718
|
+
return target.evals.goldens();
|
|
1719
|
+
case "golden":
|
|
1720
|
+
if (!sparql)
|
|
1721
|
+
throw new Error("golden requires sparql");
|
|
1722
|
+
if (surface === "search" && !embedding)
|
|
1723
|
+
throw new Error("a search golden requires embedding");
|
|
1724
|
+
return target.evals.createGolden({
|
|
1725
|
+
sparql,
|
|
1726
|
+
request,
|
|
1727
|
+
...(surface ? { surface } : {}),
|
|
1728
|
+
...(embedding ? { embedding } : {}),
|
|
1729
|
+
...(top_k ? { top_k } : {}),
|
|
1730
|
+
});
|
|
1731
|
+
case "accept":
|
|
1732
|
+
if (!golden_id)
|
|
1733
|
+
throw new Error("accept requires golden_id");
|
|
1734
|
+
return target.evals.acceptGolden(golden_id, { consistency });
|
|
1735
|
+
case "delete":
|
|
1736
|
+
if (!golden_id)
|
|
1737
|
+
throw new Error("delete requires golden_id");
|
|
1738
|
+
return target.evals.deleteGolden(golden_id);
|
|
1739
|
+
case "run":
|
|
1740
|
+
return target.evals.run({ consistency });
|
|
1741
|
+
case "run_answers":
|
|
1742
|
+
if (!answers?.length)
|
|
1743
|
+
throw new Error("run_answers requires answers");
|
|
1744
|
+
return target.evals.runAnswers({ answers, ...(runner ? { runner } : {}) }, { consistency });
|
|
1745
|
+
case "results":
|
|
1746
|
+
return target.evals.results({ limit, runner });
|
|
1747
|
+
case "settings":
|
|
1748
|
+
return target.evals.settings();
|
|
1749
|
+
case "checks_summary":
|
|
1750
|
+
return target.checks.summary({ month });
|
|
1751
|
+
case "checks":
|
|
1752
|
+
return target.checks.list({
|
|
1753
|
+
job,
|
|
1754
|
+
month,
|
|
1755
|
+
verdict,
|
|
1756
|
+
reviewed,
|
|
1757
|
+
after,
|
|
1758
|
+
limit,
|
|
1759
|
+
});
|
|
1760
|
+
case "review_check":
|
|
1761
|
+
if (!call_id)
|
|
1762
|
+
throw new Error("review_check requires call_id");
|
|
1763
|
+
if (agree === undefined)
|
|
1764
|
+
throw new Error("review_check requires agree");
|
|
1765
|
+
if (agree &&
|
|
1766
|
+
(verdict !== undefined ||
|
|
1767
|
+
score !== undefined ||
|
|
1768
|
+
reference !== undefined))
|
|
1769
|
+
throw new Error("review_check with agree=true takes no verdict, score or reference");
|
|
1770
|
+
if (!agree && !verdict)
|
|
1771
|
+
throw new Error("review_check with agree=false requires verdict");
|
|
1772
|
+
return target.checks.review(call_id, {
|
|
1773
|
+
agree,
|
|
1774
|
+
...(verdict ? { verdict } : {}),
|
|
1775
|
+
...(score !== undefined ? { score } : {}),
|
|
1776
|
+
...(reference ? { reference } : {}),
|
|
1777
|
+
...(note ? { note } : {}),
|
|
1778
|
+
});
|
|
1779
|
+
case "datasets_get":
|
|
1780
|
+
return target.evals.datasets.get(datasetId(id));
|
|
1781
|
+
case "datasets_export":
|
|
1782
|
+
return target.evals.datasets.export(datasetId(id));
|
|
1783
|
+
case "datasets_create": {
|
|
1784
|
+
if (!name)
|
|
1785
|
+
throw new Error("datasets_create requires name");
|
|
1786
|
+
const examples = items && z.array(evalDatasetItemSchema).parse(items);
|
|
1787
|
+
if (!source && !examples?.length)
|
|
1788
|
+
throw new Error("datasets_create requires items, or a source (a workflow test)");
|
|
1789
|
+
return target.evals.datasets.create({
|
|
1790
|
+
name,
|
|
1791
|
+
kind: kind ?? "values",
|
|
1792
|
+
...(examples ? { items: examples } : {}),
|
|
1793
|
+
...(install ? { install } : {}),
|
|
1794
|
+
...(about ? { about } : {}),
|
|
1795
|
+
...(source ? { source } : {}),
|
|
1796
|
+
});
|
|
1797
|
+
}
|
|
1798
|
+
case "datasets_items": {
|
|
1799
|
+
const examples = items && z.array(evalDatasetItemSchema).parse(items);
|
|
1800
|
+
if (!examples?.length)
|
|
1801
|
+
throw new Error("datasets_items requires items");
|
|
1802
|
+
return target.evals.datasets.addItems(datasetId(id), examples);
|
|
1803
|
+
}
|
|
1804
|
+
case "experiment":
|
|
1805
|
+
return target.evals.experiments.get(experimentId(id));
|
|
1806
|
+
case "experiments_cancel":
|
|
1807
|
+
return target.evals.experiments.cancel(experimentId(id));
|
|
1808
|
+
case "experiments_estimate":
|
|
1809
|
+
return target.evals.experiments.estimate(experimentRequest({
|
|
1810
|
+
workflow,
|
|
1811
|
+
dataset,
|
|
1812
|
+
b,
|
|
1813
|
+
max_items,
|
|
1814
|
+
max_cost_usd,
|
|
1815
|
+
}));
|
|
1816
|
+
case "experiments_create":
|
|
1817
|
+
return target.evals.experiments.create(experimentRequest({
|
|
1818
|
+
workflow,
|
|
1819
|
+
dataset,
|
|
1820
|
+
b,
|
|
1821
|
+
max_items,
|
|
1822
|
+
max_cost_usd,
|
|
1823
|
+
}));
|
|
1824
|
+
}
|
|
1825
|
+
});
|
|
1826
|
+
});
|
|
1827
|
+
host.register("lbb_evals_delete", {
|
|
1828
|
+
description: `Delete an Evals dataset by id, with its items. A later datasets_get or datasets_export of it answers 404, and the next save of its workflow test makes a new dataset. The server reads the dataset first, so the question names it and its item count. ${APPROVAL_TEXT}`,
|
|
1829
|
+
inputSchema: {
|
|
1830
|
+
id: z.string().describe("The dataset's id (ds_…)."),
|
|
1831
|
+
detail: detailSchema,
|
|
1832
|
+
...graphScope,
|
|
1833
|
+
},
|
|
1834
|
+
annotations: DESTRUCTIVE,
|
|
1835
|
+
}, async ({ id, detail, graph }, extra) => {
|
|
1836
|
+
const datasets = scoped(client, graph).evals.datasets;
|
|
1837
|
+
const refused = await host.approve(extra, {
|
|
1838
|
+
message: async () => {
|
|
1839
|
+
const dataset = await datasets
|
|
1840
|
+
.get(id)
|
|
1841
|
+
.catch(async (error) => {
|
|
1842
|
+
throw await enrichError(client, error);
|
|
1843
|
+
});
|
|
1844
|
+
const items = `${dataset.item_count} item${dataset.item_count === 1 ? "" : "s"}`;
|
|
1845
|
+
return `Delete the dataset "${oneLine(dataset.name)}" (${oneLine(id)}) of ${graphLabel(client, graph)} with its ${items}? The next save of its workflow test makes a new dataset.`;
|
|
1846
|
+
},
|
|
1847
|
+
});
|
|
1848
|
+
if (refused)
|
|
1849
|
+
return refused;
|
|
1850
|
+
return run(client, "lbb_evals_delete", detail, () => datasets.delete(id));
|
|
1851
|
+
});
|
|
1852
|
+
host.register("lbb_commit", {
|
|
1853
|
+
description: `Write graph facts, retract them, or label ranked results. Explicit mode=records forwards a strict records/1 body, with exact IRIs/terms and the original receipt. It requires an existing support-enabled graph; protocol recognition never activates or converts a graph. Record edits that may remove support (set/unset/remove/delete/replace_scope or asserted=false) ask for approval. mode=facts writes triplets/embeddings/properties; mode=retract removes a wrongly-added fact (by edge or by entity) without a full reset; mode=search_feedback stores query/result relevance labels (Feedback grades: 3=ideal/good, 1=partial, 0=bad; include query, search_id when available, target, rank, score). Explicit idempotency_key wins; when omitted, MCP derives a stable content hash so content-identical retries dedupe. Facts mode defaults edge_idempotency to append; pass skip_unchanged for re-runnable backfills. A retract asks the person to approve the links and records it removes first. ${APPROVAL_TEXT}`,
|
|
1309
1854
|
inputSchema: {
|
|
1310
1855
|
idempotency_key: z.string().optional(),
|
|
1311
|
-
mode: z
|
|
1856
|
+
mode: z
|
|
1857
|
+
.enum(["facts", "retract", "search_feedback", "records"])
|
|
1858
|
+
.optional(),
|
|
1859
|
+
records: jsonObjectSchema
|
|
1860
|
+
.optional()
|
|
1861
|
+
.describe("Explicit mode=records only: the unchanged RecordCommitRequest object with version=records/1. Unknown fields are preserved for strict server validation; no native IRI normalization or scalar coercion. Cannot mix legacy write arguments."),
|
|
1862
|
+
expected_graph_epoch: z
|
|
1863
|
+
.string()
|
|
1864
|
+
.optional()
|
|
1865
|
+
.describe("mode=records only: exact decimal graph incarnation guard. Protocol support does not activate this graph."),
|
|
1312
1866
|
triplets: z
|
|
1313
1867
|
.array(z.object({
|
|
1314
1868
|
source: z.object({ type: z.string(), name: z.string() }),
|
|
1315
1869
|
relation: z.string(),
|
|
1316
1870
|
target: z.object({ type: z.string(), name: z.string() }),
|
|
1317
1871
|
confidence: z.number().min(0).max(1).optional(),
|
|
1318
|
-
evidence:
|
|
1872
|
+
evidence: evidenceSchema.nullish(),
|
|
1319
1873
|
valid_time: z
|
|
1320
1874
|
.object({
|
|
1321
1875
|
start: z.string().optional(),
|
|
@@ -1328,9 +1882,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1328
1882
|
.optional(),
|
|
1329
1883
|
}))
|
|
1330
1884
|
.optional(),
|
|
1331
|
-
entity_embeddings: z
|
|
1332
|
-
.array(z.record(z.string(), z.unknown()))
|
|
1333
|
-
.optional(),
|
|
1885
|
+
entity_embeddings: z.array(entityEmbeddingSchema).optional(),
|
|
1334
1886
|
entity_properties: z
|
|
1335
1887
|
.array(z.record(z.string(), z.unknown()))
|
|
1336
1888
|
.optional()
|
|
@@ -1369,66 +1921,127 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1369
1921
|
.describe("mode=retract: entities whose every current edge is removed (a current-state tombstone; the record and its history are kept for as_of reads)."),
|
|
1370
1922
|
...graphScope,
|
|
1371
1923
|
},
|
|
1372
|
-
|
|
1373
|
-
|
|
1924
|
+
// A retract removes facts, so the tool may destroy.
|
|
1925
|
+
annotations: DESTRUCTIVE,
|
|
1926
|
+
}, async ({ idempotency_key, mode, records, expected_graph_epoch, triplets, entity_embeddings, entity_properties, search_feedback, dry_run, observed_at, edge_idempotency, retract_edges, retract_entities, graph, }, extra) => {
|
|
1927
|
+
if (mode === "records" ||
|
|
1928
|
+
records !== undefined ||
|
|
1929
|
+
expected_graph_epoch !== undefined) {
|
|
1930
|
+
if (mode !== "records")
|
|
1931
|
+
return errorResult(new ToolError("record_mode_required", "records and expected_graph_epoch require explicit mode=records; no implicit legacy write was sent"));
|
|
1932
|
+
return commitRecordsTool(host, client, extra, {
|
|
1933
|
+
records,
|
|
1934
|
+
idempotencyKey: idempotency_key,
|
|
1935
|
+
expectedGraphEpoch: expected_graph_epoch,
|
|
1936
|
+
graph,
|
|
1937
|
+
legacyInputs: [
|
|
1938
|
+
triplets,
|
|
1939
|
+
entity_embeddings,
|
|
1940
|
+
entity_properties,
|
|
1941
|
+
search_feedback,
|
|
1942
|
+
dry_run,
|
|
1943
|
+
observed_at,
|
|
1944
|
+
edge_idempotency,
|
|
1945
|
+
retract_edges,
|
|
1946
|
+
retract_entities,
|
|
1947
|
+
].some((value) => value !== undefined),
|
|
1948
|
+
});
|
|
1949
|
+
}
|
|
1374
1950
|
const commitMode = mode ??
|
|
1375
1951
|
(search_feedback
|
|
1376
1952
|
? "search_feedback"
|
|
1377
1953
|
: retract_edges || retract_entities
|
|
1378
1954
|
? "retract"
|
|
1379
1955
|
: "facts");
|
|
1380
|
-
if (
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
const
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1956
|
+
if (commitMode === "retract" &&
|
|
1957
|
+
!dry_run &&
|
|
1958
|
+
(retract_edges?.length || retract_entities?.length)) {
|
|
1959
|
+
const refused = await host.approve(extra, {
|
|
1960
|
+
message: retractMessage(graphLabel(client, graph), retract_edges ?? [], retract_entities ?? []),
|
|
1961
|
+
});
|
|
1962
|
+
if (refused)
|
|
1963
|
+
return refused;
|
|
1964
|
+
}
|
|
1965
|
+
return run(client, "lbb_commit", "standard", () => {
|
|
1966
|
+
if (dry_run && commitMode !== "facts")
|
|
1967
|
+
throw new Error("dry_run is supported only for lbb_commit mode=facts");
|
|
1968
|
+
if (commitMode === "retract") {
|
|
1969
|
+
const edges = retract_edges ?? [];
|
|
1970
|
+
const entities = retract_entities ?? [];
|
|
1971
|
+
if (edges.length === 0 && entities.length === 0) {
|
|
1972
|
+
throw new Error("lbb_commit mode=retract requires retract_edges or retract_entities");
|
|
1973
|
+
}
|
|
1974
|
+
const key = idempotency_key ??
|
|
1975
|
+
contentHashKey({ graph }, { mode: "retract", edges, entities });
|
|
1976
|
+
return scoped(client, graph).retract({ edges, entities }, { idempotencyKey: key });
|
|
1977
|
+
}
|
|
1978
|
+
if (commitMode === "search_feedback") {
|
|
1979
|
+
if (!search_feedback)
|
|
1980
|
+
throw new Error("lbb_commit mode=search_feedback requires search_feedback");
|
|
1981
|
+
const key = idempotency_key ??
|
|
1982
|
+
contentHashKey({ graph }, { mode: "search_feedback", search_feedback });
|
|
1983
|
+
return scoped(client, graph).searchFeedback({
|
|
1984
|
+
...search_feedback,
|
|
1985
|
+
// A search's snapshot goes back as the server returned it.
|
|
1986
|
+
snapshot: search_feedback.snapshot,
|
|
1987
|
+
}, { idempotencyKey: key });
|
|
1387
1988
|
}
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1989
|
+
if (search_feedback) {
|
|
1990
|
+
throw new Error("lbb_commit facts mode cannot include search_feedback");
|
|
1991
|
+
}
|
|
1992
|
+
const payload = {
|
|
1993
|
+
triplets: triplets ?? [],
|
|
1994
|
+
entity_embeddings: entity_embeddings ?? [],
|
|
1995
|
+
// A flat map or the verbose list per record; the server coerces
|
|
1996
|
+
// each value to its field's declared type.
|
|
1997
|
+
entity_properties: (entity_properties ??
|
|
1998
|
+
[]),
|
|
1999
|
+
...(observed_at ? { observed_at } : {}),
|
|
2000
|
+
edge_idempotency: edge_idempotency ?? "append",
|
|
2001
|
+
};
|
|
2002
|
+
if (!triplets?.length &&
|
|
2003
|
+
!entity_embeddings?.length &&
|
|
2004
|
+
!entity_properties?.length) {
|
|
2005
|
+
throw new Error("lbb_commit requires at least one triplet, entity embedding, or entity property");
|
|
2006
|
+
}
|
|
2007
|
+
const key = idempotency_key ?? contentHashKey({ graph }, payload);
|
|
2008
|
+
if (dry_run)
|
|
2009
|
+
return scoped(client, graph).commitDryRun(payload);
|
|
2010
|
+
return scoped(client, graph).commit(payload, {
|
|
1391
2011
|
idempotencyKey: key,
|
|
1392
2012
|
});
|
|
1393
|
-
}
|
|
1394
|
-
if (commitMode === "search_feedback") {
|
|
1395
|
-
if (!search_feedback)
|
|
1396
|
-
throw new Error("lbb_commit mode=search_feedback requires search_feedback");
|
|
1397
|
-
const key = idempotency_key ??
|
|
1398
|
-
contentHashKey({ graph }, { mode: "search_feedback", search_feedback });
|
|
1399
|
-
return scoped(client, graph).searchFeedback(search_feedback, { idempotencyKey: key });
|
|
1400
|
-
}
|
|
1401
|
-
if (search_feedback) {
|
|
1402
|
-
throw new Error("lbb_commit facts mode cannot include search_feedback");
|
|
1403
|
-
}
|
|
1404
|
-
const payload = {
|
|
1405
|
-
triplets: triplets ?? [],
|
|
1406
|
-
entity_embeddings: entity_embeddings ?? [],
|
|
1407
|
-
entity_properties: entity_properties ?? [],
|
|
1408
|
-
...(observed_at ? { observed_at } : {}),
|
|
1409
|
-
edge_idempotency: edge_idempotency ?? "append",
|
|
1410
|
-
};
|
|
1411
|
-
if (payload.triplets.length === 0 &&
|
|
1412
|
-
payload.entity_embeddings.length === 0 &&
|
|
1413
|
-
payload.entity_properties.length === 0) {
|
|
1414
|
-
throw new Error("lbb_commit requires at least one triplet, entity embedding, or entity property");
|
|
1415
|
-
}
|
|
1416
|
-
const key = idempotency_key ?? contentHashKey({ graph }, payload);
|
|
1417
|
-
if (dry_run)
|
|
1418
|
-
return scoped(client, graph).commitDryRun(payload);
|
|
1419
|
-
return scoped(client, graph).commit(payload, {
|
|
1420
|
-
idempotencyKey: key,
|
|
1421
2013
|
});
|
|
1422
|
-
})
|
|
1423
|
-
|
|
1424
|
-
description:
|
|
2014
|
+
});
|
|
2015
|
+
host.register("lbb_configure", {
|
|
2016
|
+
description: `Manage native schema metadata. Actions: define_ontology (friendly spec with super_types), evolve_ontology (ordered edits including add_super_types), list_starters (the base ontologies crm, documents and work, each with its status on the graph: absent, partial or applied, what applying adds, and conflicts), apply_starter (add what the graph lacks of a starter in one ontology version; a relation the graph has is widened; refused with starter_conflict when the graph holds a term differently; dry_run previews), publish_schema (SHACL activation), suggest_ontology_change (file a change for a person to review instead of applying it; prefer it when the graph's owner reviews ontology changes, and list the result with lbb_inspect action=ontology_suggestions; facts files graph facts for review, which accepting commits after the change, and change may be empty when facts is set), get_rewrite_profile and set_rewrite_profile (the graph's notes and up to 20 worked question-to-SPARQL examples that lbb_query mode=question reads for every question; pass the version you read as expected_version). define, evolve, publish and set_rewrite_profile support dry_run previews. Definition/import here extracts native metadata; it does NOT store the complete RDF/OWL document as queryable graph facts. Use lbb_rdf import for full OWL and lbb_rdf update for additive INSERT DATA revisions; RDF deletions are unsupported. Publish_schema accepts unchanged ontology plus shapes; use define/evolve for native ontology changes. Publication enqueues durable conformance; a preview does not validate the whole graph. An evolve_ontology that removes (remove_entity_type, remove_relation, or narrow_relation with remove_domain or remove_range) and a publish_schema with desired_mode=reject ask the person to approve first; a dry_run does not ask. ${APPROVAL_TEXT}`,
|
|
1425
2017
|
inputSchema: configureWireSchema,
|
|
1426
|
-
|
|
1427
|
-
|
|
2018
|
+
// evolve_ontology can remove types, relations and their source and
|
|
2019
|
+
// target types, and a reject-mode publish refuses writes, so the
|
|
2020
|
+
// tool may destroy.
|
|
2021
|
+
annotations: { ...MUTATING, destructiveHint: true },
|
|
2022
|
+
}, async (rawArgs, extra) => {
|
|
1428
2023
|
const parsed = configureInputSchema.safeParse(rawArgs);
|
|
1429
2024
|
if (!parsed.success)
|
|
1430
2025
|
return errorResult(parsed.error);
|
|
1431
2026
|
const args = parsed.data;
|
|
2027
|
+
if (args.action === "evolve_ontology" && !args.dry_run) {
|
|
2028
|
+
const message = ontologyRemovalMessage(graphLabel(client, args.graph), args.ops);
|
|
2029
|
+
if (message) {
|
|
2030
|
+
const refused = await host.approve(extra, { message });
|
|
2031
|
+
if (refused)
|
|
2032
|
+
return refused;
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
if (args.action === "publish_schema" &&
|
|
2036
|
+
!args.dry_run &&
|
|
2037
|
+
args.desired_mode === "reject" &&
|
|
2038
|
+
args.shapes) {
|
|
2039
|
+
const refused = await host.approve(extra, {
|
|
2040
|
+
message: shapesRejectMessage(graphLabel(client, args.graph), args.shapes),
|
|
2041
|
+
});
|
|
2042
|
+
if (refused)
|
|
2043
|
+
return refused;
|
|
2044
|
+
}
|
|
1432
2045
|
return run(client, `lbb_configure.${args.action}`, "standard", () => {
|
|
1433
2046
|
if (args.action === "define_ontology") {
|
|
1434
2047
|
return client.withScope({ graph: args.graph }).ontologyDefine(ontologyDefineBody({
|
|
@@ -1449,7 +2062,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1449
2062
|
title: args.title,
|
|
1450
2063
|
rationale: args.rationale ?? "",
|
|
1451
2064
|
change,
|
|
1452
|
-
...(args.facts ? { facts: args.facts } : {}),
|
|
2065
|
+
...(args.facts ? { facts: suggestedFactsBody(args.facts) } : {}),
|
|
1453
2066
|
origin: { kind: "agent", id: agent, label: agent },
|
|
1454
2067
|
...(args.anchor ? { anchor: args.anchor } : {}),
|
|
1455
2068
|
...(args.key ? { key: args.key } : {}),
|