@littlebigbrain/mcp 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/tools.js +177 -19
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -155,9 +155,9 @@ its original identifiers; inspect it with SPARQL when choosing query predicates.
|
|
|
155
155
|
| `lbb_fit_sources` | Inspect fit from text and preview the text the fit reads. |
|
|
156
156
|
| `lbb_fit_sources_manage` | Declare, dry-run or refresh a fit source. |
|
|
157
157
|
| `lbb_fit_sources_delete` | Delete a fit source; its suggestions stay. |
|
|
158
|
-
| `lbb_workflows` | List the starter workflows this server offers and the workflows created from them on a graph (triggered workflows), read one with its status, and preview a new one. |
|
|
159
|
-
| `lbb_workflows_manage` | Create a workflow from a starter workflow on a class (a developer workflow with its workflow type, owned outputs and batch), ask it to run now, or pause and resume a developer workflow. |
|
|
160
|
-
| `lbb_workflows_delete` | Delete a triggered workflow. What it wrote stays. |
|
|
158
|
+
| `lbb_workflows` | List the starter workflows this server offers and the workflows created from them on a graph (triggered workflows), read one with its status, and preview a new one. Read the workflows with triggers (target type, shards, routing, window, trigger) and what each watch has not sent yet, with origins. |
|
|
159
|
+
| `lbb_workflows_manage` | Create a workflow from a starter workflow on a class (a developer workflow with its workflow type, owned outputs and batch), ask it to run now, or pause and resume a developer workflow. Define a workflow with triggers of your worker (shards, routing field, window) and run it now. |
|
|
160
|
+
| `lbb_workflows_delete` | Delete a triggered workflow, or a workflow with triggers by its name. What it wrote stays. |
|
|
161
161
|
| `lbb_files` | List the files of a graph, and read the parsed pages of one: each line with its id, its box on the page and its text. |
|
|
162
162
|
| `lbb_commit` | Write or retract JSON facts, or record search feedback. |
|
|
163
163
|
| `lbb_configure` | Define record types and relationships, list and apply ontology starters (CRM, documents, work), publish validation rules, or keep the graph's notes and worked examples for questions in plain words (`get_rewrite_profile`, `set_rewrite_profile`). |
|
package/dist/tools.js
CHANGED
|
@@ -358,7 +358,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
358
358
|
});
|
|
359
359
|
});
|
|
360
360
|
server.registerTool("lbb_models", {
|
|
361
|
-
description: "Read model-training inputs or compare retrieval configurations over one pinned published snapshot, or read what the managed models did. shadow_eval takes the API ShadowEvalRequest body; dataset actions return bounded training examples at an optional signal split. activity reads one month of the stack's model use (all graphs): calls, items, estimated tokens and cost per feature (index, search, fit, judge, training) and model, by day and by graph, the months with activity, and the model each feature uses now.",
|
|
361
|
+
description: "Read model-training inputs or compare retrieval configurations over one pinned published snapshot, or read what the managed models did. shadow_eval takes the API ShadowEvalRequest body; dataset actions return bounded training examples at an optional signal split. activity reads one month of the stack's model use (all graphs): calls, items, estimated tokens and cost per feature (index, search, fit, judge, training, rerank, rewrite, trial, workflows) and model, by day and by graph, the workflows' calls per workflow and job (by_workflow), the months with activity, and the model each feature uses now.",
|
|
362
362
|
inputSchema: {
|
|
363
363
|
action: z.enum([
|
|
364
364
|
"shadow_eval",
|
|
@@ -613,7 +613,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
613
613
|
starter: z
|
|
614
614
|
.string()
|
|
615
615
|
.optional()
|
|
616
|
-
.describe("The starter workflow's catalog id from action=starters, e.g. search.embed (embeddings on a class), ontology.fit (fit from text) or workflow (a developer workflow: your worker's workflow gets the instances). get/preview/use/refresh
|
|
616
|
+
.describe("The starter workflow's catalog id from action=starters, e.g. search.embed (embeddings on a class), ontology.fit (fit from text) or workflow (a developer workflow: your worker's workflow gets the instances). get/preview/use/refresh: required. pause/resume: omit to name a workflow with triggers."),
|
|
617
617
|
class: z
|
|
618
618
|
.string()
|
|
619
619
|
.optional()
|
|
@@ -690,15 +690,90 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
690
690
|
}
|
|
691
691
|
return { starter: args.starter, name: args.name };
|
|
692
692
|
};
|
|
693
|
+
const workflowName = (args) => {
|
|
694
|
+
if (!args.name)
|
|
695
|
+
throw new Error("this action requires name");
|
|
696
|
+
return args.name;
|
|
697
|
+
};
|
|
698
|
+
// A workflow with triggers (`/v1/workflows/named`): your worker's
|
|
699
|
+
// workflow type, one watch trigger, shards, routing and window.
|
|
700
|
+
const namedSetup = {
|
|
701
|
+
shards: z
|
|
702
|
+
.number()
|
|
703
|
+
.int()
|
|
704
|
+
.min(1)
|
|
705
|
+
.max(16)
|
|
706
|
+
.optional()
|
|
707
|
+
.describe("define: engine identities that receive the messages, 1 to 16 (default 1). A change needs every shard's inbox empty."),
|
|
708
|
+
route_by: z
|
|
709
|
+
.string()
|
|
710
|
+
.optional()
|
|
711
|
+
.describe("define: a field of the class whose value keeps work in order: the changes of one value go to one shard. Default: the instance."),
|
|
712
|
+
window: z
|
|
713
|
+
.number()
|
|
714
|
+
.int()
|
|
715
|
+
.min(1)
|
|
716
|
+
.max(128)
|
|
717
|
+
.optional()
|
|
718
|
+
.describe("define: messages the watch keeps queued per shard, 1 to 128 (default 4)."),
|
|
719
|
+
trigger: z
|
|
720
|
+
.string()
|
|
721
|
+
.optional()
|
|
722
|
+
.describe("define: the watch trigger's id (default watch). queue: the trigger whose queue to read."),
|
|
723
|
+
};
|
|
724
|
+
const namedRequest = (args) => {
|
|
725
|
+
if (!args.name || !args.class || !args.workflow) {
|
|
726
|
+
throw new Error("define requires name, class and workflow");
|
|
727
|
+
}
|
|
728
|
+
return {
|
|
729
|
+
name: args.name,
|
|
730
|
+
workflow: args.workflow,
|
|
731
|
+
triggers: [
|
|
732
|
+
{
|
|
733
|
+
...(args.trigger ? { id: args.trigger } : {}),
|
|
734
|
+
watch: {
|
|
735
|
+
class: args.class,
|
|
736
|
+
...(args.fields ? { fields: args.fields } : {}),
|
|
737
|
+
},
|
|
738
|
+
},
|
|
739
|
+
],
|
|
740
|
+
...(args.owns ? { owns: args.owns } : {}),
|
|
741
|
+
...(args.batch !== undefined ? { batch: args.batch } : {}),
|
|
742
|
+
...(args.shards !== undefined ? { shards: args.shards } : {}),
|
|
743
|
+
...(args.route_by ? { route_by: args.route_by } : {}),
|
|
744
|
+
...(args.window !== undefined ? { window: args.window } : {}),
|
|
745
|
+
};
|
|
746
|
+
};
|
|
693
747
|
server.registerTool("lbb_workflows", {
|
|
694
|
-
description: "Starter workflows and triggered workflows, read only. A starter workflow is a template; a triggered workflow is a workflow created from one on the graph. It watches a class: when a published commit adds or changes instances of the class (or changes the watched fields), the workflow reads them and does its work. search.embed keeps an embedding of the class (the same as lbb_embeddings); ontology.fit reads text fields and files ontology suggestions (the same as lbb_fit_sources); workflow (a developer workflow) sends the new and changed instances to a workflow of the developer's worker as trigger messages, and that code writes the facts the workflow owns. documents.parse reads uploaded PDF files into pages with line boxes, documents.link writes DocEntry and DocMention instances and links each mention to
|
|
748
|
+
description: "Starter workflows and triggered workflows, read only. A starter workflow is a template; a triggered workflow is a workflow created from one on the graph. It watches a class: when a published commit adds or changes instances of the class (or changes the watched fields), the workflow reads them and does its work. search.embed keeps an embedding of the class (the same as lbb_embeddings); ontology.fit reads text fields and files ontology suggestions (the same as lbb_fit_sources); workflow (a developer workflow) sends the new and changed instances to a workflow of the developer's worker as trigger messages, and that code writes the facts the workflow owns. documents.parse reads uploaded PDF files into pages with line boxes, documents.link connects mentions of the same thing across the documents (it writes DocEntry and DocMention instances and links each mention to the entry that defines or describes it), documents.map proposes each link as a FactProposal in the graph's own ontology (fact_subject, fact_predicate, fact_object, fact_confidence, fact_evidence) for a review workflow, and values.extract fills chosen properties from text, each value with a quote; read the files and their pages with lbb_files. starters lists the starter workflows this server offers with their group, settings and whether each can run here. list shows every triggered workflow of the graph with its state (backfilling, ready, paused, failed, unavailable), lag, counters, this month's spend and last error; get shows one (starter and name) with its own status in details (for workflow: the messages sent, the instances waiting, the message in flight and whether it is paused). preview shows what creating one would do (the resolved watch, the instance count and the workflow's own preview; for workflow, the first messages it would send), calls no model and stores nothing. workflows lists the same workflows as workflows with triggers (/v1/workflows/named): each one's target workflow type (LBB's own are lbb.* types), its shards (engine identities starter.<name> and starter.<name>.<k>), its routing field, its window, its one watch trigger, its status and each shard's inbox; name for one. queue (name) shows what a workflow's watch has not sent yet: each waiting instance with its shard and its origin (a backfill, a commit range with the watched fields and the requests that changed it, a run now, a schema change), the pass in progress, and per shard its message in flight and its first unfinished turns with their origins; page with after and limit. Message workflows of the workflow engine are not listed here. Create, refresh, pause or resume with lbb_workflows_manage; delete with lbb_workflows_delete.",
|
|
695
749
|
inputSchema: {
|
|
696
|
-
action: z.enum([
|
|
750
|
+
action: z.enum([
|
|
751
|
+
"starters",
|
|
752
|
+
"list",
|
|
753
|
+
"get",
|
|
754
|
+
"preview",
|
|
755
|
+
"workflows",
|
|
756
|
+
"queue",
|
|
757
|
+
]),
|
|
697
758
|
name: z
|
|
698
759
|
.string()
|
|
699
760
|
.optional()
|
|
700
|
-
.describe("get: the triggered workflow's name. preview: the name a new workflow would get."),
|
|
761
|
+
.describe("get: the triggered workflow's name. preview: the name a new workflow would get. workflows: one workflow's name (omit for all). queue: required."),
|
|
701
762
|
...workflowSetup,
|
|
763
|
+
trigger: namedSetup.trigger,
|
|
764
|
+
after: z
|
|
765
|
+
.number()
|
|
766
|
+
.int()
|
|
767
|
+
.nonnegative()
|
|
768
|
+
.optional()
|
|
769
|
+
.describe("queue: the position to start from (next_after)."),
|
|
770
|
+
limit: z
|
|
771
|
+
.number()
|
|
772
|
+
.int()
|
|
773
|
+
.min(1)
|
|
774
|
+
.max(100)
|
|
775
|
+
.optional()
|
|
776
|
+
.describe("queue: items per page, 1 to 100 (default 50)."),
|
|
702
777
|
sample: z
|
|
703
778
|
.number()
|
|
704
779
|
.int()
|
|
@@ -729,12 +804,22 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
729
804
|
...(args.sample ? { sample: args.sample } : {}),
|
|
730
805
|
...(args.iris ? { iris: args.iris } : {}),
|
|
731
806
|
});
|
|
807
|
+
case "workflows":
|
|
808
|
+
return args.name
|
|
809
|
+
? workflows.named.get(args.name)
|
|
810
|
+
: workflows.named.list();
|
|
811
|
+
case "queue":
|
|
812
|
+
return workflows.named.queue(workflowName(args), {
|
|
813
|
+
...(args.trigger ? { trigger: args.trigger } : {}),
|
|
814
|
+
...(args.after !== undefined ? { after: args.after } : {}),
|
|
815
|
+
...(args.limit !== undefined ? { limit: args.limit } : {}),
|
|
816
|
+
});
|
|
732
817
|
}
|
|
733
818
|
}));
|
|
734
819
|
server.registerTool("lbb_workflows_manage", {
|
|
735
|
-
description: "Create a workflow from a starter workflow, or refresh, pause or resume a triggered workflow. Run lbb_workflows action=preview first and show the user the watch and the instance count. use creates a workflow from a starter workflow of action=starters, under a name (default: the class's local name), with its watch (class, fields) and params; the workflow then reads every instance of the class and follows each published commit. A developer workflow (starter=workflow) also takes workflow (workflow_type and version of the developer's worker), owns and batch; it needs a server with the workflow engine. use again with the same starter and name changes only what you name, and the same request again changes nothing. A workflow that calls models (ontology.fit, documents.link,
|
|
820
|
+
description: "Create a workflow from a starter workflow, or refresh, pause or resume a triggered workflow. Run lbb_workflows action=preview first and show the user the watch and the instance count. use creates a workflow from a starter workflow of action=starters, under a name (default: the class's local name), with its watch (class, fields) and params; the workflow then reads every instance of the class and follows each published commit. A developer workflow (starter=workflow) also takes workflow (workflow_type and version of the developer's worker), owns and batch; it needs a server with the workflow engine. use again with the same starter and name changes only what you name, and the same request again changes nothing. A workflow that calls models (ontology.fit, documents.link, documents.map, values.extract) spends model budget as it reads. refresh asks the workflow to run now and returns at once with queued; poll lbb_workflows action=get for progress. pause stops a workflow (a developer workflow, documents.parse, documents.link, documents.map or values.extract) from reading and sending, and stops the claims of a developer workflow's shards; resume lets it catch up. Without starter, pause and resume name a workflow with triggers. search.embed and ontology.fit cannot be paused. define creates or changes a workflow with triggers of the developer's worker (/v1/workflows/named): name, workflow, class and fields (its watch trigger), owns, batch, and shards (engine identities, 1 to 16), route_by (a field that keeps work in order) and window (messages queued per shard, default 4); it needs a server with the workflow engine, and a change of shards waits for empty inboxes. run (name) runs a workflow with triggers now: its watch sends every instance again.",
|
|
736
821
|
inputSchema: {
|
|
737
|
-
action: z.enum(["use", "refresh", "pause", "resume"]),
|
|
822
|
+
action: z.enum(["use", "refresh", "pause", "resume", "define", "run"]),
|
|
738
823
|
name: z
|
|
739
824
|
.string()
|
|
740
825
|
.optional()
|
|
@@ -751,6 +836,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
751
836
|
.nonnegative()
|
|
752
837
|
.optional()
|
|
753
838
|
.describe("use: a monthly model budget in USD for this workflow, below the stack's own."),
|
|
839
|
+
...namedSetup,
|
|
754
840
|
detail: detailSchema,
|
|
755
841
|
...graphScope,
|
|
756
842
|
},
|
|
@@ -763,17 +849,26 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
763
849
|
case "refresh":
|
|
764
850
|
return workflows.triggered.refresh(workflowRef(args));
|
|
765
851
|
case "pause":
|
|
766
|
-
return
|
|
852
|
+
return args.starter
|
|
853
|
+
? workflows.triggered.pause(workflowRef(args), true)
|
|
854
|
+
: workflows.named.pause(workflowName(args), true);
|
|
767
855
|
case "resume":
|
|
768
|
-
return
|
|
856
|
+
return args.starter
|
|
857
|
+
? workflows.triggered.pause(workflowRef(args), false)
|
|
858
|
+
: workflows.named.pause(workflowName(args), false);
|
|
859
|
+
case "define":
|
|
860
|
+
return workflows.named.put(namedRequest(args));
|
|
861
|
+
case "run":
|
|
862
|
+
return workflows.named.run(workflowName(args));
|
|
769
863
|
}
|
|
770
864
|
}));
|
|
771
865
|
server.registerTool("lbb_workflows_delete", {
|
|
772
|
-
description: "Delete a triggered workflow: it stops following its class, and what it wrote stays. Deleting search.embed deletes the embedding and its vectors, so its class stops being searchable; the suggestions an ontology.fit filed stay; a developer workflow leaves its workflow instance and the facts its code wrote. Ask the user first; confirm must repeat the name.",
|
|
866
|
+
description: "Delete a triggered workflow: it stops following its class, and what it wrote stays. Deleting search.embed deletes the embedding and its vectors, so its class stops being searchable; the suggestions an ontology.fit filed stay; a developer workflow leaves its workflow instance and the facts its code wrote. Without starter it deletes a workflow with triggers by its name, and its shards stay. Ask the user first; confirm must repeat the name.",
|
|
773
867
|
inputSchema: {
|
|
774
868
|
starter: z
|
|
775
869
|
.string()
|
|
776
|
-
.
|
|
870
|
+
.optional()
|
|
871
|
+
.describe("The starter workflow's catalog id, e.g. search.embed. Omit for a workflow with triggers."),
|
|
777
872
|
name: z.string().describe("The triggered workflow's name."),
|
|
778
873
|
confirm: z.string().describe("The same name again, to confirm."),
|
|
779
874
|
detail: detailSchema,
|
|
@@ -784,10 +879,10 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
784
879
|
if (confirm !== name) {
|
|
785
880
|
throw new Error("confirm must repeat the workflow name");
|
|
786
881
|
}
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
name
|
|
790
|
-
|
|
882
|
+
const workflows = scoped(client, graph).workflows;
|
|
883
|
+
return starter
|
|
884
|
+
? workflows.triggered.delete({ starter, name })
|
|
885
|
+
: workflows.named.delete(name);
|
|
791
886
|
}));
|
|
792
887
|
const modelUse = z
|
|
793
888
|
.enum(["ask", "route", "rerank", "fit", "label"])
|
|
@@ -921,7 +1016,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
921
1016
|
}
|
|
922
1017
|
}));
|
|
923
1018
|
server.registerTool("lbb_evals", {
|
|
924
|
-
description: "Managed evals: ground truth for the questions an app asks and the searches it runs. A query run with `request` (lbb_query, also mode=question) records a trace: the question, the query that answered it, the type of that query (`query_type`: sparql, hybrid = search by meaning with conditions, search = search by meaning alone) and one item per result (hit or row); trace reads it back with the item ids. label has two forms. On a question's trace, valid alone judges the whole answer: valid=true makes the trace's query the golden query of the question; valid=false with sparql gives the right query, which becomes the golden query; valid=false alone marks the answer wrong. item + valid (or items) judges one result, a citation of the answer: the ground truth of the hits of a search by meaning, and a result marked wrong must not come back for any type. golden freezes a stored query or a search (every result it returns now is relevant). run checks every golden at the current commit: a question is asked again through the query rewriter, then the type of the query it wrote is compared with the expected type, its rows with the rows of the golden query at the same commit (`query_check`), and its results with the judged results; a search is done again; new results open a review trace. judge lets the platform's judge model (the hosted frontier model) label unlabeled results. summary, traces, goldens, results, and settings read state. Model checks: a judge model checks a sample of the model calls LBB makes for the graph (rerank, route, ask = the answer of a question, rewrite, fit, propose, label). checks_summary reads a month per job and model (checks, score, right, partly, wrong, reviews) and the judge's agreement with people; checks lists the month's checks, newest first, with the judge's verdict, score and reason and the ground truth (`truth`). review_check records a person's review of one check (call_id): agree=true keeps the judge's verdict; agree=false with verdict (and an optional score, reference and note) corrects it. The review becomes the call's ground truth, so show the check to the user and review only what the user confirmed.",
|
|
1019
|
+
description: "Managed evals: ground truth for the questions an app asks and the searches it runs. A query run with `request` (lbb_query, also mode=question) records a trace: the question, the query that answered it, the type of that query (`query_type`: sparql, hybrid = search by meaning with conditions, search = search by meaning alone) and one item per result (hit or row); trace reads it back with the item ids. label has two forms. On a question's trace, valid alone judges the whole answer: valid=true makes the trace's query the golden query of the question; valid=false with sparql gives the right query, which becomes the golden query; valid=false alone marks the answer wrong; route with valid=true (or valid=false with sparql) names the route the question expects, and valid=true alone takes the trace's route. item + valid (or items) judges one result, a citation of the answer: the ground truth of the hits of a search by meaning, and a result marked wrong must not come back for any type. golden freezes a stored query or a search (every result it returns now is relevant). run checks every golden at the current commit: a question is asked again through the query rewriter, then the type of the query it wrote is compared with the expected type, its rows with the rows of the golden query at the same commit (`query_check`), and its results with the judged results, and the router's pick with the golden's route (route_check, which does not change the verdict); a search is done again; new results open a review trace. run_answers checks your own app's answers instead: ask each question golden's request with your own loop, then send answers=[{golden, sparql, route}] (route = the route of lbb_query mode=question with route mode, sparql absent for an unanswerable question, error for a failure); the server runs each query and checks it the same way, and the runs form the app's own series (runner; results with runner reads them). judge lets the platform's judge model (the hosted frontier model) label unlabeled results. summary, traces, goldens, results, and settings read state. Model checks: a judge model checks a sample of the model calls LBB makes for the graph (rerank, route, ask = the answer of a question, rewrite, fit, propose, label). checks_summary reads a month per job and model (checks, score, right, partly, wrong, reviews) and the judge's agreement with people; checks lists the month's checks, newest first, with the judge's verdict, score and reason and the ground truth (`truth`). review_check records a person's review of one check (call_id): agree=true keeps the judge's verdict; agree=false with verdict (and an optional score, reference and note) corrects it. The review becomes the call's ground truth, so show the check to the user and review only what the user confirmed.",
|
|
925
1020
|
inputSchema: {
|
|
926
1021
|
action: z.enum([
|
|
927
1022
|
"summary",
|
|
@@ -934,6 +1029,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
934
1029
|
"accept",
|
|
935
1030
|
"delete",
|
|
936
1031
|
"run",
|
|
1032
|
+
"run_answers",
|
|
937
1033
|
"results",
|
|
938
1034
|
"settings",
|
|
939
1035
|
"checks_summary",
|
|
@@ -974,6 +1070,58 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
974
1070
|
.string()
|
|
975
1071
|
.optional()
|
|
976
1072
|
.describe("golden: the query to freeze (the search text for surface=search). label: with valid=false and no item, the right query for the question; the server checks and runs it once."),
|
|
1073
|
+
route: z
|
|
1074
|
+
.enum([
|
|
1075
|
+
"lookup",
|
|
1076
|
+
"aggregate",
|
|
1077
|
+
"search",
|
|
1078
|
+
"history",
|
|
1079
|
+
"schema",
|
|
1080
|
+
"unanswerable",
|
|
1081
|
+
])
|
|
1082
|
+
.optional()
|
|
1083
|
+
.describe("label: with valid=true, or valid=false with sparql, the route the question expects."),
|
|
1084
|
+
answers: z
|
|
1085
|
+
.array(z
|
|
1086
|
+
.object({
|
|
1087
|
+
golden: z.string().min(1),
|
|
1088
|
+
sparql: z.string().optional(),
|
|
1089
|
+
entailment: z
|
|
1090
|
+
.enum(["none", "subclass", "rdfs", "owl"])
|
|
1091
|
+
.optional(),
|
|
1092
|
+
as_of_commit_seq: z.number().int().nonnegative().optional(),
|
|
1093
|
+
route: z
|
|
1094
|
+
.object({
|
|
1095
|
+
kind: z.enum([
|
|
1096
|
+
"lookup",
|
|
1097
|
+
"aggregate",
|
|
1098
|
+
"search",
|
|
1099
|
+
"history",
|
|
1100
|
+
"schema",
|
|
1101
|
+
"unanswerable",
|
|
1102
|
+
]),
|
|
1103
|
+
confidence: z.number(),
|
|
1104
|
+
by: z.enum(["caller", "router", "rewriter"]),
|
|
1105
|
+
probabilities: z.record(z.string(), z.number()).optional(),
|
|
1106
|
+
})
|
|
1107
|
+
.strict()
|
|
1108
|
+
.optional(),
|
|
1109
|
+
reply: z
|
|
1110
|
+
.object({
|
|
1111
|
+
text: z.string(),
|
|
1112
|
+
citations: z.array(z.string()).optional(),
|
|
1113
|
+
})
|
|
1114
|
+
.optional(),
|
|
1115
|
+
error: z.string().optional(),
|
|
1116
|
+
cost_micro_usd: z.number().int().nonnegative().optional(),
|
|
1117
|
+
})
|
|
1118
|
+
.strict())
|
|
1119
|
+
.optional()
|
|
1120
|
+
.describe("run_answers: one answer per question golden your app asked: golden (its id), sparql (the query whose rows hold the answer; leave it out when the question is unanswerable), entailment, as_of_commit_seq, route (as the router returned it), reply, error (your app failed), cost_micro_usd."),
|
|
1121
|
+
runner: z
|
|
1122
|
+
.string()
|
|
1123
|
+
.optional()
|
|
1124
|
+
.describe("run_answers / results: your app's name, lower-case letters, digits and -, at most 40 characters (default app). results without runner reads the server's runs."),
|
|
977
1125
|
surface: z
|
|
978
1126
|
.enum(["sparql", "search"])
|
|
979
1127
|
.optional()
|
|
@@ -1005,7 +1153,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1005
1153
|
consistency: z
|
|
1006
1154
|
.enum(["strong", "eventual"])
|
|
1007
1155
|
.optional()
|
|
1008
|
-
.describe("run / accept: strong reads the head, eventual the last published commit."),
|
|
1156
|
+
.describe("run / run_answers / accept: strong reads the head, eventual the last published commit."),
|
|
1009
1157
|
month: yearMonthSchema
|
|
1010
1158
|
.optional()
|
|
1011
1159
|
.describe("checks_summary / checks: the month, yyyy-mm (UTC). Defaults to the current month."),
|
|
@@ -1054,7 +1202,7 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1054
1202
|
...graphScope,
|
|
1055
1203
|
},
|
|
1056
1204
|
annotations: MUTATING,
|
|
1057
|
-
}, ({ action, trace_id, item, valid, items, by, note, sparql, surface, embedding, top_k, request, golden_id, limit, unlabeled, consistency, month, job, verdict, reviewed, after, call_id, agree, score, reference, detail, graph, }) => run(client, `lbb_evals.${action}`, detail, () => {
|
|
1205
|
+
}, ({ action, trace_id, item, valid, items, by, note, sparql, route, answers, runner, surface, embedding, top_k, request, golden_id, limit, unlabeled, consistency, month, job, verdict, reviewed, after, call_id, agree, score, reference, detail, graph, }) => run(client, `lbb_evals.${action}`, detail, () => {
|
|
1058
1206
|
const target = scoped(client, graph);
|
|
1059
1207
|
switch (action) {
|
|
1060
1208
|
case "summary":
|
|
@@ -1075,12 +1223,18 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1075
1223
|
if (sparql !== undefined &&
|
|
1076
1224
|
(item || items?.length || valid !== false))
|
|
1077
1225
|
throw new Error("label takes sparql (the right query) only with valid=false and no item");
|
|
1226
|
+
if (route !== undefined &&
|
|
1227
|
+
(item ||
|
|
1228
|
+
items?.length ||
|
|
1229
|
+
(valid !== true && sparql === undefined)))
|
|
1230
|
+
throw new Error("label takes route only with valid=true, or valid=false with sparql, and no item");
|
|
1078
1231
|
return target.evals.label(trace_id, {
|
|
1079
1232
|
...(item ? { item, valid } : {}),
|
|
1080
1233
|
...(items?.length ? { items } : {}),
|
|
1081
1234
|
// The answer of a question as a whole.
|
|
1082
1235
|
...(!item && !items?.length ? { valid } : {}),
|
|
1083
1236
|
...(sparql !== undefined ? { sparql } : {}),
|
|
1237
|
+
...(route !== undefined ? { route } : {}),
|
|
1084
1238
|
by,
|
|
1085
1239
|
note,
|
|
1086
1240
|
});
|
|
@@ -1110,8 +1264,12 @@ export function registerLbbTools(server, client, options = {}) {
|
|
|
1110
1264
|
return target.evals.deleteGolden(golden_id);
|
|
1111
1265
|
case "run":
|
|
1112
1266
|
return target.evals.run({ consistency });
|
|
1267
|
+
case "run_answers":
|
|
1268
|
+
if (!answers?.length)
|
|
1269
|
+
throw new Error("run_answers requires answers");
|
|
1270
|
+
return target.evals.runAnswers({ answers, ...(runner ? { runner } : {}) }, { consistency });
|
|
1113
1271
|
case "results":
|
|
1114
|
-
return target.evals.results({ limit });
|
|
1272
|
+
return target.evals.results({ limit, runner });
|
|
1115
1273
|
case "settings":
|
|
1116
1274
|
return target.evals.settings();
|
|
1117
1275
|
case "checks_summary":
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@littlebigbrain/mcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "MCP server for little big brain: search, graph queries, and data import",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"test:coverage": "npm run clean:test && tsc -p tsconfig.test.json && node --test --experimental-test-coverage --test-coverage-include=test-dist/*.js --test-coverage-exclude=test-dist/*.test.js --test-coverage-exclude=test-dist/stdio.js --test-coverage-exclude=test-dist/test-support.js --test-coverage-lines=90 --test-coverage-branches=70 --test-coverage-functions=85 \"test-dist/**/*.test.js\""
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@littlebigbrain/client": "^0.
|
|
55
|
+
"@littlebigbrain/client": "^0.26.0",
|
|
56
56
|
"@modelcontextprotocol/sdk": "^1",
|
|
57
57
|
"zod": "^3"
|
|
58
58
|
},
|