nexus-ai-pro 1.12.0 → 1.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/API_STABILITY.md +44 -0
  2. package/CHANGELOG.md +101 -1
  3. package/README.md +249 -0
  4. package/dist/agent/create-agent.d.ts +119 -0
  5. package/dist/agent/create-agent.js +176 -0
  6. package/dist/agent/index.d.ts +5 -0
  7. package/dist/agent/index.js +4 -0
  8. package/dist/agent/middleware.d.ts +46 -0
  9. package/dist/agent/middleware.js +77 -0
  10. package/dist/evaluate/compare.d.ts +64 -0
  11. package/dist/evaluate/compare.js +148 -0
  12. package/dist/evaluate/datasets.d.ts +79 -0
  13. package/dist/evaluate/datasets.js +144 -0
  14. package/dist/evaluate/evaluators.d.ts +71 -0
  15. package/dist/evaluate/evaluators.js +190 -0
  16. package/dist/evaluate/index.d.ts +6 -0
  17. package/dist/evaluate/index.js +5 -0
  18. package/dist/evaluate/review.d.ts +103 -0
  19. package/dist/evaluate/review.js +168 -0
  20. package/dist/evaluate/run.d.ts +34 -0
  21. package/dist/evaluate/run.js +162 -0
  22. package/dist/graph/graph.js +1 -0
  23. package/dist/mcp/client.d.ts +106 -0
  24. package/dist/mcp/client.js +243 -0
  25. package/dist/mcp/index.d.ts +3 -0
  26. package/dist/mcp/index.js +3 -0
  27. package/dist/mcp/protocol.d.ts +71 -0
  28. package/dist/mcp/protocol.js +68 -0
  29. package/dist/mcp/server.d.ts +46 -0
  30. package/dist/mcp/server.js +155 -0
  31. package/dist/store/index.d.ts +2 -0
  32. package/dist/store/index.js +1 -0
  33. package/dist/store/memory.d.ts +41 -0
  34. package/dist/store/memory.js +182 -0
  35. package/dist/store/redis.d.ts +49 -0
  36. package/dist/store/redis.js +128 -0
  37. package/dist/tracing/alerts.d.ts +60 -0
  38. package/dist/tracing/alerts.js +98 -0
  39. package/dist/tracing/compare.d.ts +31 -0
  40. package/dist/tracing/compare.js +69 -0
  41. package/dist/tracing/index.d.ts +6 -0
  42. package/dist/tracing/index.js +5 -0
  43. package/dist/tracing/instrument.d.ts +47 -0
  44. package/dist/tracing/instrument.js +138 -0
  45. package/dist/tracing/stores.d.ts +53 -0
  46. package/dist/tracing/stores.js +192 -0
  47. package/dist/tracing/tracer.d.ts +73 -0
  48. package/dist/tracing/tracer.js +182 -0
  49. package/dist/types/evaluate.d.ts +117 -0
  50. package/dist/types/evaluate.js +10 -0
  51. package/dist/types/graph.d.ts +10 -0
  52. package/dist/types/store.d.ts +61 -0
  53. package/dist/types/store.js +9 -0
  54. package/dist/types/tracing.d.ts +102 -0
  55. package/dist/types/tracing.js +10 -0
  56. package/dist-cjs/agent/create-agent.d.ts +119 -0
  57. package/dist-cjs/agent/create-agent.js +181 -0
  58. package/dist-cjs/agent/index.d.ts +5 -0
  59. package/dist-cjs/agent/index.js +16 -0
  60. package/dist-cjs/agent/middleware.d.ts +46 -0
  61. package/dist-cjs/agent/middleware.js +82 -0
  62. package/dist-cjs/evaluate/compare.d.ts +64 -0
  63. package/dist-cjs/evaluate/compare.js +153 -0
  64. package/dist-cjs/evaluate/datasets.d.ts +79 -0
  65. package/dist-cjs/evaluate/datasets.js +190 -0
  66. package/dist-cjs/evaluate/evaluators.d.ts +71 -0
  67. package/dist-cjs/evaluate/evaluators.js +202 -0
  68. package/dist-cjs/evaluate/index.d.ts +6 -0
  69. package/dist-cjs/evaluate/index.js +33 -0
  70. package/dist-cjs/evaluate/review.d.ts +103 -0
  71. package/dist-cjs/evaluate/review.js +173 -0
  72. package/dist-cjs/evaluate/run.d.ts +34 -0
  73. package/dist-cjs/evaluate/run.js +167 -0
  74. package/dist-cjs/graph/graph.js +1 -0
  75. package/dist-cjs/mcp/client.d.ts +106 -0
  76. package/dist-cjs/mcp/client.js +282 -0
  77. package/dist-cjs/mcp/index.d.ts +3 -0
  78. package/dist-cjs/mcp/index.js +15 -0
  79. package/dist-cjs/mcp/protocol.d.ts +71 -0
  80. package/dist-cjs/mcp/protocol.js +77 -0
  81. package/dist-cjs/mcp/server.d.ts +46 -0
  82. package/dist-cjs/mcp/server.js +160 -0
  83. package/dist-cjs/store/index.d.ts +2 -0
  84. package/dist-cjs/store/index.js +6 -0
  85. package/dist-cjs/store/memory.d.ts +41 -0
  86. package/dist-cjs/store/memory.js +190 -0
  87. package/dist-cjs/store/redis.d.ts +49 -0
  88. package/dist-cjs/store/redis.js +132 -0
  89. package/dist-cjs/tracing/alerts.d.ts +60 -0
  90. package/dist-cjs/tracing/alerts.js +104 -0
  91. package/dist-cjs/tracing/compare.d.ts +31 -0
  92. package/dist-cjs/tracing/compare.js +73 -0
  93. package/dist-cjs/tracing/index.d.ts +6 -0
  94. package/dist-cjs/tracing/index.js +21 -0
  95. package/dist-cjs/tracing/instrument.d.ts +47 -0
  96. package/dist-cjs/tracing/instrument.js +142 -0
  97. package/dist-cjs/tracing/stores.d.ts +53 -0
  98. package/dist-cjs/tracing/stores.js +202 -0
  99. package/dist-cjs/tracing/tracer.d.ts +73 -0
  100. package/dist-cjs/tracing/tracer.js +187 -0
  101. package/dist-cjs/types/evaluate.d.ts +117 -0
  102. package/dist-cjs/types/evaluate.js +11 -0
  103. package/dist-cjs/types/graph.d.ts +10 -0
  104. package/dist-cjs/types/store.d.ts +61 -0
  105. package/dist-cjs/types/store.js +10 -0
  106. package/dist-cjs/types/tracing.d.ts +102 -0
  107. package/dist-cjs/types/tracing.js +11 -0
  108. package/package.json +79 -1
package/API_STABILITY.md CHANGED
@@ -328,6 +328,50 @@ Also since 1.12.0, all additive:
328
328
  - A subgraph node's thread id is `<parent thread>:<task id>`, which equals the previous
329
329
  `<parent thread>:<node>` for any node not reached through `Send`.
330
330
 
331
+ ## Agent, store, MCP, and tracing stages (1.14.0)
332
+
333
+ The `nexus-ai-pro/agent`, `nexus-ai-pro/store`, `nexus-ai-pro/store/redis`, and `nexus-ai-pro/mcp`
334
+ subpaths are public and follow the 1.x rules. None is exported from the root import.
335
+
336
+ - `createAgent()` returns a compiled graph, so the graph guarantees above apply to an agent: a
337
+ superstep is atomic with respect to checkpointing, a finished task is never re-run by a resume, and
338
+ an approval survives a restart. The agent's state channels (`messages`, `iterations`, `answer`,
339
+ `stopReason`) are a normalized shape; what a model decides to put in them is not.
340
+ - `Store` is the contract; `MemoryStore` and `RedisStore` implement it. Ranking from a semantic search
341
+ depends on the embedding function a caller injects and is not a guarantee. `MemoryStore` is
342
+ process-local and bounded, and drops the least recently written item past `maxItems`.
343
+ - The MCP subpath implements a subset of the Model Context Protocol: initialize, tools, resources, and
344
+ prompts, over stdio and HTTP. Protocol versions and server behaviour are upstream contracts outside
345
+ this policy. Server-initiated streaming, sampling, and roots are not implemented; new methods may be
346
+ added in a minor release.
347
+ - `nexus-ai-pro/tracing` is public: `Tracer`, the `TraceStore` contract, `MemoryTraceStore`,
348
+ `JsonlTraceStore`, `traceGraph()`, `traceModelClient()`, `compareTraces()`, and `AlertEvaluator`.
349
+ The `Run`, `RunTree`, and `RunQuery` shapes are normalized; what a model puts in a run's inputs and
350
+ outputs is not. New `RunKind` values and query fields may be added in a minor release. Sampling is
351
+ probabilistic by definition, so which traces are kept is not a guarantee, but the tail rules are:
352
+ an error, a run past `keepSlowerThanMs`, or one past `keepCostlierThan` is always kept.
353
+ - `agentAsTool()` and the bundled middleware (`summarizeHistory`, `redactMessages`, `limitToolCalls`)
354
+ are public and additive.
355
+ - `AgentLoop`, `tool()`, and `ToolExecutor` keep their behaviour and are re-exported from
356
+ `nexus-ai-pro/agent` as well as the root.
357
+
358
+ ## Evaluation stage (1.15.0)
359
+
360
+ `nexus-ai-pro/evaluate` is public and follows the 1.x rules, and is not exported from the root.
361
+ `Dataset`, `DatasetExample`, `Experiment`, `ExampleResult`, `MetricSummary`, and the comparison shapes
362
+ are normalized; new optional fields and new bundled evaluators may be added in a minor release.
363
+
364
+ - A dataset version is derived from example content. The hash itself is an implementation detail, but
365
+ the guarantee is not: identical examples produce identical versions, and any change produces a
366
+ different one.
367
+ - `compareExperiments()` is statistical. Its intervals come from a seeded paired bootstrap, so the
368
+ same inputs give the same verdict; the exact interval bounds may change if the method improves, and
369
+ a verdict is a judgement about evidence rather than a compatibility guarantee.
370
+ - `evaluate()` reports a target's failure as a failed example and an evaluator's failure as a failed
371
+ measurement; neither throws.
372
+ - The existing `EvalRunner` and `MediaEvalRunner` keep their APIs unchanged. Rebuilding them on this
373
+ entry point is a later, internal change.
374
+
331
375
  ## Deprecation process
332
376
 
333
377
  Deprecated APIs are marked with `@deprecated` in declarations and described in the changelog. Removals
package/CHANGELOG.md CHANGED
@@ -4,6 +4,104 @@ Notable changes to this project are recorded here. The format follows [Keep a Ch
4
4
 
5
5
  ## [Unreleased]
6
6
 
7
+ ## [1.15.0] - 2026-09-21
8
+
9
+ Evaluation. One entry point for evaluating anything, datasets versioned by their content, experiments
10
+ that can be compared with a verdict rather than a vibe, and the loop from a production failure back to
11
+ a permanent regression test.
12
+
13
+ ### Added
14
+
15
+ - **`nexus-ai-pro/evaluate`.** `evaluate(target, dataset, evaluators, options)` accepts any target: a
16
+ completion, an agent, a graph, an image operation, or plain code.
17
+ - Examples run concurrently, with repetitions for a target that is not deterministic, a timeout per
18
+ example, and per-metric statistics including a 95% interval.
19
+ - A target that throws is recorded as a failed example; an evaluator that throws is recorded as a
20
+ failed measurement. Neither voids the experiment.
21
+ - **Datasets versioned by content.** Change an example and the version changes with it, so two
22
+ experiments are comparable only when they ran over the same data. `MemoryDatasetStore` and
23
+ `FileDatasetStore` (one reviewable JSON file per version), plus `splitOf()` for train and test
24
+ splits.
25
+ - **`datasetFromTraces()`.** Turns recorded runs into examples, each keeping the run it came from, so
26
+ yesterday's production failure becomes tomorrow's regression test.
27
+ - **Bundled evaluators.** `exactMatch`, `contains`, `mustNotMatch`, `completed`, `underLatency`,
28
+ `embeddingSimilarity` through any injected embedder, `pairwise` for side-by-side judgements, and
29
+ `trajectory`, which scores how an answer was reached rather than only what it said. `passRate` and
30
+ `totalCost` summarize a whole experiment.
31
+ - **`compareExperiments()` with a real verdict.** A paired bootstrap over per-example differences
32
+ gives each metric a 95% interval; a metric whose interval spans zero is `unchanged` rather than an
33
+ improvement. The bootstrap is seeded, so a CI gate cannot flip between runs. The report names the
34
+ examples that moved most, the failures that are new, the ones that were fixed, and any dataset
35
+ version mismatch. `formatComparison()` prints it for a pull request.
36
+ - **`evaluateOnline()`.** Scores sampled production runs, writes the scores back as trace feedback so
37
+ alert rules can watch them, and sends uncertain results to people.
38
+ - **`AnnotationQueue`.** Review work with a rubric, claims that expire so nothing is stranded,
39
+ consensus across several reviewers, and `toExamples()` to turn reviewed items into dataset examples.
40
+
41
+ ## [1.14.0] - 2026-09-20
42
+
43
+ The memory, agent, and MCP work planned for 1.13.0 ships here together with tracing, so no 1.13.0 was
44
+ published.
45
+
46
+ Memory, agents, MCP, and traces. An agent is now a graph rather than a loop, so it inherits
47
+ approvals, parallel tool calls, and durability. A store gives it memory that outlives a thread, MCP
48
+ connects it to tools this package does not have to ship, and every run can be recorded as a tree you
49
+ can search, compare, and alert on.
50
+
51
+ ### Added
52
+
53
+ - **`nexus-ai-pro/agent` and `createAgent()`.** Returns a compiled graph, so an agent gets
54
+ checkpoints, resume, forks, events, and a diagram from the graph runtime rather than from code
55
+ written twice.
56
+ - Tool calls run as parallel tasks, bounded by `toolConcurrency`.
57
+ - `interruptOn` makes a tool wait for a human. The approval is a checkpoint, so it can arrive days
58
+ later from another process, and it can approve, refuse with a reason the model sees, or approve
59
+ with corrected arguments.
60
+ - Middleware wraps the model call (`beforeModel`, `afterModel`) and each tool call
61
+ (`wrapToolCall`).
62
+ - `stopReason` distinguishes a finished answer from one that ran out of iterations, and malformed
63
+ tool arguments never reach a tool.
64
+ - `AgentLoop` is unchanged, for the case that wants none of this.
65
+ - **`nexus-ai-pro/store`, long-term memory.** Namespaced tuples, `put`/`get`/`delete`/`search`/
66
+ `listNamespaces`, TTLs, and field filters.
67
+ - Semantic search ranks by any embedding function you inject, so the store never imports the
68
+ embeddings runtime; without an index, a query matches text.
69
+ - `MemoryStore` is bounded, like the graph's memory checkpointer. `RedisStore`
70
+ (`nexus-ai-pro/store/redis`) shares memory across processes through a client-like interface.
71
+ - `compile({ store })` hands it to every node as `context.store`.
72
+ - **`nexus-ai-pro/tracing`, run trees you can query.** Metrics say how it is going; a trace says what
73
+ happened.
74
+ - Every family has a run kind: model, tool, graph, node, agent, retriever, embedding, image, voice,
75
+ realtime, operation. A run carries inputs, outputs, timing, tokens, cost, tags, and errors.
76
+ - `traceGraph()` turns the graph's own event stream into a run tree, so nothing inside the graph
77
+ knows about tracing and an untraced run pays nothing. `traceModelClient()` nests each model call
78
+ inside the node that made it, with its tokens and cost.
79
+ - Tail sampling keeps every error, slow run, and expensive run whatever the head rate is, because
80
+ the decision is made when a trace finishes.
81
+ - Redaction runs before storage: hidden fields are never written, and a `redact` hook gets the last
82
+ word.
83
+ - `MemoryTraceStore` is bounded; `JsonlTraceStore` appends to a file that survives a restart and can
84
+ be read back, pruned, and rotated.
85
+ - Queries filter by kind, status, name, model, tags, metadata, latency, cost, time, and feedback.
86
+ `tree()` assembles a trace, `formatTree()` prints it, and `compareTraces()` reports what changed
87
+ between two runs of the same shape.
88
+ - `recordFeedback()` attaches scores from people or from evaluators.
89
+ - `AlertEvaluator` runs rules over stored runs — error rate, latency percentiles, cost, count — and
90
+ every alert carries the runs that tripped it. `createWebhookNotifier()` posts them.
91
+ - **Bundled agent middleware.** `summarizeHistory` keeps a long transcript from growing without
92
+ bound, `redactMessages` keeps secrets out of provider requests and responses, and `limitToolCalls`
93
+ caps how often a tool may run.
94
+ - **`agentAsTool()`.** Gives one agent to another as a tool, so a supervisor can delegate while
95
+ keeping control. Each agent keeps its own memory, approvals, and thread.
96
+
97
+ - **`nexus-ai-pro/mcp`, both directions of the Model Context Protocol.** Implemented directly, with
98
+ no protocol SDK dependency.
99
+ - `McpClient` lists and calls tools, reads resources and prompts, and `toNexusTools()` turns a
100
+ server's tools into tools an agent can call, with optional name prefixes.
101
+ - `McpServer` exposes this application's tools and resources to other assistants.
102
+ - Stdio and HTTP transports are included, and the transport is an interface, so a client and a
103
+ server can be wired together in memory for a test.
104
+
7
105
  ## [1.12.0] - 2026-09-20
8
106
 
9
107
  Graph control and introspection. A node can route itself, a run can pause at breakpoints and be
@@ -653,7 +751,9 @@ shared policy vocabulary is in place for a future release that revisits this.
653
751
  - URL fetching rejects unsafe private-network targets and limits response reads.
654
752
  - CLI and audit findings no longer disclose detected secret values.
655
753
 
656
- [Unreleased]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.12.0...HEAD
754
+ [Unreleased]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.15.0...HEAD
755
+ [1.15.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.14.0...v1.15.0
756
+ [1.14.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.12.0...v1.14.0
657
757
  [1.12.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.11.0...v1.12.0
658
758
  [1.11.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.10.0...v1.11.0
659
759
  [1.10.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.9.0...v1.10.0
package/README.md CHANGED
@@ -655,6 +655,249 @@ private to the subgraph.
655
655
 
656
656
  The graph is not in the root import. It costs nothing to a user who does not build graphs.
657
657
 
658
+ ## Agents
659
+
660
+ An agent is a graph, so everything above applies to it: checkpoints, human approval, parallel work,
661
+ forks, events, and a diagram.
662
+
663
+ ```ts
664
+ import { createAgent, agentInput, tool } from 'nexus-ai-pro/agent';
665
+
666
+ const agent = createAgent({
667
+ client: ai, // anything with complete(); NexusAI qualifies
668
+ systemPrompt: 'You are a support engineer.',
669
+ tools: [refundTool, emailTool],
670
+ interruptOn: { send_email: true }, // this one waits for a human
671
+ store, // long-term memory, below
672
+ checkpointer, // survives a restart
673
+ });
674
+
675
+ const run = await agent.invoke(agentInput('Refund order 1182 and tell the customer'), { threadId });
676
+ run.state.answer; // the final text
677
+ run.state.stopReason; // 'completed' | 'max_iterations'
678
+ ```
679
+
680
+ **Tool calls run in parallel.** Each call the model requests becomes its own task, bounded by
681
+ `toolConcurrency`, so three lookups take as long as the slowest one.
682
+
683
+ **Approval is an interrupt, not a callback.** A tool listed in `interruptOn` pauses the run and
684
+ checkpoints it. The answer can approve, refuse with a reason the model sees, or approve with
685
+ corrected arguments:
686
+
687
+ ```ts
688
+ if (run.status === 'awaiting_input') {
689
+ await agent.resumeWith(threadId, { approved: true, args: { to: 'billing@example.com' } });
690
+ }
691
+ ```
692
+
693
+ Because it is a checkpoint, the approval can arrive days later, from another process.
694
+
695
+ **Middleware** wraps the parts worth controlling: `beforeModel` adjusts the request, `afterModel`
696
+ inspects or replaces the response, and `wrapToolCall` surrounds each tool call for logging, timing,
697
+ or policy. `AgentLoop` remains for the simple case that needs none of this.
698
+
699
+ ## Long-term Memory
700
+
701
+ A checkpoint remembers one conversation. A store remembers across them:
702
+
703
+ ```ts
704
+ import { MemoryStore } from 'nexus-ai-pro/store';
705
+
706
+ const store = new MemoryStore({ index: { embed, fields: ['text'] } }); // embed is yours
707
+ await store.put(['tenant-7', 'users', 'alice'], 'tone', { text: 'prefers brief answers' });
708
+
709
+ // In any node or tool, in any thread, later:
710
+ const memories = await context.store.search(['tenant-7', 'users', 'alice'], { query: 'how do they like answers?' });
711
+ ```
712
+
713
+ Namespaces are tuples, so `['tenant-7', 'users', 'alice']` is both a place to put something and a
714
+ prefix to search. Items can expire with `ttlMs`, be filtered by field, and be ranked semantically by
715
+ any embedding function you inject — the store never imports the embeddings runtime, and without an
716
+ index a query falls back to matching text. `RedisStore` from `nexus-ai-pro/store/redis` carries the
717
+ same contract across processes through a client-like interface, so no Redis package is a dependency
718
+ here.
719
+
720
+ ## MCP
721
+
722
+ The Model Context Protocol, both directions, implemented directly rather than through an SDK:
723
+
724
+ ```ts
725
+ import { McpClient, createStdioTransport } from 'nexus-ai-pro/mcp';
726
+ import { Tracer, MemoryTraceStore } from 'nexus-ai-pro/tracing';
727
+ import { evaluate, createDataset, compareExperiments } from 'nexus-ai-pro/evaluate';
728
+
729
+ const client = new McpClient(createStdioTransport({ command: 'npx', args: ['-y', 'some-mcp-server'] }));
730
+ const agent = createAgent({ client: ai, tools: await client.toNexusTools({ prefix: 'files' }) });
731
+ ```
732
+
733
+ That is how this package reaches a broad tool ecosystem without maintaining a catalogue of
734
+ integrations: anything exposed over MCP becomes tools an agent can call. The server direction lends
735
+ your tools out the same way, so a tool written once with `tool()` serves an agent here and an editor
736
+ elsewhere:
737
+
738
+ ```ts
739
+ import { McpServer, createStdioServerTransport } from 'nexus-ai-pro/mcp';
740
+
741
+ await new McpServer({ name: 'my-app', tools: [refundTool] }).connect(createStdioServerTransport());
742
+ ```
743
+
744
+ Stdio and HTTP transports are included, and the transport is an interface, so a test can wire a
745
+ client to a server in memory.
746
+
747
+ ## Traces
748
+
749
+ Metrics say how it is going. A trace says what happened: the run tree of one request, with its
750
+ inputs, outputs, tokens, cost, and errors, stored where you can search it.
751
+
752
+ ```ts
753
+ import { Tracer, MemoryTraceStore, traceGraph, traceModelClient } from 'nexus-ai-pro/tracing';
754
+
755
+ const tracer = new Tracer({
756
+ store: new MemoryTraceStore(), // or JsonlTraceStore({ file })
757
+ sampling: { rate: 0.05, keepErrors: true }, // 5% of traces, plus every failure
758
+ redaction: { hideFields: ['apiKey', 'user.email'] },
759
+ });
760
+
761
+ const tracing = traceGraph(tracer, { name: 'support-agent', kind: 'agent' });
762
+ const agent = createAgent({ client: traceModelClient(ai, tracer, { parent: () => tracing.runFor('model') }) });
763
+
764
+ const run = await agent.invoke(agentInput(question), { threadId, ...tracing.runOptions });
765
+ await tracing.finish(run);
766
+ ```
767
+
768
+ **Nothing inside the graph knows about tracing.** The graph already reports every task starting,
769
+ retrying, and finishing; a trace is that stream written down as a tree. An application that does not
770
+ trace creates no context, allocates nothing, and does no per-call work.
771
+
772
+ **Tail sampling keeps what matters.** A 5% rate still records every error, every run slower than a
773
+ threshold, and every run more expensive than one, because the decision is made when the trace
774
+ finishes rather than when it starts.
775
+
776
+ **Redaction happens before storage**, so a field you hide is never written, and a `redact` hook gets
777
+ the last word on every run.
778
+
779
+ **Then ask questions of it:**
780
+
781
+ ```ts
782
+ const failures = await store.query({ status: 'error', kind: 'model', since: yesterday, minLatencyMs: 2000 });
783
+ const tree = await store.tree(failures[0].traceId);
784
+ console.log(formatTree(tree));
785
+
786
+ await tracer.recordFeedback(runId, { key: 'thumbs', score: 0, source: 'user' });
787
+ ```
788
+
789
+ `compareTraces(a, b)` lays two runs of the same shape side by side and reports what changed — the
790
+ step that got slower, the tool that stopped being called, the output that differs — which is how
791
+ "it worked yesterday" becomes an answerable question.
792
+
793
+ **Alerts** run the same queries on a timer, and carry the runs that tripped them:
794
+
795
+ ```ts
796
+ import { AlertEvaluator, createWebhookNotifier } from 'nexus-ai-pro/tracing';
797
+
798
+ const alerts = new AlertEvaluator(
799
+ store,
800
+ [
801
+ { name: 'error rate', metric: 'errorRate', threshold: 0.05, minRuns: 20 },
802
+ { name: 'p95 latency', metric: 'latencyP95', threshold: 8000 },
803
+ { name: 'hourly spend', metric: 'cost', threshold: 25, windowMs: 3_600_000 },
804
+ ],
805
+ { notifier: createWebhookNotifier({ url: process.env.ALERT_WEBHOOK! }) },
806
+ );
807
+
808
+ setInterval(() => void alerts.evaluate(), 60_000);
809
+ ```
810
+
811
+ An alert names the rule, the measured value, and three runs that contributed, so the next step is
812
+ reading them rather than starting an investigation.
813
+
814
+ ## Evaluation
815
+
816
+ One entry point for evaluating anything: a completion, an agent, a graph, an image operation, or
817
+ plain code. A dataset of examples, a target that turns an example into an output, evaluators that
818
+ score it, and a stored experiment a later run can be compared against.
819
+
820
+ ```ts
821
+ import { evaluate, createDataset, exactMatch, contains, trajectory, passRate } from 'nexus-ai-pro/evaluate';
822
+
823
+ const dataset = createDataset({
824
+ name: 'support-questions',
825
+ examples: [
826
+ { id: 'refund', inputs: { question: 'how do I get a refund?' }, expected: 'Open the order and choose refund.' },
827
+ { id: 'hours', inputs: { question: 'when are you open?' }, expected: 'Weekdays, nine to five.' },
828
+ ],
829
+ });
830
+
831
+ const experiment = await evaluate(
832
+ (inputs) => agent.invoke(agentInput(inputs.question)),
833
+ dataset,
834
+ [contains(['refund']), trajectory({ expected: ['search_orders'] })],
835
+ { repetitions: 3, concurrency: 4, summary: [passRate()], store: experiments },
836
+ );
837
+ ```
838
+
839
+ **A dataset is versioned by its content**, so two experiments are comparable only when they really
840
+ ran over the same examples. Change an example and the version changes with it.
841
+
842
+ **Datasets can come from production.** `datasetFromTraces({ store, query: { status: 'error' } })`
843
+ turns recorded runs into examples, each keeping the run it came from — the request that failed
844
+ yesterday becomes tomorrow's regression test.
845
+
846
+ **Evaluators are ordinary functions.** Bundled: `exactMatch`, `contains`, `mustNotMatch`, `completed`,
847
+ `underLatency`, `embeddingSimilarity` through any embedder, `pairwise` for side-by-side judgements,
848
+ and `trajectory`, which scores *how* an answer was reached — an agent that gets the right answer by
849
+ calling the refund tool three times is not working. The LLM judge from `nexus-ai-pro/evals/judge`
850
+ plugs in as one more evaluator.
851
+
852
+ ### Was the change real?
853
+
854
+ ```ts
855
+ import { compareExperiments, formatComparison } from 'nexus-ai-pro/evaluate';
856
+
857
+ const comparison = compareExperiments(baseline, candidate);
858
+ console.log(formatComparison(comparison));
859
+ if (comparison.regressed) process.exit(1);
860
+ ```
861
+
862
+ A mean that moves from 0.81 to 0.83 says nothing on its own; with twenty examples that is noise, and
863
+ shipping on it makes a quality gate a coin toss. The verdict comes from a **paired bootstrap** over
864
+ the per-example differences, which assumes nothing about how the scores are distributed, and the
865
+ pairing removes the variation caused by the examples themselves. A metric whose interval spans zero
866
+ is reported as `unchanged`, not as an improvement. The bootstrap is seeded, so CI gives the same
867
+ verdict twice.
868
+
869
+ The report names the examples that moved most, the failures that are new, and the ones that were
870
+ fixed. A comparison across different dataset versions says so rather than pretending it is like for
871
+ like.
872
+
873
+ ### The runs you did not think of
874
+
875
+ ```ts
876
+ import { evaluateOnline, AnnotationQueue } from 'nexus-ai-pro/evaluate';
877
+
878
+ const queue = new AnnotationQueue({
879
+ rubric: [{ key: 'helpful', prompt: 'Did this answer the question?', type: 'boolean' }],
880
+ consensus: 2,
881
+ });
882
+
883
+ await evaluateOnline({
884
+ store: traceStore,
885
+ evaluators: [mustNotMatch([/i cannot help/i])],
886
+ sampleRate: 0.1,
887
+ reviewQueue: queue,
888
+ reviewWhen: (scores) => scores.some((score) => score.passed === false),
889
+ });
890
+ ```
891
+
892
+ A dataset tells you whether a change works on the cases you thought of; online evaluation tells you
893
+ how it is doing on the rest. Scores are written back as feedback on the run, so an alert rule can
894
+ watch them, and anything uncertain goes to a person.
895
+
896
+ **The review queue keeps track of what people owe you.** Claims expire, so a reviewer who closes the
897
+ tab does not strand an item; `consensus` lets two reviewers see the same item when one opinion is not
898
+ enough; and `toExamples()` turns reviewed items into dataset examples, which closes the loop from a
899
+ production failure to a permanent regression test.
900
+
658
901
  ## Provider Batch Tiers
659
902
 
660
903
  Both OpenAI and Anthropic sell an asynchronous tier at roughly half price, in exchange for a
@@ -1925,6 +2168,7 @@ only, so it never appears in an import graph.
1925
2168
  | `nexus-ai-pro/providers/llamacpp` | 77 KB | 13% | none |
1926
2169
  | `nexus-ai-pro/providers/lmstudio` | 77 KB | 13% | none |
1927
2170
  | `nexus-ai-pro/providers/openai` | 77 KB | 13% | none |
2171
+ | `nexus-ai-pro/agent` | 76 KB | 12% | none |
1928
2172
  | `nexus-ai-pro/providers/anthropic` | 70 KB | 11% | none |
1929
2173
  | `nexus-ai-pro/providers/google` | 66 KB | 11% | none |
1930
2174
  | `nexus-ai-pro/images` | 65 KB | 11% | none |
@@ -1938,8 +2182,10 @@ only, so it never appears in an import graph.
1938
2182
  | `nexus-ai-pro/realtime/openai-webrtc` | 46 KB | 8% | none |
1939
2183
  | `nexus-ai-pro/security` | 40 KB | 7% | +3.4 MB |
1940
2184
  | `nexus-ai-pro/models` | 35 KB | 6% | none |
2185
+ | `nexus-ai-pro/evaluate` | 33 KB | 5% | none |
1941
2186
  | `nexus-ai-pro/images/inputs` | 31 KB | 5% | none |
1942
2187
  | `nexus-ai-pro/realtime/openai-websocket` | 29 KB | 5% | none |
2188
+ | `nexus-ai-pro/tracing` | 26 KB | 4% | none |
1943
2189
  | `nexus-ai-pro/evals` | 23 KB | 4% | none |
1944
2190
  | `nexus-ai-pro/images/transform` | 22 KB | 4% | none |
1945
2191
  | `nexus-ai-pro/images/stores` | 22 KB | 4% | none |
@@ -1948,6 +2194,7 @@ only, so it never appears in an import graph.
1948
2194
  | `nexus-ai-pro/images/openai` | 19 KB | 3% | none |
1949
2195
  | `nexus-ai-pro/realtime/mock` | 19 KB | 3% | none |
1950
2196
  | `nexus-ai-pro/voice` | 18 KB | 3% | none |
2197
+ | `nexus-ai-pro/mcp` | 18 KB | 3% | none |
1951
2198
  | `nexus-ai-pro/images/comfyui` | 17 KB | 3% | none |
1952
2199
  | `nexus-ai-pro/embeddings/adapters` | 17 KB | 3% | none |
1953
2200
  | `nexus-ai-pro/realtime/conversation` | 16 KB | 3% | none |
@@ -1961,6 +2208,7 @@ only, so it never appears in an import graph.
1961
2208
  | `nexus-ai-pro/images/mock` | 12 KB | 2% | none |
1962
2209
  | `nexus-ai-pro/optimizer` | 11 KB | 2% | none |
1963
2210
  | `nexus-ai-pro/voice/session` | 11 KB | 2% | none |
2211
+ | `nexus-ai-pro/store/redis` | 11 KB | 2% | none |
1964
2212
  | `nexus-ai-pro/providers` | 10 KB | 2% | none |
1965
2213
  | `nexus-ai-pro/providers/base` | 10 KB | 2% | none |
1966
2214
  | `nexus-ai-pro/embeddings/models` | 10 KB | 2% | none |
@@ -1971,6 +2219,7 @@ only, so it never appears in an import graph.
1971
2219
  | `nexus-ai-pro/capabilities` | 8 KB | 1% | none |
1972
2220
  | `nexus-ai-pro/telephony/realtime-bridge` | 7 KB | 1% | none |
1973
2221
  | `nexus-ai-pro/graph/visualize` | 6 KB | 1.0% | none |
2222
+ | `nexus-ai-pro/store` | 6 KB | 1.0% | none |
1974
2223
  | `nexus-ai-pro/providers/errors` | 5 KB | 0.8% | none |
1975
2224
  | `nexus-ai-pro/embeddings/mock` | 5 KB | 0.8% | none |
1976
2225
  | `nexus-ai-pro/cache/semantic-cache` | 5 KB | 0.8% | none |
@@ -0,0 +1,119 @@
1
+ import { type CompiledGraph } from '../graph/graph.js';
2
+ import type { GraphCheckpointer, NodeContext } from '../types/graph.js';
3
+ import type { CompletionRequest, Message, ToolDefinition } from '../types/messages.js';
4
+ import type { NexusResponse } from '../types/response.js';
5
+ import type { Store } from '../types/store.js';
6
+ /**
7
+ * An agent as a graph.
8
+ *
9
+ * `AgentLoop` is a `while` loop: it cannot be paused, resumed after a restart, inspected halfway, or
10
+ * made to run two tool calls at once, because a loop has nowhere to keep that state. The same agent
11
+ * expressed as a graph inherits all of it — checkpoints, human approval through `interrupt()`,
12
+ * parallel tool tasks through `Send`, forks, events, and a Mermaid diagram — without this module
13
+ * implementing any of them. `AgentLoop` stays for the simple case that wants none of that.
14
+ */
15
+ /** The model call an agent makes. Injected, so this module needs no provider runtime. */
16
+ export interface AgentModelClient {
17
+ complete(request: CompletionRequest): Promise<NexusResponse>;
18
+ }
19
+ export interface AgentToolCall {
20
+ id: string;
21
+ name: string;
22
+ args: Record<string, unknown>;
23
+ }
24
+ /** What an approver may answer: allow, refuse with a reason, or allow with corrected arguments. */
25
+ export type AgentApproval = boolean | {
26
+ approved: true;
27
+ args?: Record<string, unknown>;
28
+ } | {
29
+ approved: false;
30
+ reason?: string;
31
+ };
32
+ export interface AgentApprovalPolicy {
33
+ /** The question an operator sees. Defaults to naming the tool and its arguments. */
34
+ reason?: (call: AgentToolCall) => string;
35
+ }
36
+ export interface AgentMiddleware {
37
+ name?: string;
38
+ /** Adjusts the request before it is sent: trim history, add context, swap the model. */
39
+ beforeModel?(context: {
40
+ request: CompletionRequest;
41
+ state: AgentState;
42
+ store?: Store;
43
+ }): CompletionRequest | void | Promise<CompletionRequest | void>;
44
+ /** Inspects or replaces the response: redact, validate, count. */
45
+ afterModel?(context: {
46
+ response: NexusResponse;
47
+ state: AgentState;
48
+ store?: Store;
49
+ }): NexusResponse | void | Promise<NexusResponse | void>;
50
+ /** Wraps a tool call, so a policy can log it, time it, or refuse it. */
51
+ wrapToolCall?(call: AgentToolCall, next: () => Promise<AgentToolResult>): Promise<AgentToolResult> | AgentToolResult;
52
+ }
53
+ export interface AgentToolResult {
54
+ ok: boolean;
55
+ result?: unknown;
56
+ error?: string;
57
+ }
58
+ export interface CreateAgentOptions {
59
+ client: AgentModelClient;
60
+ model?: string;
61
+ tools?: ToolDefinition[];
62
+ systemPrompt?: string;
63
+ /** Model calls before the agent stops with `stopReason: 'max_iterations'`. Defaults to 8. */
64
+ maxIterations?: number;
65
+ /** Tool calls run at once. Defaults to 8; `1` runs them one at a time. */
66
+ toolConcurrency?: number;
67
+ /** Tools that need human approval before they run, keyed by tool name. */
68
+ interruptOn?: Record<string, AgentApprovalPolicy | true>;
69
+ middleware?: AgentMiddleware[];
70
+ /** Long-term memory, available to tools and middleware as `store`. */
71
+ store?: Store;
72
+ checkpointer?: GraphCheckpointer | false;
73
+ name?: string;
74
+ temperature?: number;
75
+ maxTokens?: number;
76
+ responseFormat?: CompletionRequest['responseFormat'];
77
+ metadata?: Record<string, unknown>;
78
+ }
79
+ declare const agentChannels: () => {
80
+ messages: import("../types/graph.js").Channel<Message[]>;
81
+ iterations: import("../types/graph.js").Channel<number>;
82
+ answer: import("../types/graph.js").Channel<string>;
83
+ stopReason: import("../types/graph.js").Channel<AgentStopReason | undefined>;
84
+ };
85
+ export type AgentChannels = ReturnType<typeof agentChannels>;
86
+ export type AgentState = {
87
+ messages: Message[];
88
+ iterations: number;
89
+ answer: string;
90
+ stopReason?: AgentStopReason;
91
+ };
92
+ export type AgentStopReason = 'completed' | 'max_iterations';
93
+ export type AgentGraph = CompiledGraph<AgentChannels>;
94
+ /** Wraps a question into the state an agent starts from. */
95
+ export declare function agentInput(goal: string): {
96
+ messages: Message[];
97
+ };
98
+ /**
99
+ * Builds an agent and returns it as a compiled graph.
100
+ *
101
+ * Run it like any graph: `agent.invoke(agentInput('...'), { threadId })`. The result's `answer` is
102
+ * the final text, and `stopReason` says whether the agent finished or ran out of iterations.
103
+ */
104
+ export declare function createAgent(options: CreateAgentOptions): AgentGraph;
105
+ /**
106
+ * Turns an agent into a tool another agent can call.
107
+ *
108
+ * The simplest multi-agent shape: a supervisor keeps control and delegates, rather than handing over
109
+ * the conversation. The specialist runs as its own graph, with its own memory and approvals, and
110
+ * returns its answer as the tool result.
111
+ */
112
+ export declare function agentAsTool(options: {
113
+ agent: AgentGraph;
114
+ name: string;
115
+ description: string;
116
+ /** Threads the specialist's runs, so its work is resumable too. Defaults to a fresh thread. */
117
+ threadId?: (goal: string) => string;
118
+ }): ToolDefinition;
119
+ export type { NodeContext };