nexus-ai-pro 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +48 -0
- package/CHANGELOG.md +111 -2
- package/README.md +149 -20
- package/dist/agent/loop.d.ts +0 -1
- package/dist/agent/loop.js +23 -10
- package/dist/connectors/web.d.ts +3 -13
- package/dist/connectors/web.js +14 -342
- package/dist/graph/checkpointer.d.ts +11 -2
- package/dist/graph/checkpointer.js +22 -8
- package/dist/graph/graph.d.ts +11 -1
- package/dist/graph/graph.js +83 -17
- package/dist/images/codec.d.ts +26 -0
- package/dist/images/codec.js +200 -0
- package/dist/images/comfyui.d.ts +83 -0
- package/dist/images/comfyui.js +330 -0
- package/dist/images/evals.d.ts +146 -0
- package/dist/images/evals.js +299 -0
- package/dist/images/google.d.ts +43 -0
- package/dist/images/google.js +264 -0
- package/dist/images/image-header.d.ts +35 -0
- package/dist/images/image-header.js +128 -0
- package/dist/images/inputs.d.ts +62 -0
- package/dist/images/inputs.js +168 -0
- package/dist/images/manager.d.ts +7 -0
- package/dist/images/manager.js +37 -4
- package/dist/images/moderation.d.ts +46 -0
- package/dist/images/moderation.js +144 -0
- package/dist/images/openai.d.ts +11 -3
- package/dist/images/openai.js +24 -8
- package/dist/images/transform.d.ts +65 -0
- package/dist/images/transform.js +135 -0
- package/dist/testing/image-provider-conformance.d.ts +5 -1
- package/dist/testing/image-provider-conformance.js +42 -3
- package/dist/types/agent.d.ts +6 -0
- package/dist/types/graph.d.ts +16 -3
- package/dist/types/images.d.ts +23 -0
- package/dist/utils/safe-fetch.d.ts +56 -0
- package/dist/utils/safe-fetch.js +375 -0
- package/dist-cjs/agent/loop.d.ts +0 -1
- package/dist-cjs/agent/loop.js +24 -11
- package/dist-cjs/connectors/web.d.ts +3 -13
- package/dist-cjs/connectors/web.js +18 -346
- package/dist-cjs/graph/checkpointer.d.ts +11 -2
- package/dist-cjs/graph/checkpointer.js +22 -8
- package/dist-cjs/graph/graph.d.ts +11 -1
- package/dist-cjs/graph/graph.js +83 -17
- package/dist-cjs/images/codec.d.ts +26 -0
- package/dist-cjs/images/codec.js +207 -0
- package/dist-cjs/images/comfyui.d.ts +83 -0
- package/dist-cjs/images/comfyui.js +369 -0
- package/dist-cjs/images/evals.d.ts +146 -0
- package/dist-cjs/images/evals.js +340 -0
- package/dist-cjs/images/google.d.ts +43 -0
- package/dist-cjs/images/google.js +301 -0
- package/dist-cjs/images/image-header.d.ts +35 -0
- package/dist-cjs/images/image-header.js +141 -0
- package/dist-cjs/images/inputs.d.ts +62 -0
- package/dist-cjs/images/inputs.js +174 -0
- package/dist-cjs/images/manager.d.ts +7 -0
- package/dist-cjs/images/manager.js +37 -4
- package/dist-cjs/images/moderation.d.ts +46 -0
- package/dist-cjs/images/moderation.js +148 -0
- package/dist-cjs/images/openai.d.ts +11 -3
- package/dist-cjs/images/openai.js +57 -8
- package/dist-cjs/images/transform.d.ts +65 -0
- package/dist-cjs/images/transform.js +145 -0
- package/dist-cjs/testing/image-provider-conformance.d.ts +5 -1
- package/dist-cjs/testing/image-provider-conformance.js +42 -3
- package/dist-cjs/types/agent.d.ts +6 -0
- package/dist-cjs/types/graph.d.ts +16 -3
- package/dist-cjs/types/images.d.ts +23 -0
- package/dist-cjs/utils/safe-fetch.d.ts +56 -0
- package/dist-cjs/utils/safe-fetch.js +383 -0
- package/package.json +79 -1
package/API_STABILITY.md
CHANGED
|
@@ -106,6 +106,41 @@ exactly-once provider execution. Those capabilities will use additive adapters a
|
|
|
106
106
|
`MemoryAssetStore` is likewise process-local and does not promise cross-process durability or shared
|
|
107
107
|
tenant state.
|
|
108
108
|
|
|
109
|
+
### Image portability (Unreleased)
|
|
110
|
+
|
|
111
|
+
The following are public under the same 1.x rules. None is exported from the root or from
|
|
112
|
+
`nexus-ai-pro/images`:
|
|
113
|
+
|
|
114
|
+
- `nexus-ai-pro/images/google`
|
|
115
|
+
- `nexus-ai-pro/images/comfyui`
|
|
116
|
+
- `nexus-ai-pro/images/transform`, which re-exports the PNG codec and header helpers
|
|
117
|
+
- `nexus-ai-pro/images/inputs`
|
|
118
|
+
- `nexus-ai-pro/images/moderation`
|
|
119
|
+
- `nexus-ai-pro/images/evals`
|
|
120
|
+
|
|
121
|
+
The following changes are additive:
|
|
122
|
+
|
|
123
|
+
- `ImageConfig.inputResolver` and `ImageConfig.tenantId` are optional. With no resolver, requests are
|
|
124
|
+
handled exactly as before.
|
|
125
|
+
- `ImageAssetResolver` and `ImageAssetResolverContext` are new types.
|
|
126
|
+
- Output from the new modules is still generative. Backend wire shapes such as Imagen `:predict` and
|
|
127
|
+
the ComfyUI queue API are upstream contracts outside this policy, like `raw`.
|
|
128
|
+
|
|
129
|
+
Behaviour that changed:
|
|
130
|
+
|
|
131
|
+
- **OpenAI masks and transparency.** `OpenAIImageProvider` now declares `supportsMask: true` and
|
|
132
|
+
`supportsTransparency: true`, and accepts masked edits. A mask whose size differs from the input is
|
|
133
|
+
still refused unless `resizeMode` allows resampling. `background: 'transparent'` is refused only with
|
|
134
|
+
JPEG output.
|
|
135
|
+
- **Withheld outputs.** A provider finding with `source: 'output'`, `action: 'block'`, and
|
|
136
|
+
`metadata.withheld: true` describes an image the provider removed before responding. The manager
|
|
137
|
+
accepts one fewer asset per such finding, and does not let that finding block the images that were
|
|
138
|
+
returned. Any other blocking finding still blocks the whole result.
|
|
139
|
+
- **Masked-edit conformance case.** `runImageProviderConformance` adds a `masked-image-edit` case for
|
|
140
|
+
providers that declare `supportsMask`. Pass `testMask: false` to skip it.
|
|
141
|
+
- **`UrlPolicyError`.** Safe-fetch policy refusals are now this subclass of `Error`. Their messages are
|
|
142
|
+
unchanged.
|
|
143
|
+
|
|
109
144
|
## Embeddings API stage
|
|
110
145
|
|
|
111
146
|
The `nexus-ai-pro/embeddings` family is public and follows the 1.x rules, including the explicit
|
|
@@ -242,6 +277,19 @@ to Redis is not a checkpoint.
|
|
|
242
277
|
and threading the schema through it would force an annotation on every construction for no benefit.
|
|
243
278
|
The graph casts at that boundary and hands typed state back through `state()` and `history()`.
|
|
244
279
|
|
|
280
|
+
`compile()` without a checkpointer uses an in-process `MemoryGraphCheckpointer` capped at 1,000
|
|
281
|
+
threads (Unreleased; earlier releases created none). `checkpointer: false` disables checkpointing. A
|
|
282
|
+
run started without a `threadId` reports its generated id on the result.
|
|
283
|
+
|
|
284
|
+
Also Unreleased:
|
|
285
|
+
|
|
286
|
+
- A paused or failed superstep records its already finished nodes in the optional `completed` field.
|
|
287
|
+
Those nodes are not run again, and their edges still count when the step completes.
|
|
288
|
+
- Writing checkpoint N drops any stored checkpoints at or after N, so a rewound thread has one
|
|
289
|
+
timeline. Keeping several branches is planned work (forks), not current behaviour.
|
|
290
|
+
- A subgraph interrupt surfaces as an interrupt of the parent node.
|
|
291
|
+
- `onProgress` receives `context.report()` calls.
|
|
292
|
+
|
|
245
293
|
Scheduling within a superstep is not a guarantee. Nodes in a fan-out currently run in edge order and
|
|
246
294
|
sequentially; running them concurrently is a valid future change, so a node must not depend on
|
|
247
295
|
observing another node's write within the same superstep — that is what channels are for. The default
|
package/CHANGELOG.md
CHANGED
|
@@ -4,7 +4,115 @@ Notable changes to this project are recorded here. The format follows [Keep a Ch
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
-
## [
|
|
7
|
+
## [1.10.0] - 2026-09-16
|
|
8
|
+
|
|
9
|
+
Image portability. Three image backends now sit behind one contract: OpenAI, Google Imagen, and a
|
|
10
|
+
self-hosted ComfyUI server. Masked edits work on all three, and every input is validated before any
|
|
11
|
+
provider sees it. Visual moderation inspects both the request and the generated images. Image output
|
|
12
|
+
can be evaluated for alignment, rendered text, preservation, and safety. Each piece is its own subpath,
|
|
13
|
+
and none is loaded by the root import.
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- **Google Imagen adapter** (`nexus-ai-pro/images/google`). It calls the `:predict` protocol on the
|
|
18
|
+
Gemini API with an API key, or on Vertex AI with a bearer token, for generation and masked
|
|
19
|
+
inpainting.
|
|
20
|
+
- It differs from OpenAI, and negotiation says so. Output is sized by aspect ratio, so `dimensions`
|
|
21
|
+
is refused. `seed` and `negativePrompt` are supported.
|
|
22
|
+
- A seed disables the watermark, and the result reports that as a warning.
|
|
23
|
+
- When Imagen filters some images in a batch, the images that passed are still returned.
|
|
24
|
+
- **ComfyUI adapter** (`nexus-ai-pro/images/comfyui`) for a self-hosted server.
|
|
25
|
+
- It uploads inputs, queues a workflow, polls with backoff, and downloads saved outputs.
|
|
26
|
+
- It records the seed it used, even when the caller chose none, so a good run can be reproduced.
|
|
27
|
+
- It removes an abandoned or failed prompt from the queue and interrupts it, so the GPU is not left
|
|
28
|
+
working on a result nobody is waiting for.
|
|
29
|
+
- The workflow graph belongs to the application. Stock text-to-image and inpainting builders are
|
|
30
|
+
included.
|
|
31
|
+
- **Mask conversion** (`nexus-ai-pro/images/transform`). You draw a mask once, with either polarity, and
|
|
32
|
+
it becomes OpenAI's alpha channel or the white-is-editable greyscale that Imagen and ComfyUI read.
|
|
33
|
+
- A size mismatch is refused unless `resizeMode` allows `stretch`, `contain`, or `cover`. Resampling
|
|
34
|
+
is nearest-neighbour, so a binary mask gains no grey fringe, and `contain` padding is never
|
|
35
|
+
editable.
|
|
36
|
+
- Partly transparent pixels count as non-editable, so a soft brush edge never widens an edit.
|
|
37
|
+
- The PNG codec loads on the first masked request. An application that never masks never pays for
|
|
38
|
+
it.
|
|
39
|
+
- `AssetTransformer` is an injection seam for applications that want JPEG or WebP masks.
|
|
40
|
+
- **Validated inputs** (`nexus-ai-pro/images/inputs`). `ImageInputResolver` applies the same checks to
|
|
41
|
+
byte uploads, remote URLs, and stored assets, and plugs into `ImageConfig.inputResolver`.
|
|
42
|
+
- The type is decided by the bytes. A declared or served MIME type that disagrees is refused rather
|
|
43
|
+
than corrected.
|
|
44
|
+
- Byte and pixel limits are enforced. The pixel limit is read from the header before decoding, so a
|
|
45
|
+
small file claiming a huge canvas is refused without allocating anything.
|
|
46
|
+
- Remote fetches get the web connector's SSRF protection: pinned DNS, redirect revalidation, and
|
|
47
|
+
blocking of private networks and cloud metadata.
|
|
48
|
+
- Refusals are `ImageInputError`s with a machine-readable `reason`.
|
|
49
|
+
- **Visual moderation** (`nexus-ai-pro/images/moderation`). `createOpenAIVisualModeration` screens the
|
|
50
|
+
prompt, input and reference images, and every generated image, because an innocuous prompt can
|
|
51
|
+
still produce an unsafe image.
|
|
52
|
+
- Block and review thresholds can be set per category.
|
|
53
|
+
- If the moderation call fails, the request is blocked unless `failOpen` is set.
|
|
54
|
+
- A stored output that cannot be sent for moderation is routed to review.
|
|
55
|
+
- `combineSafetyPolicies` runs several policies and keeps every finding.
|
|
56
|
+
- **Media evaluation** (`nexus-ai-pro/images/evals`). `MediaEvalRunner` checks four things:
|
|
57
|
+
- prompt alignment, using an injected judge;
|
|
58
|
+
- rendered text, using OCR and edit distance;
|
|
59
|
+
- edit preservation, using a perceptual hash that survives re-encoding;
|
|
60
|
+
- blocking behaviour, reported as false-positive and false-negative rates.
|
|
61
|
+
|
|
62
|
+
Every case runs repeatedly and reports mean, spread, and a 95% interval, plus latency, cost, error,
|
|
63
|
+
and failover figures. Scores inside a configured uncertainty band go to a `ReviewQueue` instead of
|
|
64
|
+
being decided automatically.
|
|
65
|
+
- **A masked-edit conformance case.** It runs for every provider that declares `supportsMask`. The
|
|
66
|
+
fixture is a literal PNG, so the testing entry point does not load the codec.
|
|
67
|
+
|
|
68
|
+
### Changed
|
|
69
|
+
|
|
70
|
+
- **OpenAI masked edits and transparency are enabled.** `OpenAIImageProvider` now declares
|
|
71
|
+
`supportsMask` and `supportsTransparency`. `background: 'transparent'` is refused only with JPEG
|
|
72
|
+
output.
|
|
73
|
+
- **A provider can explain missing images.** A blocking output finding marked `metadata.withheld`
|
|
74
|
+
counts toward the requested image count, and it does not block the images that were returned. Any
|
|
75
|
+
other blocking finding still blocks the whole result.
|
|
76
|
+
- **SSRF protection is shared.** It moved out of the web connector into one module that the connector
|
|
77
|
+
and image inputs both use. The web connector behaves the same, and its security tests pass
|
|
78
|
+
unchanged. Policy refusals are now a `UrlPolicyError`, with the same messages.
|
|
79
|
+
- **Graphs checkpoint by default, as the documentation always said.** `compile()` now uses an
|
|
80
|
+
in-process `MemoryGraphCheckpointer` when none is given, so `interrupt()`, `state()`, and
|
|
81
|
+
`history()` work without setup. The default keeps at most 1,000 threads, dropping the least recently
|
|
82
|
+
written one, so a service that never resumes cannot grow without limit. `maxThreads` is configurable,
|
|
83
|
+
and `checkpointer: false` turns checkpointing off. Before this fix, `compile()` created no
|
|
84
|
+
checkpointer and `interrupt()` threw.
|
|
85
|
+
- **The whole-package size limit is higher.** The new modules are about 200 KB unpacked across both
|
|
86
|
+
builds, so the tarball limit rose from 460 KB to 500 KB packed and from 2.85 MB to 3.1 MB unpacked.
|
|
87
|
+
No existing import got heavier because of them. Each new module has its own per-subpath budget, and
|
|
88
|
+
the root import loads none of them.
|
|
89
|
+
|
|
90
|
+
### Fixed
|
|
91
|
+
|
|
92
|
+
- **A failed graph step no longer re-runs the siblings that finished.** The failed checkpoint used to
|
|
93
|
+
discard their writes, so `continue()` ran them again and repeated their side effects. It now keeps
|
|
94
|
+
those writes, and `continue()` retries only the nodes that did not finish.
|
|
95
|
+
- **Finished siblings still route onward after a paused or failed step resumes.** The checkpoint
|
|
96
|
+
carried only the unfinished nodes, so the outgoing edges of siblings that had already finished were
|
|
97
|
+
lost. A new optional `completed` field on the checkpoint records those siblings.
|
|
98
|
+
- **A subgraph that asks a question now pauses its parent.** `asNode()` used to treat an interrupted
|
|
99
|
+
subgraph as finished: the parent merged its partial state and carried on. The parent now interrupts
|
|
100
|
+
with the same question, and resuming the parent passes the answer into the subgraph, which continues
|
|
101
|
+
where it stopped. A subgraph that asks several questions works across several resumes.
|
|
102
|
+
- **Rewinding no longer leaves checkpoints from the abandoned timeline.** After `resumeFrom(step)`,
|
|
103
|
+
both checkpointers kept the later steps, so `state()` could return a step that no longer existed.
|
|
104
|
+
Writing a step now drops any stored steps at or after it.
|
|
105
|
+
- **`context.report()` works.** It did nothing. Progress now reaches the new
|
|
106
|
+
`GraphRunOptions.onProgress` callback, and a callback that throws does not fail the node.
|
|
107
|
+
- **`CompileOptions.name` is used.** It was documented but never read. It is now recorded on every
|
|
108
|
+
checkpoint as `metadata.graph`.
|
|
109
|
+
- **An unnamed graph run now reports its thread id.** `invoke()` without a `threadId` returned
|
|
110
|
+
`threadId: ''` even though it had generated one, so the run could not be inspected or resumed.
|
|
111
|
+
- **`AgentLoop` no longer runs a tool with arguments the model did not send.** Malformed tool-call
|
|
112
|
+
JSON, or arguments that are not an object, used to become `{}` and the tool ran anyway. The tool is
|
|
113
|
+
now skipped, and the parse error goes back to the model as the tool result.
|
|
114
|
+
- **`AgentLoop` says why it stopped.** `AgentResult.stopReason` is `completed` or `max_iterations`, so
|
|
115
|
+
a caller can tell a final answer from a run that hit the iteration limit.
|
|
8
116
|
|
|
9
117
|
## [1.9.0] - 2026-09-15
|
|
10
118
|
|
|
@@ -457,7 +565,8 @@ shared policy vocabulary is in place for a future release that revisits this.
|
|
|
457
565
|
- URL fetching rejects unsafe private-network targets and limits response reads.
|
|
458
566
|
- CLI and audit findings no longer disclose detected secret values.
|
|
459
567
|
|
|
460
|
-
[Unreleased]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.
|
|
568
|
+
[Unreleased]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.10.0...HEAD
|
|
569
|
+
[1.10.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.9.0...v1.10.0
|
|
461
570
|
[1.9.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.8.0...v1.9.0
|
|
462
571
|
[1.8.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.7.0...v1.8.0
|
|
463
572
|
[1.7.0]: https://github.com/mkhitar-abrahamyan/nexus-ai/compare/v1.6.0...v1.7.0
|
package/README.md
CHANGED
|
@@ -921,9 +921,123 @@ console.log(image.assets[0]?.location);
|
|
|
921
921
|
|
|
922
922
|
`ImageProvider` is independent from completion providers. Explicit options are negotiated against the
|
|
923
923
|
selected provider's declared capabilities, so unsupported formats, delivery kinds, masks, seeds, or
|
|
924
|
-
dimensions fail before a provider call
|
|
925
|
-
|
|
926
|
-
|
|
924
|
+
dimensions fail before a provider call rather than being silently dropped.
|
|
925
|
+
|
|
926
|
+
### Three backends, one contract
|
|
927
|
+
|
|
928
|
+
The same request runs against OpenAI, Google Imagen, or a self-hosted ComfyUI server. Each adapter is
|
|
929
|
+
its own subpath, so an application loads only the backends it registers.
|
|
930
|
+
|
|
931
|
+
```ts
|
|
932
|
+
import { ComfyUIImageProvider } from 'nexus-ai-pro/images/comfyui';
|
|
933
|
+
import { GoogleImageProvider } from 'nexus-ai-pro/images/google';
|
|
934
|
+
|
|
935
|
+
const portable = new NexusAI({
|
|
936
|
+
providers: {},
|
|
937
|
+
images: {
|
|
938
|
+
defaultProvider: 'google',
|
|
939
|
+
providers: {
|
|
940
|
+
openai: openaiImages,
|
|
941
|
+
google: new GoogleImageProvider({ apiKey: process.env.GEMINI_API_KEY! }),
|
|
942
|
+
local: new ComfyUIImageProvider({ baseUrl: 'http://127.0.0.1:8188' }),
|
|
943
|
+
},
|
|
944
|
+
},
|
|
945
|
+
});
|
|
946
|
+
```
|
|
947
|
+
|
|
948
|
+
The backends really do differ, and negotiation says so instead of hiding it. Imagen sizes output by
|
|
949
|
+
aspect ratio and refuses `dimensions`. It honours `seed` and `negativePrompt`, which OpenAI refuses, and
|
|
950
|
+
setting a seed turns its watermark off, which the result reports as a warning. When Imagen filters some
|
|
951
|
+
images in a batch, you get the ones that passed plus a withheld finding for each missing one. ComfyUI
|
|
952
|
+
queues work and polls for it, records the seed it used so any run can be reproduced, and removes an
|
|
953
|
+
abandoned prompt from the server queue. Its graph is yours: pass a `workflow` builder, or use the bundled
|
|
954
|
+
`comfyTextToImageWorkflow` and `comfyInpaintWorkflow`.
|
|
955
|
+
|
|
956
|
+
### Masked edits
|
|
957
|
+
|
|
958
|
+
Draw the mask once, with either polarity. Each adapter converts it to what its backend expects:
|
|
959
|
+
OpenAI's alpha channel, or Imagen's and ComfyUI's white-is-editable greyscale.
|
|
960
|
+
|
|
961
|
+
```ts
|
|
962
|
+
const edited = await portable.images.edit({
|
|
963
|
+
provider: 'openai',
|
|
964
|
+
prompt: 'Replace the sky with a sunset',
|
|
965
|
+
input: { location: { kind: 'bytes', data: photo }, mimeType: 'image/png' },
|
|
966
|
+
mask: {
|
|
967
|
+
location: { kind: 'bytes', data: maskPng },
|
|
968
|
+
mimeType: 'image/png',
|
|
969
|
+
polarity: 'white-is-editable',
|
|
970
|
+
resizeMode: 'reject', // or 'stretch' | 'contain' | 'cover'
|
|
971
|
+
},
|
|
972
|
+
});
|
|
973
|
+
```
|
|
974
|
+
|
|
975
|
+
A mask whose size differs from the image is refused unless `resizeMode` allows resampling.
|
|
976
|
+
Partly transparent mask pixels count as non-editable, so a soft brush edge never widens the edit.
|
|
977
|
+
The PNG codec behind the bundled `PngMaskTransformer` loads on the first masked request, so an
|
|
978
|
+
application that never masks never loads it. To accept JPEG or WebP masks, pass your own
|
|
979
|
+
`maskTransformer`.
|
|
980
|
+
|
|
981
|
+
### Validating inputs
|
|
982
|
+
|
|
983
|
+
Byte uploads, remote URLs, and stored assets all go through the same checks before any provider sees
|
|
984
|
+
them:
|
|
985
|
+
|
|
986
|
+
```ts
|
|
987
|
+
import { createImageInputResolver } from 'nexus-ai-pro/images/inputs';
|
|
988
|
+
|
|
989
|
+
const guarded = new NexusAI({
|
|
990
|
+
providers: {},
|
|
991
|
+
images: {
|
|
992
|
+
providers: { google: googleImages },
|
|
993
|
+
inputResolver: createImageInputResolver({
|
|
994
|
+
maxBytes: 10 * 1024 * 1024,
|
|
995
|
+
maxPixels: 16_000_000,
|
|
996
|
+
allowedDomains: ['cdn.example.com'],
|
|
997
|
+
}),
|
|
998
|
+
},
|
|
999
|
+
});
|
|
1000
|
+
```
|
|
1001
|
+
|
|
1002
|
+
The file's own bytes decide its type. A declared or served MIME type that disagrees is refused. Pixel
|
|
1003
|
+
limits are checked from the header before decoding, so a small file that claims a huge canvas costs
|
|
1004
|
+
nothing. Remote URLs use the same SSRF protection as the web connector: pinned DNS, redirect
|
|
1005
|
+
revalidation, and blocking of private networks and cloud metadata endpoints. A refusal is an
|
|
1006
|
+
`ImageInputError` whose `reason` is machine-readable (`too-large`, `mime-mismatch`, `blocked-url`, and
|
|
1007
|
+
so on). Without a resolver, nothing runs and nothing is loaded.
|
|
1008
|
+
|
|
1009
|
+
### Visual moderation
|
|
1010
|
+
|
|
1011
|
+
```ts
|
|
1012
|
+
import { combineSafetyPolicies, createOpenAIVisualModeration } from 'nexus-ai-pro/images/moderation';
|
|
1013
|
+
|
|
1014
|
+
const safety = combineSafetyPolicies(
|
|
1015
|
+
createOpenAIVisualModeration({ apiKey: process.env.OPENAI_API_KEY!, reviewThreshold: 0.4 }),
|
|
1016
|
+
myBrandPolicy,
|
|
1017
|
+
);
|
|
1018
|
+
```
|
|
1019
|
+
|
|
1020
|
+
Visual moderation screens the prompt, the input and reference images, and every generated image. An
|
|
1021
|
+
innocuous prompt can still produce an unsafe image, so text-only checks are not enough. Scores map to
|
|
1022
|
+
`block` or `review` findings, with per-category thresholds. If the moderation call itself fails, the
|
|
1023
|
+
request is blocked unless you set `failOpen`.
|
|
1024
|
+
|
|
1025
|
+
### Evaluating media
|
|
1026
|
+
|
|
1027
|
+
`MediaEvalRunner` from `nexus-ai-pro/images/evals` scores what a string golden file cannot:
|
|
1028
|
+
|
|
1029
|
+
- prompt alignment, scored by your judge;
|
|
1030
|
+
- rendered text, through your OCR function and edit distance;
|
|
1031
|
+
- whether an edit preserved the rest of the image, through a perceptual hash that survives re-encoding;
|
|
1032
|
+
- whether content was blocked when it should have been, reported as false-positive and false-negative
|
|
1033
|
+
rates.
|
|
1034
|
+
|
|
1035
|
+
Each case runs several times and reports mean, spread, and a 95% interval. Scores inside a configured
|
|
1036
|
+
uncertainty band go to a `ReviewQueue` for a person to judge instead of being decided automatically.
|
|
1037
|
+
|
|
1038
|
+
Image support is still marked experimental. The adapters are verified against recorded wire shapes
|
|
1039
|
+
and a shared conformance suite, which now includes a masked-edit case. The label comes off after the
|
|
1040
|
+
opt-in live conformance suite passes against each hosted backend.
|
|
927
1041
|
|
|
928
1042
|
Use `submit()` for a cancellable local operation handle with replayable lifecycle events:
|
|
929
1043
|
|
|
@@ -1615,6 +1729,12 @@ import { ImageManager } from 'nexus-ai-pro/images';
|
|
|
1615
1729
|
import { MemoryAssetStore } from 'nexus-ai-pro/images/assets';
|
|
1616
1730
|
import { MockImageProvider } from 'nexus-ai-pro/images/mock';
|
|
1617
1731
|
import { OpenAIImageProvider } from 'nexus-ai-pro/images/openai';
|
|
1732
|
+
import { GoogleImageProvider } from 'nexus-ai-pro/images/google';
|
|
1733
|
+
import { ComfyUIImageProvider } from 'nexus-ai-pro/images/comfyui';
|
|
1734
|
+
import { PngMaskTransformer } from 'nexus-ai-pro/images/transform';
|
|
1735
|
+
import { createImageInputResolver } from 'nexus-ai-pro/images/inputs';
|
|
1736
|
+
import { createOpenAIVisualModeration } from 'nexus-ai-pro/images/moderation';
|
|
1737
|
+
import { MediaEvalRunner } from 'nexus-ai-pro/images/evals';
|
|
1618
1738
|
import { createRealtimeSession } from 'nexus-ai-pro/realtime/session';
|
|
1619
1739
|
import { OpenAIWebRTCTransport } from 'nexus-ai-pro/realtime/openai-webrtc';
|
|
1620
1740
|
import { TelephonyManager } from 'nexus-ai-pro/telephony';
|
|
@@ -1633,12 +1753,12 @@ past its budget — so the numbers stay true rather than aspirational.
|
|
|
1633
1753
|
<!-- size-table:start -->
|
|
1634
1754
|
| Import | Size | Share of root |
|
|
1635
1755
|
| --- | --- | --- |
|
|
1636
|
-
| `nexus-ai-pro` |
|
|
1637
|
-
| `nexus-ai-pro/config` |
|
|
1638
|
-
| `nexus-ai-pro/core` |
|
|
1756
|
+
| `nexus-ai-pro` | 612 KB | 100% |
|
|
1757
|
+
| `nexus-ai-pro/config` | 468 KB | 76% |
|
|
1758
|
+
| `nexus-ai-pro/core` | 462 KB | 75% |
|
|
1639
1759
|
| `nexus-ai-pro/realtime` | 157 KB | 26% |
|
|
1640
1760
|
| `nexus-ai-pro/batch` | 118 KB | 19% |
|
|
1641
|
-
| `nexus-ai-pro/realtime/session` | 94 KB |
|
|
1761
|
+
| `nexus-ai-pro/realtime/session` | 94 KB | 15% |
|
|
1642
1762
|
| `nexus-ai-pro/embeddings` | 91 KB | 15% |
|
|
1643
1763
|
| `nexus-ai-pro/providers/groq` | 79 KB | 13% |
|
|
1644
1764
|
| `nexus-ai-pro/providers/mistral` | 79 KB | 13% |
|
|
@@ -1648,29 +1768,34 @@ past its budget — so the numbers stay true rather than aspirational.
|
|
|
1648
1768
|
| `nexus-ai-pro/providers/llamacpp` | 77 KB | 13% |
|
|
1649
1769
|
| `nexus-ai-pro/providers/lmstudio` | 77 KB | 13% |
|
|
1650
1770
|
| `nexus-ai-pro/providers/openai` | 77 KB | 13% |
|
|
1651
|
-
| `nexus-ai-pro/providers/anthropic` | 70 KB |
|
|
1771
|
+
| `nexus-ai-pro/providers/anthropic` | 70 KB | 11% |
|
|
1652
1772
|
| `nexus-ai-pro/providers/google` | 66 KB | 11% |
|
|
1653
|
-
| `nexus-ai-pro/images` |
|
|
1654
|
-
| `nexus-ai-pro/providers/ollama` | 58 KB |
|
|
1655
|
-
| `nexus-ai-pro/batch/openai` | 52 KB |
|
|
1656
|
-
| `nexus-ai-pro/batch/anthropic` | 52 KB |
|
|
1773
|
+
| `nexus-ai-pro/images` | 65 KB | 11% |
|
|
1774
|
+
| `nexus-ai-pro/providers/ollama` | 58 KB | 9% |
|
|
1775
|
+
| `nexus-ai-pro/batch/openai` | 52 KB | 8% |
|
|
1776
|
+
| `nexus-ai-pro/batch/anthropic` | 52 KB | 8% |
|
|
1657
1777
|
| `nexus-ai-pro/operations` | 49 KB | 8% |
|
|
1658
1778
|
| `nexus-ai-pro/batch/mock` | 47 KB | 8% |
|
|
1659
1779
|
| `nexus-ai-pro/providers/cohere` | 46 KB | 8% |
|
|
1660
1780
|
| `nexus-ai-pro/realtime/openai-webrtc` | 46 KB | 8% |
|
|
1661
1781
|
| `nexus-ai-pro/security` | 40 KB | 7% |
|
|
1662
1782
|
| `nexus-ai-pro/models` | 35 KB | 6% |
|
|
1663
|
-
| `nexus-ai-pro/graph` |
|
|
1783
|
+
| `nexus-ai-pro/graph` | 33 KB | 5% |
|
|
1784
|
+
| `nexus-ai-pro/images/inputs` | 31 KB | 5% |
|
|
1664
1785
|
| `nexus-ai-pro/realtime/openai-websocket` | 29 KB | 5% |
|
|
1665
1786
|
| `nexus-ai-pro/evals` | 23 KB | 4% |
|
|
1787
|
+
| `nexus-ai-pro/images/transform` | 22 KB | 4% |
|
|
1666
1788
|
| `nexus-ai-pro/images/stores` | 22 KB | 4% |
|
|
1667
1789
|
| `nexus-ai-pro/telephony/twilio` | 21 KB | 3% |
|
|
1668
1790
|
| `nexus-ai-pro/telephony` | 20 KB | 3% |
|
|
1791
|
+
| `nexus-ai-pro/images/openai` | 19 KB | 3% |
|
|
1669
1792
|
| `nexus-ai-pro/realtime/mock` | 19 KB | 3% |
|
|
1670
1793
|
| `nexus-ai-pro/voice` | 18 KB | 3% |
|
|
1671
|
-
| `nexus-ai-pro/images/
|
|
1794
|
+
| `nexus-ai-pro/images/comfyui` | 17 KB | 3% |
|
|
1672
1795
|
| `nexus-ai-pro/embeddings/adapters` | 17 KB | 3% |
|
|
1673
1796
|
| `nexus-ai-pro/realtime/conversation` | 16 KB | 3% |
|
|
1797
|
+
| `nexus-ai-pro/images/google` | 15 KB | 2% |
|
|
1798
|
+
| `nexus-ai-pro/images/evals` | 15 KB | 2% |
|
|
1674
1799
|
| `nexus-ai-pro/images/assets` | 14 KB | 2% |
|
|
1675
1800
|
| `nexus-ai-pro/context` | 13 KB | 2% |
|
|
1676
1801
|
| `nexus-ai-pro/operations/adapters` | 13 KB | 2% |
|
|
@@ -1683,6 +1808,7 @@ past its budget — so the numbers stay true rather than aspirational.
|
|
|
1683
1808
|
| `nexus-ai-pro/providers/base` | 10 KB | 2% |
|
|
1684
1809
|
| `nexus-ai-pro/embeddings/models` | 10 KB | 2% |
|
|
1685
1810
|
| `nexus-ai-pro/voice/openai` | 9 KB | 1% |
|
|
1811
|
+
| `nexus-ai-pro/images/moderation` | 9 KB | 1% |
|
|
1686
1812
|
| `nexus-ai-pro/ops/circuit-breaker` | 8 KB | 1% |
|
|
1687
1813
|
| `nexus-ai-pro/realtime/openai-server` | 8 KB | 1% |
|
|
1688
1814
|
| `nexus-ai-pro/capabilities` | 8 KB | 1% |
|
|
@@ -1760,13 +1886,16 @@ See [ROADMAP.md](https://github.com/mkhitar-abrahamyan/nexus-ai/blob/main/ROADMA
|
|
|
1760
1886
|
shipped and what is planned. It is a design proposal, not a compatibility promise; the guarantees
|
|
1761
1887
|
live in [API_STABILITY.md](./API_STABILITY.md).
|
|
1762
1888
|
|
|
1763
|
-
|
|
1764
|
-
|
|
1889
|
+
Shipped so far:
|
|
1890
|
+
|
|
1891
|
+
- completion controls and capability negotiation;
|
|
1892
|
+
- durable operations, provider batch tiers, and distributed resilience;
|
|
1893
|
+
- first-class embeddings and graphs;
|
|
1894
|
+
- per-entry-point size budgets.
|
|
1765
1895
|
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
[ROADMAP.md](https://github.com/mkhitar-abrahamyan/nexus-ai/blob/main/ROADMAP.md).
|
|
1896
|
+
The current work is image portability: three backends behind one contract, masked edits, validated
|
|
1897
|
+
inputs, visual moderation, and media evaluation. Promotion out of experimental waits on live
|
|
1898
|
+
conformance.
|
|
1770
1899
|
|
|
1771
1900
|
## Known Limitations
|
|
1772
1901
|
|
package/dist/agent/loop.d.ts
CHANGED
package/dist/agent/loop.js
CHANGED
|
@@ -41,13 +41,19 @@ export class AgentLoop {
|
|
|
41
41
|
content: response.content,
|
|
42
42
|
steps,
|
|
43
43
|
iterations: iteration,
|
|
44
|
+
stopReason: 'completed',
|
|
44
45
|
response,
|
|
45
46
|
messages,
|
|
46
47
|
};
|
|
47
48
|
}
|
|
48
49
|
for (const toolCall of response.toolCalls) {
|
|
49
|
-
const
|
|
50
|
-
|
|
50
|
+
const parsed = parseToolArgs(toolCall.function.arguments);
|
|
51
|
+
// Malformed arguments are reported back to the model rather than guessed at: running a tool
|
|
52
|
+
// with empty arguments could perform an action the model never asked for.
|
|
53
|
+
const result = parsed.ok
|
|
54
|
+
? await executor.execute(toolCall.function.name, parsed.args)
|
|
55
|
+
: { ok: false, error: parsed.error };
|
|
56
|
+
const args = parsed.ok ? parsed.args : {};
|
|
51
57
|
const toolStep = {
|
|
52
58
|
iteration,
|
|
53
59
|
type: 'tool',
|
|
@@ -73,17 +79,24 @@ export class AgentLoop {
|
|
|
73
79
|
content: lastResponse.content,
|
|
74
80
|
steps,
|
|
75
81
|
iterations: maxIterations,
|
|
82
|
+
stopReason: 'max_iterations',
|
|
76
83
|
response: lastResponse,
|
|
77
84
|
messages,
|
|
78
85
|
};
|
|
79
86
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
87
|
+
}
|
|
88
|
+
function parseToolArgs(raw) {
|
|
89
|
+
// An absent argument string is a call with no arguments, which is legitimate.
|
|
90
|
+
if (!raw?.trim())
|
|
91
|
+
return { ok: true, args: {} };
|
|
92
|
+
try {
|
|
93
|
+
const parsed = JSON.parse(raw);
|
|
94
|
+
if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed))
|
|
95
|
+
return { ok: true, args: parsed };
|
|
96
|
+
return { ok: false, error: 'Tool arguments must be a JSON object; the tool was not run' };
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
100
|
+
return { ok: false, error: `Tool arguments are not valid JSON (${reason}); the tool was not run` };
|
|
88
101
|
}
|
|
89
102
|
}
|
package/dist/connectors/web.d.ts
CHANGED
|
@@ -1,20 +1,10 @@
|
|
|
1
1
|
import type { ToolDefinition } from '../types/messages.js';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
};
|
|
6
|
-
export interface WebConnectorOptions {
|
|
7
|
-
allowedDomains?: string[];
|
|
2
|
+
import { type SafeFetchPolicy, type WebResolvedAddress } from '../utils/safe-fetch.js';
|
|
3
|
+
export type { WebResolvedAddress };
|
|
4
|
+
export interface WebConnectorOptions extends SafeFetchPolicy {
|
|
8
5
|
timeoutMs?: number;
|
|
9
|
-
userAgent?: string;
|
|
10
6
|
maxResponseBytes?: number;
|
|
11
7
|
maxRedirects?: number;
|
|
12
|
-
/** Explicitly allow loopback/private targets, for example a trusted local development service. */
|
|
13
|
-
allowPrivateNetworks?: boolean;
|
|
14
|
-
/** Cloud metadata endpoints remain denied unless this separate high-risk opt-in is enabled. */
|
|
15
|
-
allowCloudMetadata?: boolean;
|
|
16
|
-
/** Optional resolver for split-horizon DNS or deterministic tests. Every returned address is validated. */
|
|
17
|
-
resolveHostname?: (hostname: string) => Promise<readonly WebResolvedAddress[]>;
|
|
18
8
|
}
|
|
19
9
|
export declare function createFetchUrlTool(options?: WebConnectorOptions): ToolDefinition;
|
|
20
10
|
export declare function createSearchTool(search: (query: string) => Promise<unknown>, name?: string): ToolDefinition;
|