@juspay/neurolink 12.9.6 → 12.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -3
- package/dist/artifacts/artifactBanking.d.ts +6 -2
- package/dist/artifacts/artifactBanking.js +9 -14
- package/dist/artifacts/artifactReader.d.ts +69 -0
- package/dist/artifacts/artifactReader.js +157 -0
- package/dist/artifacts/artifactStore.d.ts +7 -3
- package/dist/artifacts/artifactStore.js +10 -21
- package/dist/artifacts/artifactStoreFactory.d.ts +39 -0
- package/dist/artifacts/artifactStoreFactory.js +101 -0
- package/dist/artifacts/redisArtifactStore.d.ts +84 -0
- package/dist/artifacts/redisArtifactStore.js +270 -0
- package/dist/browser/neurolink.min.js +367 -371
- package/dist/index.d.ts +3 -0
- package/dist/index.js +7 -0
- package/dist/memory/memoryRetrievalTools.js +104 -35
- package/dist/neurolink.d.ts +45 -1
- package/dist/neurolink.js +128 -18
- package/dist/types/artifact.d.ts +124 -3
- package/dist/types/config.d.ts +7 -0
- package/dist/utils/redis.d.ts +15 -0
- package/dist/utils/redis.js +64 -6
- package/package.json +5 -5
package/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
## [12.
|
|
1
|
+
## [12.10.0](https://github.com/juspay/neurolink/compare/v12.9.6...v12.10.0) (2026-09-03)
|
|
2
2
|
|
|
3
|
-
###
|
|
3
|
+
### Features
|
|
4
4
|
|
|
5
|
-
- **(
|
|
5
|
+
- **(artifacts):** Redis and custom stores, range reads, artifact search ([cf59810](https://github.com/juspay/neurolink/commit/cf59810092343b6fadfdcc1557824929d6b2c4e0))
|
|
6
6
|
|
|
7
7
|
## [11.2.3](https://github.com/juspay/neurolink/compare/v11.2.2...v11.2.3) (2026-08-19)
|
|
8
8
|
|
|
@@ -13,8 +13,9 @@
|
|
|
13
13
|
* Read-back is the tool that already exists: `retrieve_context({ artifactId,
|
|
14
14
|
* offset, limit })` paginates any artifact and reports `totalSize` / `hasMore`.
|
|
15
15
|
* There is no second read tool and no second storage layer — this module is a
|
|
16
|
-
* thin, typed front door onto
|
|
17
|
-
* output normalizer externalizes
|
|
16
|
+
* thin, typed front door onto the instance's artifact store (local temp,
|
|
17
|
+
* Redis, or injected), the same store the MCP output normalizer externalizes
|
|
18
|
+
* into.
|
|
18
19
|
*
|
|
19
20
|
* The one thing it adds is that the store no longer has to pre-exist: it is
|
|
20
21
|
* created on first use, whether or not `mcp.outputLimits` was ever configured,
|
|
@@ -52,6 +53,9 @@ export declare function bankArtifact(host: NeuroLink, payload: string, options:
|
|
|
52
53
|
* because a host that asks for the artifact is asking for the artifact.
|
|
53
54
|
* Returns null when the id is unknown or the file is gone.
|
|
54
55
|
*
|
|
56
|
+
* With a `page`, a backend that supports range reads moves only the window
|
|
57
|
+
* (see `readArtifactWindow`) — the same path `retrieve_context` takes.
|
|
58
|
+
*
|
|
55
59
|
* @param page Character window. `offset` defaults to 0, `limit` to the rest.
|
|
56
60
|
*/
|
|
57
61
|
export declare function readArtifact(host: NeuroLink, id: string, page?: ArtifactPageRequest): Promise<string | null>;
|
|
@@ -13,8 +13,9 @@
|
|
|
13
13
|
* Read-back is the tool that already exists: `retrieve_context({ artifactId,
|
|
14
14
|
* offset, limit })` paginates any artifact and reports `totalSize` / `hasMore`.
|
|
15
15
|
* There is no second read tool and no second storage layer — this module is a
|
|
16
|
-
* thin, typed front door onto
|
|
17
|
-
* output normalizer externalizes
|
|
16
|
+
* thin, typed front door onto the instance's artifact store (local temp,
|
|
17
|
+
* Redis, or injected), the same store the MCP output normalizer externalizes
|
|
18
|
+
* into.
|
|
18
19
|
*
|
|
19
20
|
* The one thing it adds is that the store no longer has to pre-exist: it is
|
|
20
21
|
* created on first use, whether or not `mcp.outputLimits` was ever configured,
|
|
@@ -22,6 +23,7 @@
|
|
|
22
23
|
*
|
|
23
24
|
* @module artifacts/artifactBanking
|
|
24
25
|
*/
|
|
26
|
+
import { readArtifactWindow } from "./artifactReader.js";
|
|
25
27
|
/** Preview length when the caller does not ask for one. */
|
|
26
28
|
const DEFAULT_BANK_PREVIEW_CHARS = 1000;
|
|
27
29
|
/**
|
|
@@ -104,20 +106,13 @@ export async function bankArtifact(host, payload, options) {
|
|
|
104
106
|
* because a host that asks for the artifact is asking for the artifact.
|
|
105
107
|
* Returns null when the id is unknown or the file is gone.
|
|
106
108
|
*
|
|
109
|
+
* With a `page`, a backend that supports range reads moves only the window
|
|
110
|
+
* (see `readArtifactWindow`) — the same path `retrieve_context` takes.
|
|
111
|
+
*
|
|
107
112
|
* @param page Character window. `offset` defaults to 0, `limit` to the rest.
|
|
108
113
|
*/
|
|
109
114
|
export async function readArtifact(host, id, page) {
|
|
110
115
|
const store = ensureArtifactStore(host);
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
return null;
|
|
114
|
-
}
|
|
115
|
-
if (!page) {
|
|
116
|
-
return content;
|
|
117
|
-
}
|
|
118
|
-
const offset = Math.max(0, page.offset ?? 0);
|
|
119
|
-
if (page.limit === undefined) {
|
|
120
|
-
return content.slice(offset);
|
|
121
|
-
}
|
|
122
|
-
return content.slice(offset, offset + Math.max(0, page.limit));
|
|
116
|
+
const window = await readArtifactWindow(store, id, page);
|
|
117
|
+
return window === null ? null : window.content;
|
|
123
118
|
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact reading primitives shared by every backend and every reader.
|
|
3
|
+
*
|
|
4
|
+
* Two readers exist — the model's `retrieve_context({ artifactId })` and the
|
|
5
|
+
* host's `readArtifact()` — and two shipped backends. Everything they agree
|
|
6
|
+
* on lives here so it is agreed on exactly once:
|
|
7
|
+
*
|
|
8
|
+
* - `readArtifactWindow` is THE paged read. It asks a backend for a window
|
|
9
|
+
* when the backend can produce one (`retrieveRange`), and otherwise reads
|
|
10
|
+
* the whole payload and slices. A reader never has to know which.
|
|
11
|
+
* - `searchArtifactContent` is the literal search over an artifact. It
|
|
12
|
+
* returns bounded snippets with character offsets, never whole lines:
|
|
13
|
+
* an MCP artifact is usually one compact JSON line, so "the matching line"
|
|
14
|
+
* would be the payload the search was meant to avoid reading.
|
|
15
|
+
* - `isSafeArtifactId` is the shape check every backend applies before an
|
|
16
|
+
* id reaches a path or a key. Ids arrive from the model.
|
|
17
|
+
*
|
|
18
|
+
* @module artifacts/artifactReader
|
|
19
|
+
*/
|
|
20
|
+
import type { ArtifactPageRequest, ArtifactSearchResult, ArtifactStore, ArtifactWindow } from "../types/index.js";
|
|
21
|
+
/** Characters used for the quick preview embedded in surrogate results. */
|
|
22
|
+
export declare const DEFAULT_PREVIEW_CHARS = 500;
|
|
23
|
+
/** Matches returned by one artifact search; the rest are counted, not sent. */
|
|
24
|
+
export declare const MAX_ARTIFACT_SEARCH_MATCHES = 50;
|
|
25
|
+
/** Longest search pattern accepted — bounds the work per call. */
|
|
26
|
+
export declare const MAX_SEARCH_PATTERN_CHARS = 200;
|
|
27
|
+
/** True when `id` is shaped like something a backend may look up. */
|
|
28
|
+
export declare function isSafeArtifactId(id: string): boolean;
|
|
29
|
+
/** Head slice of a payload for surrogate headers, ellipsised when cut. */
|
|
30
|
+
export declare function generateArtifactPreview(payload: string, chars?: number): string;
|
|
31
|
+
/** Cut one window out of a payload already in memory. */
|
|
32
|
+
export declare function sliceArtifactWindow(content: string, page?: ArtifactPageRequest): ArtifactWindow;
|
|
33
|
+
/**
|
|
34
|
+
* Read one window of an artifact through whatever the backend supports.
|
|
35
|
+
*
|
|
36
|
+
* With a `page` and a backend that implements `retrieveRange`, only the
|
|
37
|
+
* window crosses the wire. Otherwise the whole payload is fetched and cut
|
|
38
|
+
* here. Without a `page` the whole payload is always fetched — a caller that
|
|
39
|
+
* omits the window is asking for the artifact.
|
|
40
|
+
*
|
|
41
|
+
* Returns `null` when the id is unknown, unsafe, or expired.
|
|
42
|
+
*/
|
|
43
|
+
export declare function readArtifactWindow(store: ArtifactStore, id: string, page?: ArtifactPageRequest): Promise<ArtifactWindow | null>;
|
|
44
|
+
/**
|
|
45
|
+
* Why a search pattern cannot be used, or `undefined` when it can.
|
|
46
|
+
*
|
|
47
|
+
* Empty would match at every position; over-long bounds the scan. Both are
|
|
48
|
+
* reported to the model rather than silently ignored — the whole point of
|
|
49
|
+
* the artifact search is that "search did not happen" is never invisible.
|
|
50
|
+
*/
|
|
51
|
+
export declare function validateSearchPattern(pattern: string): string | undefined;
|
|
52
|
+
/**
|
|
53
|
+
* Literal, case-insensitive search over an artifact.
|
|
54
|
+
*
|
|
55
|
+
* Regex metacharacters in `pattern` are matched literally — the model's
|
|
56
|
+
* input is never compiled as a regex, so it cannot be made catastrophic.
|
|
57
|
+
* Each hit carries the character `offset` of the match (what to pass back as
|
|
58
|
+
* `offset` for a targeted read), its 1-based line, and a bounded snippet.
|
|
59
|
+
*
|
|
60
|
+
* Scanning starts at `from`, so a caller can walk a long payload by passing
|
|
61
|
+
* `nextSearchOffset` back in. Every match after `from` is counted in
|
|
62
|
+
* `totalMatches`; only the first `maxMatches` are returned.
|
|
63
|
+
*
|
|
64
|
+
* `pattern` must already have passed {@link validateSearchPattern}.
|
|
65
|
+
*/
|
|
66
|
+
export declare function searchArtifactContent(content: string, pattern: string, options?: {
|
|
67
|
+
from?: number;
|
|
68
|
+
maxMatches?: number;
|
|
69
|
+
}): ArtifactSearchResult;
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact reading primitives shared by every backend and every reader.
|
|
3
|
+
*
|
|
4
|
+
* Two readers exist — the model's `retrieve_context({ artifactId })` and the
|
|
5
|
+
* host's `readArtifact()` — and two shipped backends. Everything they agree
|
|
6
|
+
* on lives here so it is agreed on exactly once:
|
|
7
|
+
*
|
|
8
|
+
* - `readArtifactWindow` is THE paged read. It asks a backend for a window
|
|
9
|
+
* when the backend can produce one (`retrieveRange`), and otherwise reads
|
|
10
|
+
* the whole payload and slices. A reader never has to know which.
|
|
11
|
+
* - `searchArtifactContent` is the literal search over an artifact. It
|
|
12
|
+
* returns bounded snippets with character offsets, never whole lines:
|
|
13
|
+
* an MCP artifact is usually one compact JSON line, so "the matching line"
|
|
14
|
+
* would be the payload the search was meant to avoid reading.
|
|
15
|
+
* - `isSafeArtifactId` is the shape check every backend applies before an
|
|
16
|
+
* id reaches a path or a key. Ids arrive from the model.
|
|
17
|
+
*
|
|
18
|
+
* @module artifacts/artifactReader
|
|
19
|
+
*/
|
|
20
|
+
/** Characters used for the quick preview embedded in surrogate results. */
|
|
21
|
+
export const DEFAULT_PREVIEW_CHARS = 500;
|
|
22
|
+
/** Matches returned by one artifact search; the rest are counted, not sent. */
|
|
23
|
+
export const MAX_ARTIFACT_SEARCH_MATCHES = 50;
|
|
24
|
+
/** Longest search pattern accepted — bounds the work per call. */
|
|
25
|
+
export const MAX_SEARCH_PATTERN_CHARS = 200;
|
|
26
|
+
/** Characters kept on each side of a hit in its snippet. */
|
|
27
|
+
const SNIPPET_CONTEXT_CHARS = 120;
|
|
28
|
+
/**
|
|
29
|
+
* Ids that may be turned into a path or a key.
|
|
30
|
+
*
|
|
31
|
+
* Ids reach the backends straight from the model through `retrieve_context`.
|
|
32
|
+
* No dots and no separators means `../../etc/passwd` can never become a file
|
|
33
|
+
* probe, and no glob characters means an id can never widen a key pattern.
|
|
34
|
+
* Real ids are UUIDs.
|
|
35
|
+
*/
|
|
36
|
+
const SAFE_ARTIFACT_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
|
|
37
|
+
/** True when `id` is shaped like something a backend may look up. */
|
|
38
|
+
export function isSafeArtifactId(id) {
|
|
39
|
+
return SAFE_ARTIFACT_ID.test(id);
|
|
40
|
+
}
|
|
41
|
+
/** Head slice of a payload for surrogate headers, ellipsised when cut. */
|
|
42
|
+
export function generateArtifactPreview(payload, chars = DEFAULT_PREVIEW_CHARS) {
|
|
43
|
+
if (payload.length <= chars) {
|
|
44
|
+
return payload;
|
|
45
|
+
}
|
|
46
|
+
return `${payload.slice(0, chars)}…`;
|
|
47
|
+
}
|
|
48
|
+
/** Cut one window out of a payload already in memory. */
|
|
49
|
+
export function sliceArtifactWindow(content, page) {
|
|
50
|
+
const offset = Math.max(0, page?.offset ?? 0);
|
|
51
|
+
const limit = page?.limit;
|
|
52
|
+
const window = limit === undefined
|
|
53
|
+
? content.slice(offset)
|
|
54
|
+
: content.slice(offset, offset + Math.max(0, limit));
|
|
55
|
+
return { content: window, offset, totalLength: content.length };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Read one window of an artifact through whatever the backend supports.
|
|
59
|
+
*
|
|
60
|
+
* With a `page` and a backend that implements `retrieveRange`, only the
|
|
61
|
+
* window crosses the wire. Otherwise the whole payload is fetched and cut
|
|
62
|
+
* here. Without a `page` the whole payload is always fetched — a caller that
|
|
63
|
+
* omits the window is asking for the artifact.
|
|
64
|
+
*
|
|
65
|
+
* Returns `null` when the id is unknown, unsafe, or expired.
|
|
66
|
+
*/
|
|
67
|
+
export async function readArtifactWindow(store, id, page) {
|
|
68
|
+
if (page && store.retrieveRange) {
|
|
69
|
+
return store.retrieveRange(id, {
|
|
70
|
+
offset: Math.max(0, page.offset ?? 0),
|
|
71
|
+
limit: page.limit === undefined ? undefined : Math.max(0, page.limit),
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
const content = await store.retrieve(id);
|
|
75
|
+
if (content === null) {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
return sliceArtifactWindow(content, page);
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Why a search pattern cannot be used, or `undefined` when it can.
|
|
82
|
+
*
|
|
83
|
+
* Empty would match at every position; over-long bounds the scan. Both are
|
|
84
|
+
* reported to the model rather than silently ignored — the whole point of
|
|
85
|
+
* the artifact search is that "search did not happen" is never invisible.
|
|
86
|
+
*/
|
|
87
|
+
export function validateSearchPattern(pattern) {
|
|
88
|
+
if (pattern.length === 0) {
|
|
89
|
+
return "Search pattern must not be empty";
|
|
90
|
+
}
|
|
91
|
+
if (pattern.length > MAX_SEARCH_PATTERN_CHARS) {
|
|
92
|
+
return `Search pattern too long (max ${MAX_SEARCH_PATTERN_CHARS} chars)`;
|
|
93
|
+
}
|
|
94
|
+
return undefined;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Literal, case-insensitive search over an artifact.
|
|
98
|
+
*
|
|
99
|
+
* Regex metacharacters in `pattern` are matched literally — the model's
|
|
100
|
+
* input is never compiled as a regex, so it cannot be made catastrophic.
|
|
101
|
+
* Each hit carries the character `offset` of the match (what to pass back as
|
|
102
|
+
* `offset` for a targeted read), its 1-based line, and a bounded snippet.
|
|
103
|
+
*
|
|
104
|
+
* Scanning starts at `from`, so a caller can walk a long payload by passing
|
|
105
|
+
* `nextSearchOffset` back in. Every match after `from` is counted in
|
|
106
|
+
* `totalMatches`; only the first `maxMatches` are returned.
|
|
107
|
+
*
|
|
108
|
+
* `pattern` must already have passed {@link validateSearchPattern}.
|
|
109
|
+
*/
|
|
110
|
+
export function searchArtifactContent(content, pattern, options) {
|
|
111
|
+
const from = Math.max(0, options?.from ?? 0);
|
|
112
|
+
const maxMatches = options?.maxMatches ?? MAX_ARTIFACT_SEARCH_MATCHES;
|
|
113
|
+
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
114
|
+
const regex = new RegExp(escaped, "gi");
|
|
115
|
+
regex.lastIndex = from;
|
|
116
|
+
const matches = [];
|
|
117
|
+
let totalMatches = 0;
|
|
118
|
+
let nextSearchOffset;
|
|
119
|
+
// Line numbers are counted incrementally from the start of the payload so
|
|
120
|
+
// the whole scan stays a single pass.
|
|
121
|
+
let line = 1;
|
|
122
|
+
let lineScanPos = 0;
|
|
123
|
+
let match;
|
|
124
|
+
while ((match = regex.exec(content)) !== null) {
|
|
125
|
+
totalMatches += 1;
|
|
126
|
+
if (matches.length < maxMatches) {
|
|
127
|
+
for (let i = lineScanPos; i < match.index; i += 1) {
|
|
128
|
+
if (content.charCodeAt(i) === 10) {
|
|
129
|
+
line += 1;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
lineScanPos = match.index;
|
|
133
|
+
const snippetOffset = Math.max(0, match.index - SNIPPET_CONTEXT_CHARS);
|
|
134
|
+
const snippetEnd = Math.min(content.length, match.index + match[0].length + SNIPPET_CONTEXT_CHARS);
|
|
135
|
+
matches.push({
|
|
136
|
+
offset: match.index,
|
|
137
|
+
line,
|
|
138
|
+
snippetOffset,
|
|
139
|
+
snippet: content.slice(snippetOffset, snippetEnd),
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
else if (nextSearchOffset === undefined) {
|
|
143
|
+
nextSearchOffset = match.index;
|
|
144
|
+
}
|
|
145
|
+
if (match[0].length === 0) {
|
|
146
|
+
// Unreachable for a non-empty pattern; keeps the loop finite regardless.
|
|
147
|
+
regex.lastIndex += 1;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
matches,
|
|
152
|
+
matchCount: matches.length,
|
|
153
|
+
totalMatches,
|
|
154
|
+
truncated: totalMatches > matches.length,
|
|
155
|
+
...(nextSearchOffset === undefined ? {} : { nextSearchOffset }),
|
|
156
|
+
};
|
|
157
|
+
}
|
|
@@ -12,8 +12,9 @@
|
|
|
12
12
|
* ArtifactStore (interface) — canonical types in src/lib/types/artifactTypes.ts
|
|
13
13
|
* LocalTempArtifactStore — single-process, filesystem-backed implementation
|
|
14
14
|
*
|
|
15
|
-
*
|
|
16
|
-
* ArtifactStore
|
|
15
|
+
* `RedisArtifactStore` is the shared-across-replicas backend; anything else
|
|
16
|
+
* (S3, a database) implements ArtifactStore and is injected via
|
|
17
|
+
* `artifacts.store` or `setArtifactStore()`.
|
|
17
18
|
*
|
|
18
19
|
* @module artifacts/artifactStore
|
|
19
20
|
*/
|
|
@@ -77,7 +78,10 @@ export declare class LocalTempArtifactStore implements ArtifactStore {
|
|
|
77
78
|
* falls back to probing the payload file itself, so an artifact whose
|
|
78
79
|
* sidecar was lost is still readable with metadata recovered from `stat`.
|
|
79
80
|
*
|
|
80
|
-
* Returns undefined for an unsafe id without touching the filesystem
|
|
81
|
+
* Returns undefined for an unsafe id without touching the filesystem: before
|
|
82
|
+
* this fallback existed an unknown id simply missed the in-memory map, and
|
|
83
|
+
* now that a miss probes `join(dir, id + ext)` the id — which arrives from
|
|
84
|
+
* the model — reaches the path layer. See `isSafeArtifactId`.
|
|
81
85
|
*/
|
|
82
86
|
private rehydrate;
|
|
83
87
|
}
|
|
@@ -12,8 +12,9 @@
|
|
|
12
12
|
* ArtifactStore (interface) — canonical types in src/lib/types/artifactTypes.ts
|
|
13
13
|
* LocalTempArtifactStore — single-process, filesystem-backed implementation
|
|
14
14
|
*
|
|
15
|
-
*
|
|
16
|
-
* ArtifactStore
|
|
15
|
+
* `RedisArtifactStore` is the shared-across-replicas backend; anything else
|
|
16
|
+
* (S3, a database) implements ArtifactStore and is injected via
|
|
17
|
+
* `artifacts.store` or `setArtifactStore()`.
|
|
17
18
|
*
|
|
18
19
|
* @module artifacts/artifactStore
|
|
19
20
|
*/
|
|
@@ -22,27 +23,15 @@ import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
|
22
23
|
import { tmpdir } from "node:os";
|
|
23
24
|
import { join } from "node:path";
|
|
24
25
|
import { logger } from "../utils/logger.js";
|
|
25
|
-
|
|
26
|
+
import { generateArtifactPreview, isSafeArtifactId } from "./artifactReader.js";
|
|
26
27
|
// ---------------------------------------------------------------------------
|
|
27
28
|
// LocalTempArtifactStore
|
|
28
29
|
// ---------------------------------------------------------------------------
|
|
29
|
-
/** Characters used for the quick preview embedded in surrogate results. */
|
|
30
|
-
const DEFAULT_PREVIEW_CHARS = 500;
|
|
31
30
|
/**
|
|
32
31
|
* Sidecar written beside every payload so a process that never called
|
|
33
32
|
* `store()` can still resolve the id (see the index-miss path in `retrieve`).
|
|
34
33
|
*/
|
|
35
34
|
const META_SUFFIX = ".meta.json";
|
|
36
|
-
/**
|
|
37
|
-
* Ids that may be turned into a path.
|
|
38
|
-
*
|
|
39
|
-
* Before the index-miss fallback existed, an unknown id simply missed the
|
|
40
|
-
* in-memory map and no filesystem lookup happened. Now that a miss probes
|
|
41
|
-
* `join(dir, id + ext)`, the id reaches the path layer — and ids arrive from
|
|
42
|
-
* the model through `retrieve_context`. No dots and no separators means
|
|
43
|
-
* `../../etc/passwd` can never become a probe. Real ids are UUIDs.
|
|
44
|
-
*/
|
|
45
|
-
const SAFE_ARTIFACT_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
|
|
46
35
|
/** Extensions `store()` can produce, in the order the fallback probes them. */
|
|
47
36
|
const PAYLOAD_EXTENSIONS = [".json", ".txt"];
|
|
48
37
|
/**
|
|
@@ -130,10 +119,7 @@ export class LocalTempArtifactStore {
|
|
|
130
119
|
process.env.NEUROLINK_ARTIFACT_REHYDRATE !== "false";
|
|
131
120
|
}
|
|
132
121
|
generatePreview(payload) {
|
|
133
|
-
|
|
134
|
-
return payload;
|
|
135
|
-
}
|
|
136
|
-
return `${payload.slice(0, DEFAULT_PREVIEW_CHARS)}…`;
|
|
122
|
+
return generateArtifactPreview(payload);
|
|
137
123
|
}
|
|
138
124
|
async store(payload, meta) {
|
|
139
125
|
await mkdir(this.dir, { recursive: true, mode: 0o700 });
|
|
@@ -231,10 +217,13 @@ export class LocalTempArtifactStore {
|
|
|
231
217
|
* falls back to probing the payload file itself, so an artifact whose
|
|
232
218
|
* sidecar was lost is still readable with metadata recovered from `stat`.
|
|
233
219
|
*
|
|
234
|
-
* Returns undefined for an unsafe id without touching the filesystem
|
|
220
|
+
* Returns undefined for an unsafe id without touching the filesystem: before
|
|
221
|
+
* this fallback existed an unknown id simply missed the in-memory map, and
|
|
222
|
+
* now that a miss probes `join(dir, id + ext)` the id — which arrives from
|
|
223
|
+
* the model — reaches the path layer. See `isSafeArtifactId`.
|
|
235
224
|
*/
|
|
236
225
|
async rehydrate(id) {
|
|
237
|
-
if (!
|
|
226
|
+
if (!isSafeArtifactId(id)) {
|
|
238
227
|
logger.debug(`[ArtifactStore] Rejected unsafe artifact id "${id}"`);
|
|
239
228
|
return undefined;
|
|
240
229
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact store factory — picks a backend the way conversation memory does.
|
|
3
|
+
*
|
|
4
|
+
* Backend: `artifacts.store` → `artifacts.storage` → `STORAGE_TYPE` → local
|
|
5
|
+
* Connection: `artifacts.redisConfig` → conversation memory's `redisConfig`
|
|
6
|
+
* → `REDIS_URL` / `REDIS_HOST` … environment variables
|
|
7
|
+
*
|
|
8
|
+
* So `STORAGE_TYPE=redis`, which already moves sessions to Redis, moves
|
|
9
|
+
* artifacts there too, on the same pooled connection, with nothing new to
|
|
10
|
+
* set. The one thing never inherited is the KEY PREFIX: artifacts get their
|
|
11
|
+
* own (`neurolink:artifact:`) even when the connection came from the
|
|
12
|
+
* conversation config or `REDIS_KEY_PREFIX`.
|
|
13
|
+
*
|
|
14
|
+
* @module artifacts/artifactStoreFactory
|
|
15
|
+
*/
|
|
16
|
+
import type { ArtifactStorageConfig, ArtifactStorageType, ArtifactStore, RedisStorageConfig } from "../types/index.js";
|
|
17
|
+
/**
|
|
18
|
+
* The Redis connection described by the environment — the same variables
|
|
19
|
+
* conversation memory reads, minus `REDIS_KEY_PREFIX`, which names the
|
|
20
|
+
* conversation keyspace and must not name this one.
|
|
21
|
+
*/
|
|
22
|
+
export declare function readArtifactRedisConfigFromEnv(): RedisStorageConfig;
|
|
23
|
+
/** Which backend `config` and the environment select. */
|
|
24
|
+
export declare function resolveArtifactStorageType(config?: ArtifactStorageConfig): ArtifactStorageType;
|
|
25
|
+
/**
|
|
26
|
+
* The Redis connection an artifact store should use, with its own key prefix.
|
|
27
|
+
*
|
|
28
|
+
* @param config `artifacts` constructor config
|
|
29
|
+
* @param fallback conversation memory's `redisConfig`, when one was given
|
|
30
|
+
*/
|
|
31
|
+
export declare function resolveArtifactRedisConfig(config?: ArtifactStorageConfig, fallback?: RedisStorageConfig): RedisStorageConfig;
|
|
32
|
+
/**
|
|
33
|
+
* Build the artifact store `config` asks for.
|
|
34
|
+
*
|
|
35
|
+
* @param config `artifacts` constructor config
|
|
36
|
+
* @param fallbackRedis conversation memory's `redisConfig`, consulted for the
|
|
37
|
+
* connection when `config.redisConfig` is absent
|
|
38
|
+
*/
|
|
39
|
+
export declare function createArtifactStore(config?: ArtifactStorageConfig, fallbackRedis?: RedisStorageConfig): ArtifactStore;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact store factory — picks a backend the way conversation memory does.
|
|
3
|
+
*
|
|
4
|
+
* Backend: `artifacts.store` → `artifacts.storage` → `STORAGE_TYPE` → local
|
|
5
|
+
* Connection: `artifacts.redisConfig` → conversation memory's `redisConfig`
|
|
6
|
+
* → `REDIS_URL` / `REDIS_HOST` … environment variables
|
|
7
|
+
*
|
|
8
|
+
* So `STORAGE_TYPE=redis`, which already moves sessions to Redis, moves
|
|
9
|
+
* artifacts there too, on the same pooled connection, with nothing new to
|
|
10
|
+
* set. The one thing never inherited is the KEY PREFIX: artifacts get their
|
|
11
|
+
* own (`neurolink:artifact:`) even when the connection came from the
|
|
12
|
+
* conversation config or `REDIS_KEY_PREFIX`.
|
|
13
|
+
*
|
|
14
|
+
* @module artifacts/artifactStoreFactory
|
|
15
|
+
*/
|
|
16
|
+
import { logger } from "../utils/logger.js";
|
|
17
|
+
import { LocalTempArtifactStore } from "./artifactStore.js";
|
|
18
|
+
import { DEFAULT_ARTIFACT_KEY_PREFIX, RedisArtifactStore, } from "./redisArtifactStore.js";
|
|
19
|
+
function numberFromEnv(name) {
|
|
20
|
+
const raw = process.env[name];
|
|
21
|
+
if (raw === undefined || raw === "") {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
const value = Number(raw);
|
|
25
|
+
return Number.isFinite(value) ? value : undefined;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The Redis connection described by the environment — the same variables
|
|
29
|
+
* conversation memory reads, minus `REDIS_KEY_PREFIX`, which names the
|
|
30
|
+
* conversation keyspace and must not name this one.
|
|
31
|
+
*/
|
|
32
|
+
export function readArtifactRedisConfigFromEnv() {
|
|
33
|
+
const config = {
|
|
34
|
+
host: process.env.REDIS_HOST,
|
|
35
|
+
port: numberFromEnv("REDIS_PORT"),
|
|
36
|
+
password: process.env.REDIS_PASSWORD,
|
|
37
|
+
db: numberFromEnv("REDIS_DB"),
|
|
38
|
+
ttl: numberFromEnv("REDIS_TTL"),
|
|
39
|
+
connectionOptions: {
|
|
40
|
+
connectTimeout: numberFromEnv("REDIS_CONNECT_TIMEOUT"),
|
|
41
|
+
maxRetriesPerRequest: numberFromEnv("REDIS_MAX_RETRIES"),
|
|
42
|
+
retryDelayOnFailover: numberFromEnv("REDIS_RETRY_DELAY"),
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
if (process.env.REDIS_URL) {
|
|
46
|
+
config.url = process.env.REDIS_URL;
|
|
47
|
+
}
|
|
48
|
+
return config;
|
|
49
|
+
}
|
|
50
|
+
/** Which backend `config` and the environment select. */
|
|
51
|
+
export function resolveArtifactStorageType(config) {
|
|
52
|
+
if (config?.storage) {
|
|
53
|
+
return config.storage;
|
|
54
|
+
}
|
|
55
|
+
const fromEnv = process.env.STORAGE_TYPE?.trim().toLowerCase();
|
|
56
|
+
return fromEnv === "redis" ? "redis" : "local";
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* The Redis connection an artifact store should use, with its own key prefix.
|
|
60
|
+
*
|
|
61
|
+
* @param config `artifacts` constructor config
|
|
62
|
+
* @param fallback conversation memory's `redisConfig`, when one was given
|
|
63
|
+
*/
|
|
64
|
+
export function resolveArtifactRedisConfig(config, fallback) {
|
|
65
|
+
const source = config?.redisConfig ?? fallback ?? readArtifactRedisConfigFromEnv();
|
|
66
|
+
return {
|
|
67
|
+
url: source.url,
|
|
68
|
+
username: source.username,
|
|
69
|
+
host: source.host,
|
|
70
|
+
port: source.port,
|
|
71
|
+
password: source.password,
|
|
72
|
+
db: source.db,
|
|
73
|
+
ttl: source.ttl,
|
|
74
|
+
connectionOptions: source.connectionOptions,
|
|
75
|
+
keyPrefix: config?.redisConfig?.keyPrefix ?? DEFAULT_ARTIFACT_KEY_PREFIX,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Build the artifact store `config` asks for.
|
|
80
|
+
*
|
|
81
|
+
* @param config `artifacts` constructor config
|
|
82
|
+
* @param fallbackRedis conversation memory's `redisConfig`, consulted for the
|
|
83
|
+
* connection when `config.redisConfig` is absent
|
|
84
|
+
*/
|
|
85
|
+
export function createArtifactStore(config, fallbackRedis) {
|
|
86
|
+
if (config?.store) {
|
|
87
|
+
logger.debug("[ArtifactStore] Using the injected artifact store");
|
|
88
|
+
return config.store;
|
|
89
|
+
}
|
|
90
|
+
const storage = resolveArtifactStorageType(config);
|
|
91
|
+
if (storage === "redis") {
|
|
92
|
+
const redis = resolveArtifactRedisConfig(config, fallbackRedis);
|
|
93
|
+
logger.debug("[ArtifactStore] Artifact store backend: redis", {
|
|
94
|
+
host: redis.host ?? (redis.url ? "(url)" : "localhost"),
|
|
95
|
+
keyPrefix: redis.keyPrefix,
|
|
96
|
+
});
|
|
97
|
+
return new RedisArtifactStore(redis);
|
|
98
|
+
}
|
|
99
|
+
logger.debug("[ArtifactStore] Artifact store backend: local-temp");
|
|
100
|
+
return new LocalTempArtifactStore();
|
|
101
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Redis Artifact Store
|
|
3
|
+
*
|
|
4
|
+
* The artifact backend for more than one machine. `LocalTempArtifactStore`
|
|
5
|
+
* writes to a pod's own `/tmp`: a replica that did not store an artifact
|
|
6
|
+
* cannot read it, a redeploy loses all of them, and `cleanup()` can only
|
|
7
|
+
* expire what its own process wrote. Here every replica sees every artifact,
|
|
8
|
+
* Redis expires them by TTL, and a paged read moves only the window.
|
|
9
|
+
*
|
|
10
|
+
* Layout, under `keyPrefix` (default `neurolink:artifact:`):
|
|
11
|
+
*
|
|
12
|
+
* <prefix><id> STRING the payload, verbatim
|
|
13
|
+
* <prefix><id>:meta STRING JSON `RedisArtifactRecord`
|
|
14
|
+
*
|
|
15
|
+
* Both keys carry the same TTL and are written in one MULTI, so an id either
|
|
16
|
+
* resolves completely or not at all.
|
|
17
|
+
*
|
|
18
|
+
* Range reads are honest about units. `retrieve_context` addresses characters;
|
|
19
|
+
* `GETRANGE` addresses bytes. The record stores the payload's character length
|
|
20
|
+
* next to its byte length, and only when the two are equal — pure ASCII, which
|
|
21
|
+
* is what JSON tool output and logs almost always are — is a byte range used
|
|
22
|
+
* as a character range. Anything else falls back to a whole read and a slice,
|
|
23
|
+
* which is slower and still correct. A window never starts on the wrong
|
|
24
|
+
* character.
|
|
25
|
+
*
|
|
26
|
+
* The connection is the same pool Redis conversation memory uses, keyed by
|
|
27
|
+
* host, port and database, so a deployment that already keeps sessions in
|
|
28
|
+
* Redis adds no connection by keeping artifacts there too.
|
|
29
|
+
*
|
|
30
|
+
* @module artifacts/redisArtifactStore
|
|
31
|
+
*/
|
|
32
|
+
import type { ArtifactMeta, ArtifactPageRequest, ArtifactRef, ArtifactStore, ArtifactWindow, RedisStorageConfig } from "../types/index.js";
|
|
33
|
+
/** Key prefix when the caller does not choose one. Never the conversation prefix. */
|
|
34
|
+
export declare const DEFAULT_ARTIFACT_KEY_PREFIX = "neurolink:artifact:";
|
|
35
|
+
/** Expiry when the caller does not choose one, or chooses an unusable one. */
|
|
36
|
+
export declare const DEFAULT_ARTIFACT_TTL_SECONDS = 86400;
|
|
37
|
+
/**
|
|
38
|
+
* Redis-backed artifact store: shared across replicas, expired by TTL,
|
|
39
|
+
* range reads for ASCII payloads.
|
|
40
|
+
*
|
|
41
|
+
* @example
|
|
42
|
+
* ```typescript
|
|
43
|
+
* const store = new RedisArtifactStore({ url: process.env.REDIS_URL });
|
|
44
|
+
* const neurolink = new NeuroLink({ artifacts: { store } });
|
|
45
|
+
* // or let NeuroLink build it: { artifacts: { storage: "redis" } }
|
|
46
|
+
* ```
|
|
47
|
+
*/
|
|
48
|
+
export declare class RedisArtifactStore implements ArtifactStore {
|
|
49
|
+
private readonly config;
|
|
50
|
+
private client?;
|
|
51
|
+
private connecting?;
|
|
52
|
+
/**
|
|
53
|
+
* @param config - Connection and key settings. `keyPrefix` defaults to
|
|
54
|
+
* `neurolink:artifact:`. `ttl` is seconds and must be positive; it
|
|
55
|
+
* defaults to 86400 (24 hours), and zero, negative or non-finite values
|
|
56
|
+
* are replaced by that default with a warning — artifacts in Redis always
|
|
57
|
+
* expire, there is no "keep forever". `userSessionsKeyPrefix` is
|
|
58
|
+
* meaningless here and ignored.
|
|
59
|
+
*/
|
|
60
|
+
constructor(config?: RedisStorageConfig);
|
|
61
|
+
generatePreview(payload: string): string;
|
|
62
|
+
store(payload: string, meta: Omit<ArtifactMeta, "createdAt">): Promise<ArtifactRef>;
|
|
63
|
+
retrieve(id: string): Promise<string | null>;
|
|
64
|
+
retrieveRange(id: string, range: ArtifactPageRequest): Promise<ArtifactWindow | null>;
|
|
65
|
+
delete(id: string): Promise<void>;
|
|
66
|
+
/**
|
|
67
|
+
* Nothing to sweep: Redis expires every artifact `ttl` seconds after it was
|
|
68
|
+
* written, on every replica at once, which is what `cleanup()` on the local
|
|
69
|
+
* store could never do.
|
|
70
|
+
*/
|
|
71
|
+
cleanup(olderThanMs: number): Promise<number>;
|
|
72
|
+
/**
|
|
73
|
+
* Release this store's reference on the pooled connection.
|
|
74
|
+
*
|
|
75
|
+
* Waits for a connect that is still in flight: otherwise the reference it
|
|
76
|
+
* is about to acquire would be assigned after this returned, and nothing
|
|
77
|
+
* would ever release it.
|
|
78
|
+
*/
|
|
79
|
+
close(): Promise<void>;
|
|
80
|
+
private payloadKey;
|
|
81
|
+
private metaKey;
|
|
82
|
+
/** Connect on first use, once, and share the pooled client afterwards. */
|
|
83
|
+
private getClient;
|
|
84
|
+
}
|