@zenera/neo 1.1.24 → 1.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/lock.d.ts +27 -0
- package/dist/lock.js +66 -0
- package/dist/match.d.ts +44 -0
- package/dist/match.js +100 -0
- package/dist/memory/diff.d.ts +39 -0
- package/dist/memory/diff.js +95 -0
- package/dist/memory/files.d.ts +1 -1
- package/dist/memory/grep.d.ts +1 -1
- package/dist/memory/grep.js +2 -2
- package/dist/memory/merge.js +1 -31
- package/dist/memory/store.d.ts +1 -5
- package/dist/memory/store.js +4 -51
- package/dist/memory/twin.d.ts +6 -0
- package/dist/memory/twin.js +40 -0
- package/dist/models/factory.d.ts +24 -0
- package/dist/models/factory.js +153 -14
- package/dist/runner.js +7 -0
- package/dist/tools/sandbox.js +40 -2
- package/package.json +1 -1
- package/dist/memory/match.d.ts +0 -18
- package/dist/memory/match.js +0 -61
package/dist/index.d.ts
CHANGED
|
@@ -9,6 +9,9 @@ export * from './frontmatter.ts';
|
|
|
9
9
|
export * from './ids.ts';
|
|
10
10
|
export * from './inspect/index.ts';
|
|
11
11
|
export * as Kernel from './kernel.ts';
|
|
12
|
+
export * from './lock.ts';
|
|
13
|
+
export * from './match.ts';
|
|
14
|
+
export * from './memory/diff.ts';
|
|
12
15
|
export * from './memory/files.ts';
|
|
13
16
|
export * from './memory/graph.ts';
|
|
14
17
|
export * from './memory/grep.ts';
|
package/dist/index.js
CHANGED
|
@@ -9,6 +9,9 @@ export * from "./frontmatter.js";
|
|
|
9
9
|
export * from "./ids.js";
|
|
10
10
|
export * from "./inspect/index.js";
|
|
11
11
|
export * as Kernel from "./kernel.js";
|
|
12
|
+
export * from "./lock.js";
|
|
13
|
+
export * from "./match.js";
|
|
14
|
+
export * from "./memory/diff.js";
|
|
12
15
|
export * from "./memory/files.js";
|
|
13
16
|
export * from "./memory/graph.js";
|
|
14
17
|
export * from "./memory/grep.js";
|
package/dist/lock.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export interface Lock {
|
|
2
|
+
pid: number;
|
|
3
|
+
host: string;
|
|
4
|
+
startedAt: string;
|
|
5
|
+
}
|
|
6
|
+
/** A lock body for this process, with whatever else the holder wants to say. */
|
|
7
|
+
export declare function ownLock<T extends object>(extra?: T): Lock & T;
|
|
8
|
+
/**
|
|
9
|
+
* `kill(pid, 0)` sends no signal and only asks whether the process exists.
|
|
10
|
+
* EPERM means it exists and belongs to someone else, which still counts.
|
|
11
|
+
*/
|
|
12
|
+
export declare function pidAlive(pid: number): boolean;
|
|
13
|
+
/** The lock file as written, or undefined when absent or unreadable. */
|
|
14
|
+
export declare function readLock<T extends Lock = Lock>(path: string): T | undefined;
|
|
15
|
+
/**
|
|
16
|
+
* Who holds a lock right now, if anyone. Read-only: it never claims the lock
|
|
17
|
+
* and never clears a stale one. A lock from another host cannot be verified
|
|
18
|
+
* here, so it is not reported as held.
|
|
19
|
+
*/
|
|
20
|
+
export declare function liveHolder<T extends Lock = Lock>(path: string): T | undefined;
|
|
21
|
+
/**
|
|
22
|
+
* Writes `lock` to `path`, or calls `refuse` with the live holder and throws
|
|
23
|
+
* what it returns. `wx` makes the create and the check one operation, so two
|
|
24
|
+
* processes racing for the same directory cannot both win.
|
|
25
|
+
*/
|
|
26
|
+
export declare function claimLock<T extends Lock>(path: string, lock: T, refuse: (held: T) => Error, indent?: number): void;
|
|
27
|
+
//# sourceMappingURL=lock.d.ts.map
|
package/dist/lock.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { hostname } from 'node:os';
|
|
3
|
+
/** A lock body for this process, with whatever else the holder wants to say. */
|
|
4
|
+
export function ownLock(extra) {
|
|
5
|
+
return {
|
|
6
|
+
pid: process.pid,
|
|
7
|
+
host: hostname(),
|
|
8
|
+
startedAt: new Date().toISOString(),
|
|
9
|
+
...extra,
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* `kill(pid, 0)` sends no signal and only asks whether the process exists.
|
|
14
|
+
* EPERM means it exists and belongs to someone else, which still counts.
|
|
15
|
+
*/
|
|
16
|
+
export function pidAlive(pid) {
|
|
17
|
+
try {
|
|
18
|
+
process.kill(pid, 0);
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
catch (err) {
|
|
22
|
+
return err.code === 'EPERM';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
/** The lock file as written, or undefined when absent or unreadable. */
|
|
26
|
+
export function readLock(path) {
|
|
27
|
+
try {
|
|
28
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return undefined;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Who holds a lock right now, if anyone. Read-only: it never claims the lock
|
|
36
|
+
* and never clears a stale one. A lock from another host cannot be verified
|
|
37
|
+
* here, so it is not reported as held.
|
|
38
|
+
*/
|
|
39
|
+
export function liveHolder(path) {
|
|
40
|
+
const held = readLock(path);
|
|
41
|
+
return held && held.host === hostname() && pidAlive(held.pid) ? held : undefined;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Writes `lock` to `path`, or calls `refuse` with the live holder and throws
|
|
45
|
+
* what it returns. `wx` makes the create and the check one operation, so two
|
|
46
|
+
* processes racing for the same directory cannot both win.
|
|
47
|
+
*/
|
|
48
|
+
export function claimLock(path, lock, refuse, indent = 4) {
|
|
49
|
+
const body = `${JSON.stringify(lock, null, indent)}\n`;
|
|
50
|
+
try {
|
|
51
|
+
writeFileSync(path, body, { flag: 'wx' });
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
if (err.code !== 'EEXIST') {
|
|
56
|
+
throw err;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const held = liveHolder(path);
|
|
60
|
+
if (held) {
|
|
61
|
+
throw refuse(held);
|
|
62
|
+
}
|
|
63
|
+
// Stale, or from another machine's run that cannot be verified here.
|
|
64
|
+
writeFileSync(path, body);
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=lock.js.map
|
package/dist/match.d.ts
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Long enough for any honest pattern, short enough to bound a bad one. */
|
|
2
|
+
export declare const MAX_PATTERN = 200;
|
|
3
|
+
export declare class PatternError extends Error {
|
|
4
|
+
}
|
|
5
|
+
export interface MatchOptions {
|
|
6
|
+
/** read the pattern as a regular expression rather than as a literal */
|
|
7
|
+
regex?: boolean;
|
|
8
|
+
caseSensitive?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export type Matcher = (text: string) => boolean;
|
|
11
|
+
/** The offset of the first match in `text`, or -1. */
|
|
12
|
+
export type Finder = (text: string) => number;
|
|
13
|
+
/**
|
|
14
|
+
* Where a pattern first occurs. Literal by default: someone typing `user.id`
|
|
15
|
+
* means those seven characters, and a dot that quietly matched anything would
|
|
16
|
+
* be a worse answer than no answer.
|
|
17
|
+
*/
|
|
18
|
+
export declare function finder(pattern: string, options?: MatchOptions): Finder;
|
|
19
|
+
/** Whether a pattern occurs at all, read the same way `finder` reads it. */
|
|
20
|
+
export declare function matcher(pattern: string, options?: MatchOptions): Matcher;
|
|
21
|
+
/**
|
|
22
|
+
* A glob, matched against the whole string. Globs rather than regexes because
|
|
23
|
+
* these are for naming things — a path, a schema — and a star is what everyone
|
|
24
|
+
* reaches for first.
|
|
25
|
+
*/
|
|
26
|
+
export declare function wildcard(pattern: string, options?: {
|
|
27
|
+
caseSensitive?: boolean;
|
|
28
|
+
}): Matcher;
|
|
29
|
+
/** Whether a pattern is asking to be read as a glob at all. */
|
|
30
|
+
export declare const isGlob: (pattern: string) => boolean;
|
|
31
|
+
/**
|
|
32
|
+
* What someone means when they type a name into a filter. With a star in it,
|
|
33
|
+
* a glob; without one, a substring — because `password` typed into `--name` is
|
|
34
|
+
* a search for the word, and a whole-string match would answer nothing and
|
|
35
|
+
* look like the field does not exist.
|
|
36
|
+
*
|
|
37
|
+
* `regex` settles it outright, and has to: a star is punctuation in both
|
|
38
|
+
* languages, so `^/(users|teams)/.*` read as a glob would match nothing and
|
|
39
|
+
* never say why.
|
|
40
|
+
*/
|
|
41
|
+
export declare function loose(pattern: string, options?: MatchOptions): Matcher;
|
|
42
|
+
/** True when any of the patterns matches; no patterns means no opinion. */
|
|
43
|
+
export declare function anyOf(matchers: readonly Matcher[]): Matcher | undefined;
|
|
44
|
+
//# sourceMappingURL=match.d.ts.map
|
package/dist/match.js
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// Matching, with nothing learned in between
|
|
3
|
+
//
|
|
4
|
+
// Everything here is exact. A glob matches the characters it names and a
|
|
5
|
+
// substring is a substring, which is the whole point of the surfaces built on
|
|
6
|
+
// it: a vector index answers "what is near this", and near is a ranking, so it
|
|
7
|
+
// can only ever return the top of a list. When the question is "does the word
|
|
8
|
+
// `password` appear anywhere at all", a ranking is the wrong instrument and no
|
|
9
|
+
// amount of tuning makes it the right one.
|
|
10
|
+
//
|
|
11
|
+
// Memory grep and @zenera/rag's lookups both stand on this. A `finder` says
|
|
12
|
+
// *where*, for grep, which reports the line; a `matcher` says *whether*.
|
|
13
|
+
//
|
|
14
|
+
// A pattern may arrive from a model, so a regex is a bounded promise: the
|
|
15
|
+
// length is capped here and the scan that uses it keeps a deadline.
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
/** Long enough for any honest pattern, short enough to bound a bad one. */
|
|
18
|
+
export const MAX_PATTERN = 200;
|
|
19
|
+
export class PatternError extends Error {
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Where a pattern first occurs. Literal by default: someone typing `user.id`
|
|
23
|
+
* means those seven characters, and a dot that quietly matched anything would
|
|
24
|
+
* be a worse answer than no answer.
|
|
25
|
+
*/
|
|
26
|
+
export function finder(pattern, options = {}) {
|
|
27
|
+
guard(pattern);
|
|
28
|
+
if (!options.regex) {
|
|
29
|
+
if (options.caseSensitive) {
|
|
30
|
+
return (text) => text.indexOf(pattern);
|
|
31
|
+
}
|
|
32
|
+
const needle = pattern.toLowerCase();
|
|
33
|
+
return (text) => text.toLowerCase().indexOf(needle);
|
|
34
|
+
}
|
|
35
|
+
const expression = compile(pattern, options.caseSensitive ? '' : 'i');
|
|
36
|
+
// `lastIndex` is not carried between calls: the flags never include `g`.
|
|
37
|
+
return (text) => text.search(expression);
|
|
38
|
+
}
|
|
39
|
+
/** Whether a pattern occurs at all, read the same way `finder` reads it. */
|
|
40
|
+
export function matcher(pattern, options = {}) {
|
|
41
|
+
const find = finder(pattern, options);
|
|
42
|
+
return (text) => find(text) >= 0;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* A glob, matched against the whole string. Globs rather than regexes because
|
|
46
|
+
* these are for naming things — a path, a schema — and a star is what everyone
|
|
47
|
+
* reaches for first.
|
|
48
|
+
*/
|
|
49
|
+
export function wildcard(pattern, options = {}) {
|
|
50
|
+
guard(pattern);
|
|
51
|
+
const source = [...pattern]
|
|
52
|
+
.map((char) => (char === '*' ? '.*' : char === '?' ? '.' : escape(char)))
|
|
53
|
+
.join('');
|
|
54
|
+
const expression = compile(`^${source}$`, options.caseSensitive ? '' : 'i');
|
|
55
|
+
return (text) => expression.test(text);
|
|
56
|
+
}
|
|
57
|
+
/** Whether a pattern is asking to be read as a glob at all. */
|
|
58
|
+
export const isGlob = (pattern) => /[*?]/.test(pattern);
|
|
59
|
+
/**
|
|
60
|
+
* What someone means when they type a name into a filter. With a star in it,
|
|
61
|
+
* a glob; without one, a substring — because `password` typed into `--name` is
|
|
62
|
+
* a search for the word, and a whole-string match would answer nothing and
|
|
63
|
+
* look like the field does not exist.
|
|
64
|
+
*
|
|
65
|
+
* `regex` settles it outright, and has to: a star is punctuation in both
|
|
66
|
+
* languages, so `^/(users|teams)/.*` read as a glob would match nothing and
|
|
67
|
+
* never say why.
|
|
68
|
+
*/
|
|
69
|
+
export function loose(pattern, options = {}) {
|
|
70
|
+
if (options.regex) {
|
|
71
|
+
return matcher(pattern, options);
|
|
72
|
+
}
|
|
73
|
+
return isGlob(pattern) ? wildcard(pattern, options) : matcher(pattern, options);
|
|
74
|
+
}
|
|
75
|
+
/** True when any of the patterns matches; no patterns means no opinion. */
|
|
76
|
+
export function anyOf(matchers) {
|
|
77
|
+
if (matchers.length === 0) {
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
return (text) => matchers.some((match) => match(text));
|
|
81
|
+
}
|
|
82
|
+
// ---------------------------------------------------------------------------
|
|
83
|
+
function guard(pattern) {
|
|
84
|
+
if (pattern.length === 0) {
|
|
85
|
+
throw new PatternError('the pattern is empty');
|
|
86
|
+
}
|
|
87
|
+
if (pattern.length > MAX_PATTERN) {
|
|
88
|
+
throw new PatternError(`the pattern is longer than ${MAX_PATTERN} characters`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function compile(source, flags) {
|
|
92
|
+
try {
|
|
93
|
+
return new RegExp(source, flags);
|
|
94
|
+
}
|
|
95
|
+
catch (err) {
|
|
96
|
+
throw new PatternError(`invalid pattern: ${err.message}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const escape = (char) => char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
100
|
+
//# sourceMappingURL=match.js.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { MemoryStore } from './store.ts';
|
|
2
|
+
import type { MemoryNode } from './types.ts';
|
|
3
|
+
export interface MemoryNearest {
|
|
4
|
+
id: string;
|
|
5
|
+
score: number;
|
|
6
|
+
text: string;
|
|
7
|
+
}
|
|
8
|
+
export interface MemoryDiffAdded {
|
|
9
|
+
node: MemoryNode;
|
|
10
|
+
/** the closest comparable node the base held, if any */
|
|
11
|
+
nearest?: MemoryNearest;
|
|
12
|
+
/** `merge` would fold it onto `nearest`: the base already said this */
|
|
13
|
+
twin: boolean;
|
|
14
|
+
}
|
|
15
|
+
export interface MemoryDiff {
|
|
16
|
+
base: string;
|
|
17
|
+
other: string;
|
|
18
|
+
/** how `nearest` was scored: cosine when both sides hold vectors, else term overlap */
|
|
19
|
+
by: 'vector' | 'text';
|
|
20
|
+
added: MemoryDiffAdded[];
|
|
21
|
+
revised: {
|
|
22
|
+
before: MemoryNode;
|
|
23
|
+
after: MemoryNode;
|
|
24
|
+
}[];
|
|
25
|
+
/** base nodes `memory_load` returned since — what the run actually read */
|
|
26
|
+
used: {
|
|
27
|
+
node: MemoryNode;
|
|
28
|
+
loads: number;
|
|
29
|
+
}[];
|
|
30
|
+
removed: MemoryNode[];
|
|
31
|
+
/** edges `other` has that `base` did not */
|
|
32
|
+
edges: number;
|
|
33
|
+
supersedes: {
|
|
34
|
+
source: string;
|
|
35
|
+
target: string;
|
|
36
|
+
}[];
|
|
37
|
+
}
|
|
38
|
+
export declare function diffMemories(base: MemoryStore, other: MemoryStore): MemoryDiff;
|
|
39
|
+
//# sourceMappingURL=diff.d.ts.map
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { DUPLICATE_SCORE } from "./index.js";
|
|
2
|
+
import { comparable, sameContent, twinKey } from "./twin.js";
|
|
3
|
+
import { overlap, tokenize } from "./vectors.js";
|
|
4
|
+
export function diffMemories(base, other) {
|
|
5
|
+
const stale = base.graph.superseded();
|
|
6
|
+
const block = base.vectors;
|
|
7
|
+
const theirs = other.vectors;
|
|
8
|
+
const byVector = !!block?.rows && !!theirs && theirs.dims === block.dims;
|
|
9
|
+
const held = base.graph.nodes();
|
|
10
|
+
const lexical = new Map(held.filter((n) => !stale.has(n.id)).map((n) => [twinKey(n), n.id]));
|
|
11
|
+
const nearestOf = (node) => {
|
|
12
|
+
if (node.file) {
|
|
13
|
+
return { twin: false };
|
|
14
|
+
}
|
|
15
|
+
const allow = (id) => {
|
|
16
|
+
const n = base.graph.get(id);
|
|
17
|
+
return !stale.has(id) && !!n && comparable(n, node);
|
|
18
|
+
};
|
|
19
|
+
const exact = lexical.get(twinKey(node));
|
|
20
|
+
const vector = byVector ? theirs.get(node.id) : undefined;
|
|
21
|
+
if (vector) {
|
|
22
|
+
const hit = block.topK(vector, 1, allow)[0];
|
|
23
|
+
if (hit) {
|
|
24
|
+
const text = base.graph.get(hit.id).text;
|
|
25
|
+
return {
|
|
26
|
+
nearest: { id: hit.id, score: hit.score, text },
|
|
27
|
+
twin: hit.score >= DUPLICATE_SCORE,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (exact) {
|
|
32
|
+
return {
|
|
33
|
+
nearest: { id: exact, score: 1, text: base.graph.get(exact).text },
|
|
34
|
+
twin: true,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
const words = tokenize(node.text);
|
|
38
|
+
let best;
|
|
39
|
+
for (const n of held) {
|
|
40
|
+
if (!allow(n.id)) {
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
const score = overlap(words, tokenize(n.text));
|
|
44
|
+
if (score > 0 && (!best || score > best.score)) {
|
|
45
|
+
best = { id: n.id, score, text: n.text };
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return { nearest: best, twin: false };
|
|
49
|
+
};
|
|
50
|
+
const diff = {
|
|
51
|
+
base: base.dir,
|
|
52
|
+
other: other.dir,
|
|
53
|
+
by: byVector ? 'vector' : 'text',
|
|
54
|
+
added: [],
|
|
55
|
+
revised: [],
|
|
56
|
+
used: [],
|
|
57
|
+
removed: [],
|
|
58
|
+
edges: 0,
|
|
59
|
+
supersedes: [],
|
|
60
|
+
};
|
|
61
|
+
for (const after of other.graph.nodes()) {
|
|
62
|
+
const before = base.graph.get(after.id);
|
|
63
|
+
if (!before) {
|
|
64
|
+
diff.added.push({ node: after, ...nearestOf(after) });
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
if (before.revision !== after.revision || !sameContent(before, after)) {
|
|
68
|
+
diff.revised.push({ before, after });
|
|
69
|
+
}
|
|
70
|
+
if (after.useCount > before.useCount) {
|
|
71
|
+
diff.used.push({ node: after, loads: after.useCount - before.useCount });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
for (const node of held) {
|
|
75
|
+
if (!other.graph.has(node.id)) {
|
|
76
|
+
diff.removed.push(node);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const key = (source, target, relation) => `${source}\u0000${target}\u0000${relation}`;
|
|
80
|
+
const had = new Set(base.graph.links().map((l) => key(l.source, l.target, l.attrs.relation)));
|
|
81
|
+
for (const link of other.graph.links()) {
|
|
82
|
+
if (had.has(key(link.source, link.target, link.attrs.relation))) {
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
diff.edges++;
|
|
86
|
+
if (link.attrs.relation === 'SUPERSEDES') {
|
|
87
|
+
diff.supersedes.push({ source: link.source, target: link.target });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const newest = (a, b) => b.node.createdAt.localeCompare(a.node.createdAt);
|
|
91
|
+
diff.added.sort(newest);
|
|
92
|
+
diff.used.sort((a, b) => b.loads - a.loads);
|
|
93
|
+
return diff;
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=diff.js.map
|
package/dist/memory/files.d.ts
CHANGED
|
@@ -20,7 +20,7 @@ export interface RememberOptions {
|
|
|
20
20
|
*/
|
|
21
21
|
export declare function rememberFile(dir: string, source: string, id: string, opts?: RememberOptions): Promise<MemoryFile>;
|
|
22
22
|
/** The host path of a remembered file, for reading it back or deleting it. */
|
|
23
|
-
export declare function hostPath(dir: string, file: MemoryFile): string;
|
|
23
|
+
export declare function hostPath(dir: string, file: Pick<MemoryFile, 'path'>): string;
|
|
24
24
|
/** Drop the bytes of a forgotten node. Missing is success — the goal is absence. */
|
|
25
25
|
export declare function forgetFile(dir: string, file: MemoryFile): Promise<void>;
|
|
26
26
|
//# sourceMappingURL=files.d.ts.map
|
package/dist/memory/grep.d.ts
CHANGED
package/dist/memory/grep.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile, stat } from 'node:fs/promises';
|
|
2
|
+
import { PatternError, finder } from "../match.js";
|
|
2
3
|
import { hostPath } from "./files.js";
|
|
3
|
-
import { PatternError, matcher } from "./match.js";
|
|
4
4
|
import { MemoryError } from "./types.js";
|
|
5
5
|
// ---------------------------------------------------------------------------
|
|
6
6
|
// The memory, read exhaustively
|
|
@@ -106,7 +106,7 @@ export async function grepMemory(store, pattern, filter = {}) {
|
|
|
106
106
|
*/
|
|
107
107
|
function compile(pattern, options) {
|
|
108
108
|
try {
|
|
109
|
-
return
|
|
109
|
+
return finder(pattern, options);
|
|
110
110
|
}
|
|
111
111
|
catch (err) {
|
|
112
112
|
if (err instanceof PatternError) {
|
package/dist/memory/merge.js
CHANGED
|
@@ -4,6 +4,7 @@ import { join, resolve } from 'node:path';
|
|
|
4
4
|
import { hostPath } from "./files.js";
|
|
5
5
|
import { DUPLICATE_SCORE, flatten } from "./index.js";
|
|
6
6
|
import { MANIFEST_FILE, MemoryStore, lockHolder } from "./store.js";
|
|
7
|
+
import { comparable, sameContent, twinKey } from "./twin.js";
|
|
7
8
|
import { MemoryError } from "./types.js";
|
|
8
9
|
// ---------------------------------------------------------------------------
|
|
9
10
|
// Folding several memories into one
|
|
@@ -251,15 +252,6 @@ function twinOf(target, src, node, lexical, stale) {
|
|
|
251
252
|
const id = lexical.get(twinKey(node));
|
|
252
253
|
return id && !stale.has(id) ? id : undefined;
|
|
253
254
|
}
|
|
254
|
-
function comparable(a, b) {
|
|
255
|
-
return a.id !== b.id && a.kind === b.kind && !a.file && audienceKey(a) === audienceKey(b);
|
|
256
|
-
}
|
|
257
|
-
function twinKey(node) {
|
|
258
|
-
return [node.kind, audienceKey(node), flatten(node.text)].join('\u0000');
|
|
259
|
-
}
|
|
260
|
-
function audienceKey(node) {
|
|
261
|
-
return [...node.audience].sort().join('|');
|
|
262
|
-
}
|
|
263
255
|
function carryVector(from, to, id) {
|
|
264
256
|
const vector = from.vectors?.get(id);
|
|
265
257
|
const block = to.vectors;
|
|
@@ -284,28 +276,6 @@ function reconcile(keep, other) {
|
|
|
284
276
|
function newer(a, b) {
|
|
285
277
|
return a.revision === b.revision ? a.updatedAt > b.updatedAt : a.revision > b.revision;
|
|
286
278
|
}
|
|
287
|
-
function sameContent(a, b) {
|
|
288
|
-
return (a.kind === b.kind &&
|
|
289
|
-
a.text === b.text &&
|
|
290
|
-
audienceKey(a) === audienceKey(b) &&
|
|
291
|
-
a.file?.sha256 === b.file?.sha256 &&
|
|
292
|
-
deep(a.metadata, b.metadata));
|
|
293
|
-
}
|
|
294
|
-
function deep(a, b) {
|
|
295
|
-
if (a === b) {
|
|
296
|
-
return true;
|
|
297
|
-
}
|
|
298
|
-
if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) {
|
|
299
|
-
return false;
|
|
300
|
-
}
|
|
301
|
-
if (Array.isArray(a) !== Array.isArray(b)) {
|
|
302
|
-
return false;
|
|
303
|
-
}
|
|
304
|
-
const left = a;
|
|
305
|
-
const right = b;
|
|
306
|
-
const keys = Object.keys(left);
|
|
307
|
-
return keys.length === Object.keys(right).length && keys.every((k) => deep(left[k], right[k]));
|
|
308
|
-
}
|
|
309
279
|
/** Stat every file the merge would copy before it copies any of them. */
|
|
310
280
|
async function checkFiles(plan) {
|
|
311
281
|
for (const planned of plan.values()) {
|
package/dist/memory/store.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type Lock } from '../lock.ts';
|
|
1
2
|
import { MemoryGraph } from './graph.ts';
|
|
2
3
|
import { VectorBlock } from './vectors.ts';
|
|
3
4
|
export declare const MEMORY_VERSION = 1;
|
|
@@ -19,11 +20,6 @@ export interface MemoryManifest {
|
|
|
19
20
|
edges: number;
|
|
20
21
|
updatedAt: string;
|
|
21
22
|
}
|
|
22
|
-
export interface Lock {
|
|
23
|
-
pid: number;
|
|
24
|
-
host: string;
|
|
25
|
-
startedAt: string;
|
|
26
|
-
}
|
|
27
23
|
export interface OpenOptions {
|
|
28
24
|
/** the embedder the caller intends to use; a mismatch with the manifest is fatal */
|
|
29
25
|
embedding?: MemoryEmbedding;
|
package/dist/memory/store.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { unlinkSync } from 'node:fs';
|
|
2
2
|
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
3
|
-
import { hostname } from 'node:os';
|
|
4
3
|
import { join } from 'node:path';
|
|
4
|
+
import { claimLock, liveHolder, ownLock } from "../lock.js";
|
|
5
5
|
import { MemoryGraph } from "./graph.js";
|
|
6
6
|
import { MemoryError } from "./types.js";
|
|
7
7
|
import { VectorBlock } from "./vectors.js";
|
|
@@ -143,33 +143,8 @@ export class MemoryStore {
|
|
|
143
143
|
// Already gone, or taken over as stale: neither is worth failing a run for.
|
|
144
144
|
}
|
|
145
145
|
}
|
|
146
|
-
/**
|
|
147
|
-
* `wx` makes the create and the check one operation, so two runs racing for
|
|
148
|
-
* the same project cannot both win.
|
|
149
|
-
*/
|
|
150
146
|
#claim() {
|
|
151
|
-
|
|
152
|
-
const lock = {
|
|
153
|
-
pid: process.pid,
|
|
154
|
-
host: hostname(),
|
|
155
|
-
startedAt: new Date().toISOString(),
|
|
156
|
-
};
|
|
157
|
-
const body = `${JSON.stringify(lock, null, 4)}\n`;
|
|
158
|
-
try {
|
|
159
|
-
writeFileSync(path, body, { flag: 'wx' });
|
|
160
|
-
this.#locked = true;
|
|
161
|
-
return;
|
|
162
|
-
}
|
|
163
|
-
catch (err) {
|
|
164
|
-
if (err.code !== 'EEXIST') {
|
|
165
|
-
throw err;
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
const held = readLock(path);
|
|
169
|
-
if (held && held.host === hostname() && alive(held.pid)) {
|
|
170
|
-
throw new MemoryError(`this memory is in use (pid ${held.pid}, since ${held.startedAt})`, 'wait for that run to finish, or remove .lock if it crashed');
|
|
171
|
-
}
|
|
172
|
-
writeFileSync(path, body);
|
|
147
|
+
claimLock(join(this.dir, LOCK_FILE), ownLock(), (held) => new MemoryError(`this memory is in use (pid ${held.pid}, since ${held.startedAt})`, 'wait for that run to finish, or remove .lock if it crashed'));
|
|
173
148
|
this.#locked = true;
|
|
174
149
|
}
|
|
175
150
|
}
|
|
@@ -226,28 +201,6 @@ async function atomic(path, body) {
|
|
|
226
201
|
* directory this process has no intention of opening for writing.
|
|
227
202
|
*/
|
|
228
203
|
export function lockHolder(dir) {
|
|
229
|
-
|
|
230
|
-
return held && held.host === hostname() && alive(held.pid) ? held : undefined;
|
|
231
|
-
}
|
|
232
|
-
function readLock(path) {
|
|
233
|
-
try {
|
|
234
|
-
return JSON.parse(readFileSync(path, 'utf8'));
|
|
235
|
-
}
|
|
236
|
-
catch {
|
|
237
|
-
return undefined;
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
/**
|
|
241
|
-
* `kill(pid, 0)` sends no signal and only asks whether the process exists.
|
|
242
|
-
* EPERM means it exists and belongs to someone else, which still counts.
|
|
243
|
-
*/
|
|
244
|
-
function alive(pid) {
|
|
245
|
-
try {
|
|
246
|
-
process.kill(pid, 0);
|
|
247
|
-
return true;
|
|
248
|
-
}
|
|
249
|
-
catch (err) {
|
|
250
|
-
return err.code === 'EPERM';
|
|
251
|
-
}
|
|
204
|
+
return liveHolder(join(dir, LOCK_FILE));
|
|
252
205
|
}
|
|
253
206
|
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { MemoryNode } from './types.ts';
|
|
2
|
+
export declare function comparable(a: MemoryNode, b: MemoryNode): boolean;
|
|
3
|
+
export declare function twinKey(node: MemoryNode): string;
|
|
4
|
+
export declare function audienceKey(node: MemoryNode): string;
|
|
5
|
+
export declare function sameContent(a: MemoryNode, b: MemoryNode): boolean;
|
|
6
|
+
//# sourceMappingURL=twin.d.ts.map
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { flatten } from "./index.js";
|
|
2
|
+
// ---------------------------------------------------------------------------
|
|
3
|
+
// When two memories are the same memory
|
|
4
|
+
//
|
|
5
|
+
// Shared by `merge`, which folds a twin onto the node already held, and `diff`,
|
|
6
|
+
// which reports it. One rule, so what diff calls a re-commit is exactly what
|
|
7
|
+
// merge would have folded.
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
export function comparable(a, b) {
|
|
10
|
+
return a.id !== b.id && a.kind === b.kind && !a.file && audienceKey(a) === audienceKey(b);
|
|
11
|
+
}
|
|
12
|
+
export function twinKey(node) {
|
|
13
|
+
return [node.kind, audienceKey(node), flatten(node.text)].join('\u0000');
|
|
14
|
+
}
|
|
15
|
+
export function audienceKey(node) {
|
|
16
|
+
return [...node.audience].sort().join('|');
|
|
17
|
+
}
|
|
18
|
+
export function sameContent(a, b) {
|
|
19
|
+
return (a.kind === b.kind &&
|
|
20
|
+
a.text === b.text &&
|
|
21
|
+
audienceKey(a) === audienceKey(b) &&
|
|
22
|
+
a.file?.sha256 === b.file?.sha256 &&
|
|
23
|
+
deep(a.metadata, b.metadata));
|
|
24
|
+
}
|
|
25
|
+
function deep(a, b) {
|
|
26
|
+
if (a === b) {
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
if (Array.isArray(a) !== Array.isArray(b)) {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
const left = a;
|
|
36
|
+
const right = b;
|
|
37
|
+
const keys = Object.keys(left);
|
|
38
|
+
return keys.length === Object.keys(right).length && keys.every((k) => deep(left[k], right[k]));
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=twin.js.map
|
package/dist/models/factory.d.ts
CHANGED
|
@@ -150,6 +150,15 @@ export interface ModelRequirement {
|
|
|
150
150
|
}
|
|
151
151
|
export declare const providerKinds: ProviderKind[];
|
|
152
152
|
export declare function isProviderKind(name: string): name is ProviderKind;
|
|
153
|
+
/**
|
|
154
|
+
* Vertex serves three request formats behind one account, and the model id
|
|
155
|
+
* says which: Google's own models through the GenAI API, Claude through
|
|
156
|
+
* Anthropic's Messages format at `rawPredict`, and every other publisher's
|
|
157
|
+
* (`xai/grok-4.1-fast-reasoning`, `qwen/qwen3-…-maas`) through the
|
|
158
|
+
* OpenAI-compatible endpoint, which names its models `publisher/model`.
|
|
159
|
+
*/
|
|
160
|
+
export type VertexRoute = 'gemini' | 'anthropic' | 'openai';
|
|
161
|
+
export declare function vertexRoute(model: string): VertexRoute;
|
|
153
162
|
/**
|
|
154
163
|
* Which knobs each adapter actually reads.
|
|
155
164
|
*
|
|
@@ -240,6 +249,21 @@ export declare class ModelRegistry {
|
|
|
240
249
|
/** A declared kind wins; otherwise a built-in name is its own kind. */
|
|
241
250
|
kindOf(name: string): ProviderKind;
|
|
242
251
|
}
|
|
252
|
+
export interface VertexEndpoint {
|
|
253
|
+
project: string;
|
|
254
|
+
location: string;
|
|
255
|
+
/** `https://…/v1/projects/<project>/locations/<location>` */
|
|
256
|
+
base: string;
|
|
257
|
+
/** a fresh access token from Application Default Credentials, or `spec.token` */
|
|
258
|
+
token: () => Promise<string>;
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Where a Vertex provider's partner models are served, and how to authenticate
|
|
262
|
+
* there. The GenAI SDK does both for Gemini; nothing does them for Claude or
|
|
263
|
+
* the OpenAI-compatible endpoint, so they are resolved here the same way:
|
|
264
|
+
* `project`, then GOOGLE_CLOUD_PROJECT, then the service-account file.
|
|
265
|
+
*/
|
|
266
|
+
export declare function vertexEndpoint(spec?: ProviderSpec, name?: string): VertexEndpoint;
|
|
243
267
|
/**
|
|
244
268
|
* The registry behind the bare `createModel`. An application that never
|
|
245
269
|
* declares a provider still gets shared clients through it, because the
|
package/dist/models/factory.js
CHANGED
|
@@ -114,6 +114,20 @@ export const providerKinds = Object.keys(KINDS);
|
|
|
114
114
|
export function isProviderKind(name) {
|
|
115
115
|
return Object.hasOwn(KINDS, name);
|
|
116
116
|
}
|
|
117
|
+
export function vertexRoute(model) {
|
|
118
|
+
if (/^(anthropic\/)?claude-/.test(model)) {
|
|
119
|
+
return 'anthropic';
|
|
120
|
+
}
|
|
121
|
+
return model.includes('/') ? 'openai' : 'gemini';
|
|
122
|
+
}
|
|
123
|
+
/** The protocol one model on one kind is spoken in. Only Vertex has more than one. */
|
|
124
|
+
function protocolOf(kind, model) {
|
|
125
|
+
return kind === 'vertex' ? vertexRoute(model) : KINDS[kind].protocol;
|
|
126
|
+
}
|
|
127
|
+
/** The OpenAI-compatible endpoint on Vertex has chat completions and nothing else. */
|
|
128
|
+
function apisOf(kind, protocol) {
|
|
129
|
+
return kind === 'vertex' && protocol === 'openai' ? ['chat'] : KINDS[kind].apis;
|
|
130
|
+
}
|
|
117
131
|
// ---------------------------------------------------------------------------
|
|
118
132
|
// Tuning knobs
|
|
119
133
|
// ---------------------------------------------------------------------------
|
|
@@ -214,7 +228,11 @@ export class ModelRegistry {
|
|
|
214
228
|
this.#providers.set(name, spec);
|
|
215
229
|
// Re-declaring after a client was handed out would otherwise leave the
|
|
216
230
|
// old credentials in service for the rest of the process.
|
|
217
|
-
this.#clients.
|
|
231
|
+
for (const key of this.#clients.keys()) {
|
|
232
|
+
if (key === name || key.startsWith(`${name}\0`)) {
|
|
233
|
+
this.#clients.delete(key);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
218
236
|
this.#limiters.delete(name);
|
|
219
237
|
return this;
|
|
220
238
|
}
|
|
@@ -247,6 +265,17 @@ export class ModelRegistry {
|
|
|
247
265
|
this.#clients.set(name, built);
|
|
248
266
|
return built;
|
|
249
267
|
}
|
|
268
|
+
/** A Vertex provider's Claude or partner client: built once, beside its GenAI one. */
|
|
269
|
+
#partner(name, protocol, spec) {
|
|
270
|
+
const key = `${name}\0${protocol}`;
|
|
271
|
+
const existing = this.#clients.get(key);
|
|
272
|
+
if (existing) {
|
|
273
|
+
return existing;
|
|
274
|
+
}
|
|
275
|
+
const built = buildVertexPartner(name, protocol, spec);
|
|
276
|
+
this.#clients.set(key, built);
|
|
277
|
+
return built;
|
|
278
|
+
}
|
|
250
279
|
/**
|
|
251
280
|
* The pacing shared by every embedder on a provider, so that what one of
|
|
252
281
|
* them learns from a 429 the others do not have to learn again — and so
|
|
@@ -275,14 +304,16 @@ export class ModelRegistry {
|
|
|
275
304
|
*/
|
|
276
305
|
knobs(ref) {
|
|
277
306
|
const spec = typeof ref === 'string' ? this.parse(ref) : ref;
|
|
278
|
-
const
|
|
279
|
-
|
|
280
|
-
|
|
307
|
+
const kind = this.kindOf(spec.provider ?? this.#default);
|
|
308
|
+
const protocol = protocolOf(kind, spec.model);
|
|
309
|
+
if (protocol !== 'openai') {
|
|
310
|
+
return unreadKnobs(spec, protocol);
|
|
281
311
|
}
|
|
282
312
|
// An api this vendor does not speak already has an error of its own;
|
|
283
313
|
// answering about its knobs would only bury it.
|
|
284
|
-
const
|
|
285
|
-
|
|
314
|
+
const apis = apisOf(kind, protocol);
|
|
315
|
+
const api = spec.api ?? apis[0];
|
|
316
|
+
return api && apis.includes(api) ? unreadKnobs(spec, `openai/${api}`) : [];
|
|
286
317
|
}
|
|
287
318
|
/** Turns a shorthand or a spec into a `Model`. */
|
|
288
319
|
model(ref) {
|
|
@@ -290,7 +321,7 @@ export class ModelRegistry {
|
|
|
290
321
|
const name = spec.provider ?? this.#default;
|
|
291
322
|
const provider = this.#spec(name);
|
|
292
323
|
const kind = this.kindOf(name);
|
|
293
|
-
const
|
|
324
|
+
const protocol = protocolOf(kind, spec.model);
|
|
294
325
|
// Before the client, so a knob for the wrong vendor is reported as the
|
|
295
326
|
// config mistake it is rather than behind a missing credential.
|
|
296
327
|
const [stray] = this.knobs(spec);
|
|
@@ -302,30 +333,35 @@ export class ModelRegistry {
|
|
|
302
333
|
// different connection, and handing back the memoized one would
|
|
303
334
|
// silently ignore them.
|
|
304
335
|
const client = spec.client ??
|
|
305
|
-
(
|
|
336
|
+
(protocol !== KINDS[kind].protocol
|
|
337
|
+
? this.#partner(name, protocol, provider)
|
|
338
|
+
: hasCredentials(spec)
|
|
339
|
+
? buildClient(name, kind, spec, provider)
|
|
340
|
+
: this.client(name));
|
|
306
341
|
// The *resolved* name, not `spec.provider`: a bare ref carries none,
|
|
307
342
|
// and a failure that cannot say which connection refused it is the
|
|
308
343
|
// thing this is here to prevent.
|
|
309
344
|
const options = { ...spec, provider: name };
|
|
310
345
|
// Every protocol but OpenAI's has exactly one API, so naming one is a
|
|
311
346
|
// mistake worth reporting rather than a field to ignore.
|
|
312
|
-
if (
|
|
347
|
+
if (protocol !== 'openai') {
|
|
313
348
|
if (spec.api) {
|
|
314
349
|
throw new TypeError(`provider "${name}" (${kind}) has one api, so "${spec.api}" means nothing here`);
|
|
315
350
|
}
|
|
316
|
-
switch (
|
|
351
|
+
switch (protocol) {
|
|
317
352
|
case 'anthropic':
|
|
318
|
-
return new AnthropicModel(spec.model, client, options);
|
|
353
|
+
return new AnthropicModel(spec.model.replace(/^anthropic\//, ''), client, options);
|
|
319
354
|
case 'gemini':
|
|
320
355
|
return new GeminiModel(spec.model, client, options);
|
|
321
356
|
case 'openrouter':
|
|
322
357
|
return new OpenRouterModel(spec.model, client, options);
|
|
323
358
|
}
|
|
324
359
|
}
|
|
325
|
-
const
|
|
326
|
-
|
|
360
|
+
const apis = apisOf(kind, protocol);
|
|
361
|
+
const api = spec.api ?? apis[0];
|
|
362
|
+
if (!apis.includes(api)) {
|
|
327
363
|
throw new TypeError(`provider "${name}" (${kind}) does not speak the "${api}" api ` +
|
|
328
|
-
`(supported: ${
|
|
364
|
+
`(supported: ${apis.join(', ')})`);
|
|
329
365
|
}
|
|
330
366
|
return api === 'responses'
|
|
331
367
|
? new OpenAIResponsesModel(spec.model, client, options)
|
|
@@ -658,6 +694,109 @@ function projectFromKeyFile() {
|
|
|
658
694
|
return undefined;
|
|
659
695
|
}
|
|
660
696
|
}
|
|
697
|
+
// ---------------------------------------------------------------------------
|
|
698
|
+
// Vertex partner models
|
|
699
|
+
// ---------------------------------------------------------------------------
|
|
700
|
+
/** Gemini has these; the partner endpoints do not, and answer them with a bare 404. */
|
|
701
|
+
const MULTI_REGIONS = new Set(['us', 'eu']);
|
|
702
|
+
/**
|
|
703
|
+
* Where a Vertex provider's partner models are served, and how to authenticate
|
|
704
|
+
* there. The GenAI SDK does both for Gemini; nothing does them for Claude or
|
|
705
|
+
* the OpenAI-compatible endpoint, so they are resolved here the same way:
|
|
706
|
+
* `project`, then GOOGLE_CLOUD_PROJECT, then the service-account file.
|
|
707
|
+
*/
|
|
708
|
+
export function vertexEndpoint(spec = {}, name = 'vertex') {
|
|
709
|
+
const where = `provider "${name}"`;
|
|
710
|
+
const project = expand(spec.project, `${where}: project`) ??
|
|
711
|
+
fromEnv('GOOGLE_CLOUD_PROJECT') ??
|
|
712
|
+
projectFromKeyFile();
|
|
713
|
+
if (!project) {
|
|
714
|
+
throw new Error(`${where}: Claude and partner models on vertex need \`project\`, or ` +
|
|
715
|
+
`GOOGLE_CLOUD_PROJECT, and a service account — an express-mode key reaches Gemini only`);
|
|
716
|
+
}
|
|
717
|
+
const named = expand(spec.location, `${where}: location`) ?? fromEnv('GOOGLE_CLOUD_LOCATION') ?? 'global';
|
|
718
|
+
const location = MULTI_REGIONS.has(named) ? 'global' : named;
|
|
719
|
+
const host = location === 'global'
|
|
720
|
+
? 'https://aiplatform.googleapis.com'
|
|
721
|
+
: `https://${location}-aiplatform.googleapis.com`;
|
|
722
|
+
const token = spec.token;
|
|
723
|
+
return {
|
|
724
|
+
project,
|
|
725
|
+
location,
|
|
726
|
+
base: `${host}/v1/projects/${project}/locations/${location}`,
|
|
727
|
+
token: token ? async () => token() : adcToken(),
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
/** Mints and refreshes tokens the way the GenAI SDK does for Gemini. */
|
|
731
|
+
function adcToken() {
|
|
732
|
+
let auth;
|
|
733
|
+
return async () => {
|
|
734
|
+
if (!auth) {
|
|
735
|
+
// `@google/genai`'s own dependency, so it is resolved from there rather than declared twice.
|
|
736
|
+
const genai = sdkPath('@google/genai', 'vertex');
|
|
737
|
+
const { GoogleAuth } = createRequire(genai)('google-auth-library');
|
|
738
|
+
auth = new GoogleAuth({ scopes: ['https://www.googleapis.com/auth/cloud-platform'] });
|
|
739
|
+
}
|
|
740
|
+
const token = await auth.getAccessToken();
|
|
741
|
+
if (!token) {
|
|
742
|
+
throw new Error('vertex: Application Default Credentials gave no access token');
|
|
743
|
+
}
|
|
744
|
+
return token;
|
|
745
|
+
};
|
|
746
|
+
}
|
|
747
|
+
function sdkPath(pkg, kind) {
|
|
748
|
+
try {
|
|
749
|
+
return requirePeer.resolve(pkg);
|
|
750
|
+
}
|
|
751
|
+
catch (cause) {
|
|
752
|
+
throw new Error(`provider kind "${kind}" needs ${pkg}, which is not installed — run: npm i ${pkg}`, { cause });
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
function buildVertexPartner(name, protocol, opts) {
|
|
756
|
+
const at = vertexEndpoint(opts, name);
|
|
757
|
+
const common = {
|
|
758
|
+
apiKey: OAUTH_PLACEHOLDER,
|
|
759
|
+
defaultHeaders: expandHeaders(opts.headers, `provider "${name}"`),
|
|
760
|
+
timeout: opts.timeoutMs,
|
|
761
|
+
maxRetries: retriesOf(opts),
|
|
762
|
+
};
|
|
763
|
+
if (protocol === 'anthropic') {
|
|
764
|
+
const { Anthropic } = sdk('@anthropic-ai/sdk', 'vertex');
|
|
765
|
+
return new Anthropic({
|
|
766
|
+
...common,
|
|
767
|
+
baseURL: at.base,
|
|
768
|
+
fetch: funded(rawPredictFetch(at.base, at.token)),
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
const { OpenAI } = sdk('openai', 'vertex');
|
|
772
|
+
return new OpenAI({
|
|
773
|
+
...common,
|
|
774
|
+
baseURL: `${at.base}/endpoints/openapi`,
|
|
775
|
+
fetch: funded(bearerFetch(at.token)),
|
|
776
|
+
});
|
|
777
|
+
}
|
|
778
|
+
/**
|
|
779
|
+
* Anthropic's client posts to `/v1/messages`; Vertex wants the model in the
|
|
780
|
+
* path and its own `anthropic_version` in the body instead.
|
|
781
|
+
*/
|
|
782
|
+
function rawPredictFetch(base, token) {
|
|
783
|
+
const authed = bearerFetch(token);
|
|
784
|
+
return async (input, init) => {
|
|
785
|
+
const url = input instanceof Request ? input.url : String(input);
|
|
786
|
+
if (!new URL(url).pathname.endsWith('/v1/messages') || typeof init?.body !== 'string') {
|
|
787
|
+
return authed(input, init);
|
|
788
|
+
}
|
|
789
|
+
const { model, ...body } = JSON.parse(init.body);
|
|
790
|
+
const headers = new Headers(init.headers);
|
|
791
|
+
headers.delete('x-api-key');
|
|
792
|
+
const verb = body.stream ? 'streamRawPredict' : 'rawPredict';
|
|
793
|
+
return authed(`${base}/publishers/anthropic/models/${model}:${verb}`, {
|
|
794
|
+
...init,
|
|
795
|
+
headers,
|
|
796
|
+
body: JSON.stringify({ ...body, anthropic_version: 'vertex-2023-10-16' }),
|
|
797
|
+
});
|
|
798
|
+
};
|
|
799
|
+
}
|
|
661
800
|
// Both SDKs refuse to construct without a key. When a token callback owns the
|
|
662
801
|
// Authorization header this value is never sent, so it only has to be non-empty.
|
|
663
802
|
const OAUTH_PLACEHOLDER = 'oauth';
|
package/dist/runner.js
CHANGED
|
@@ -449,6 +449,13 @@ export class AgentRunner {
|
|
|
449
449
|
catch (e) {
|
|
450
450
|
return fail(`error: arguments are not valid JSON: ${String(e)}`);
|
|
451
451
|
}
|
|
452
|
+
if (typeof args !== 'object' || args === null || Array.isArray(args)) {
|
|
453
|
+
return fail(`error: arguments to ${def.name} must be a JSON object`);
|
|
454
|
+
}
|
|
455
|
+
const missing = (def.parameters.required ?? []).filter((k) => args[k] == null);
|
|
456
|
+
if (missing.length) {
|
|
457
|
+
return fail(`error: ${def.name} is missing required arguments: ${missing.join(', ')}`);
|
|
458
|
+
}
|
|
452
459
|
try {
|
|
453
460
|
const out = await def.execute(args, {
|
|
454
461
|
ctx: state.context,
|
package/dist/tools/sandbox.js
CHANGED
|
@@ -243,7 +243,10 @@ export class Sandbox {
|
|
|
243
243
|
this.#created = true;
|
|
244
244
|
return;
|
|
245
245
|
}
|
|
246
|
-
await this.#
|
|
246
|
+
const made = await this.#podman(this.#createArgs());
|
|
247
|
+
if (made.code !== 0) {
|
|
248
|
+
throw new SandboxError(`could not create container ${this.name}: ${message(made)}`, LOCKS.test(made.stderr) ? locksHint(this.spec.engine) : undefined);
|
|
249
|
+
}
|
|
247
250
|
this.#created = true;
|
|
248
251
|
}
|
|
249
252
|
#createArgs() {
|
|
@@ -320,7 +323,13 @@ export class Sandbox {
|
|
|
320
323
|
const cwd = this.inside(opts.cwd);
|
|
321
324
|
const timeoutMs = this.#timeout(opts.timeout);
|
|
322
325
|
const startedAt = Date.now();
|
|
323
|
-
const
|
|
326
|
+
const once = () => this.#run(this.spec.engine, ['exec', '--interactive', '--workdir', cwd, this.name, '/bin/sh', '-s'], { input: command, timeoutMs, signal: opts.signal });
|
|
327
|
+
let res = await once();
|
|
328
|
+
if (gone(res)) {
|
|
329
|
+
// podman refused before running anything, so the retry cannot run the command twice.
|
|
330
|
+
await this.#revive(res);
|
|
331
|
+
res = await once();
|
|
332
|
+
}
|
|
324
333
|
return {
|
|
325
334
|
exit_code: res.code,
|
|
326
335
|
stdout: res.stdout,
|
|
@@ -527,6 +536,18 @@ export class Sandbox {
|
|
|
527
536
|
#podman(args) {
|
|
528
537
|
return this.#run(this.spec.engine, args, { timeoutMs: 120_000 });
|
|
529
538
|
}
|
|
539
|
+
/** The container stopped or vanished under us (machine slept, restarted, or someone removed it). */
|
|
540
|
+
async #revive(res) {
|
|
541
|
+
this.#ready = undefined;
|
|
542
|
+
try {
|
|
543
|
+
await this.start();
|
|
544
|
+
}
|
|
545
|
+
catch (err) {
|
|
546
|
+
throw new SandboxError(`the sandbox container ${this.name} stopped (${message(res)}) and could not be restarted: ` +
|
|
547
|
+
(err instanceof Error ? err.message : String(err)), `the ${this.spec.engine} machine may have stopped or restarted — check \`${this.spec.engine} machine list\`, ` +
|
|
548
|
+
`start it with \`${this.spec.engine} machine start\`, then retry`);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
530
551
|
async #must(args, what) {
|
|
531
552
|
const res = await this.#podman(args);
|
|
532
553
|
if (res.code !== 0) {
|
|
@@ -541,6 +562,23 @@ function mount(m) {
|
|
|
541
562
|
function message(res) {
|
|
542
563
|
return (res.stderr.trim() || res.stdout.trim() || `exit ${res.code}`).split('\n')[0];
|
|
543
564
|
}
|
|
565
|
+
/** podman's words for "every lock is taken": `allocating lock … exceeded num_locks (2048)`. */
|
|
566
|
+
const LOCKS = /exceeded num_locks|allocating lock/i;
|
|
567
|
+
/**
|
|
568
|
+
* A stopped container holds a lock as surely as a running one, and a
|
|
569
|
+
* persisted sandbox is stopped, never removed — so they pile up one per
|
|
570
|
+
* session until the engine can create nothing at all, for anyone.
|
|
571
|
+
*/
|
|
572
|
+
function locksHint(engine) {
|
|
573
|
+
return (`the ${engine} engine has no locks left: every container it holds, stopped ones ` +
|
|
574
|
+
`included, takes one. Remove the stopped sandbox containers with ` +
|
|
575
|
+
`\`${engine} container prune --force --filter label=zenera=1\`, then retry`);
|
|
576
|
+
}
|
|
577
|
+
/** podman's own exit code for "could not exec at all", with the two reasons that mean the container is not there to exec in. */
|
|
578
|
+
function gone(res) {
|
|
579
|
+
return (res.code === 125 &&
|
|
580
|
+
/can only create exec sessions on running containers|container state improper|no such container/i.test(res.stderr));
|
|
581
|
+
}
|
|
544
582
|
/** Only ever applied to paths this module derived, never to a model's text. */
|
|
545
583
|
function quote(s) {
|
|
546
584
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
package/package.json
CHANGED
package/dist/memory/match.d.ts
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
/** Long enough for any honest pattern, short enough to bound a bad one. */
|
|
2
|
-
export declare const MAX_PATTERN = 200;
|
|
3
|
-
export declare class PatternError extends Error {
|
|
4
|
-
}
|
|
5
|
-
export interface MatchOptions {
|
|
6
|
-
/** read the pattern as a regular expression rather than as a literal */
|
|
7
|
-
regex?: boolean;
|
|
8
|
-
caseSensitive?: boolean;
|
|
9
|
-
}
|
|
10
|
-
/** The offset of the first match in `text`, or -1. */
|
|
11
|
-
export type Matcher = (text: string) => number;
|
|
12
|
-
/**
|
|
13
|
-
* A finder over a string. Literal by default: someone typing `user.id` means
|
|
14
|
-
* those seven characters, and a dot that quietly matched anything would be a
|
|
15
|
-
* worse answer than no answer.
|
|
16
|
-
*/
|
|
17
|
-
export declare function matcher(pattern: string, options?: MatchOptions): Matcher;
|
|
18
|
-
//# sourceMappingURL=match.d.ts.map
|
package/dist/memory/match.js
DELETED
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
// ---------------------------------------------------------------------------
|
|
2
|
-
// Matching, with nothing learned in between
|
|
3
|
-
//
|
|
4
|
-
// Everything here is exact. A substring is a substring, which is the whole
|
|
5
|
-
// point of the surface built on it: the vector index answers "what is near
|
|
6
|
-
// this", and near is a ranking, so it can only ever return the top of a list.
|
|
7
|
-
// When the question is "does the word `password` appear anywhere at all", a
|
|
8
|
-
// ranking is the wrong instrument and no amount of tuning makes it the right
|
|
9
|
-
// one.
|
|
10
|
-
//
|
|
11
|
-
// This is a copy of `common/match.ts` in @zenera/rag, and has to be: the
|
|
12
|
-
// dependency runs rag → cli → neo, so neo importing rag would close a cycle.
|
|
13
|
-
// Two differences earn their place rather than being drift. A matcher returns
|
|
14
|
-
// the offset it matched at, because grep reports *where*, not merely whether;
|
|
15
|
-
// and there are no globs, because the haystack here is prose and a glob is
|
|
16
|
-
// anchored to the whole string, so `error*` would match a line only when the
|
|
17
|
-
// line began with the word.
|
|
18
|
-
//
|
|
19
|
-
// A pattern may arrive from a model, so a regex is a bounded promise: the
|
|
20
|
-
// length is capped here and the scan that uses it keeps a deadline.
|
|
21
|
-
// ---------------------------------------------------------------------------
|
|
22
|
-
/** Long enough for any honest pattern, short enough to bound a bad one. */
|
|
23
|
-
export const MAX_PATTERN = 200;
|
|
24
|
-
export class PatternError extends Error {
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* A finder over a string. Literal by default: someone typing `user.id` means
|
|
28
|
-
* those seven characters, and a dot that quietly matched anything would be a
|
|
29
|
-
* worse answer than no answer.
|
|
30
|
-
*/
|
|
31
|
-
export function matcher(pattern, options = {}) {
|
|
32
|
-
guard(pattern);
|
|
33
|
-
if (!options.regex) {
|
|
34
|
-
if (options.caseSensitive) {
|
|
35
|
-
return (text) => text.indexOf(pattern);
|
|
36
|
-
}
|
|
37
|
-
const needle = pattern.toLowerCase();
|
|
38
|
-
return (text) => text.toLowerCase().indexOf(needle);
|
|
39
|
-
}
|
|
40
|
-
const expression = compile(pattern, options.caseSensitive ? '' : 'i');
|
|
41
|
-
// `lastIndex` is not carried between calls: the flags never include `g`.
|
|
42
|
-
return (text) => text.search(expression);
|
|
43
|
-
}
|
|
44
|
-
// ---------------------------------------------------------------------------
|
|
45
|
-
function guard(pattern) {
|
|
46
|
-
if (pattern.length === 0) {
|
|
47
|
-
throw new PatternError('the pattern is empty');
|
|
48
|
-
}
|
|
49
|
-
if (pattern.length > MAX_PATTERN) {
|
|
50
|
-
throw new PatternError(`the pattern is longer than ${MAX_PATTERN} characters`);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
function compile(source, flags) {
|
|
54
|
-
try {
|
|
55
|
-
return new RegExp(source, flags);
|
|
56
|
-
}
|
|
57
|
-
catch (err) {
|
|
58
|
-
throw new PatternError(`invalid pattern: ${err.message}`);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
//# sourceMappingURL=match.js.map
|