@hexdecimal16/nexo 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/cli/src/main.js
CHANGED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { release, type } from "node:os";
|
|
3
|
+
function git(cwd, args) {
|
|
4
|
+
try {
|
|
5
|
+
return execFileSync("git", args, { cwd, encoding: "utf8", timeout: 2000, stdio: ["ignore", "pipe", "ignore"] }).trim();
|
|
6
|
+
}
|
|
7
|
+
catch {
|
|
8
|
+
return null;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
/** The current branch, or `detached at <sha>` on a detached HEAD; null outside a git repo. */
|
|
12
|
+
export function gitBranch(cwd) {
|
|
13
|
+
if (git(cwd, ["rev-parse", "--is-inside-work-tree"]) !== "true")
|
|
14
|
+
return null;
|
|
15
|
+
const branch = git(cwd, ["branch", "--show-current"]);
|
|
16
|
+
if (branch)
|
|
17
|
+
return branch;
|
|
18
|
+
const sha = git(cwd, ["rev-parse", "--short", "HEAD"]);
|
|
19
|
+
return sha ? `detached at ${sha}` : "unknown";
|
|
20
|
+
}
|
|
21
|
+
/** Local time as `YYYY-MM-DD HH:mm <zone> (UTC±hh:mm)`. */
|
|
22
|
+
export function localTime(now) {
|
|
23
|
+
const pad = (n) => String(Math.abs(n)).padStart(2, "0");
|
|
24
|
+
const offset = -now.getTimezoneOffset();
|
|
25
|
+
const sign = offset >= 0 ? "+" : "-";
|
|
26
|
+
const zone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
27
|
+
return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())} ${zone} (UTC${sign}${pad(Math.trunc(offset / 60))}:${pad(offset % 60)})`;
|
|
28
|
+
}
|
|
29
|
+
/** The prompt's environment section: where nexo was launched, git state, platform, shell and the current local time. */
|
|
30
|
+
export function environmentBlock(input) {
|
|
31
|
+
const lines = ["Environment:"];
|
|
32
|
+
if (input.projectDir) {
|
|
33
|
+
const branch = gitBranch(input.projectDir);
|
|
34
|
+
lines.push(`- Launch directory: ${input.projectDir}`);
|
|
35
|
+
lines.push(`- Git repository: ${branch !== null}`);
|
|
36
|
+
if (branch !== null)
|
|
37
|
+
lines.push(`- Git branch: ${branch}`);
|
|
38
|
+
lines.push(input.workspaceShared
|
|
39
|
+
? "- The workspace is a shared directory that runs edit in place."
|
|
40
|
+
: "- The user's project files live in the launch directory. The workspace is a scratch copy for files you create; edit project files by absolute path in the launch directory.");
|
|
41
|
+
}
|
|
42
|
+
lines.push(`- Platform: ${process.platform} (${type()} ${release()})`);
|
|
43
|
+
if (input.shell)
|
|
44
|
+
lines.push(`- Shell: ${input.shell}`);
|
|
45
|
+
lines.push(`- Current local time: ${localTime(input.now)}`);
|
|
46
|
+
return lines.join("\n");
|
|
47
|
+
}
|
|
@@ -30,5 +30,8 @@ export function renderSystemPrompt(parts) {
|
|
|
30
30
|
sections.push(parts.projectInstructions);
|
|
31
31
|
if (parts.memories)
|
|
32
32
|
sections.push(parts.memories);
|
|
33
|
+
// Last: it holds the current time, which changes every run and would otherwise break prefix caching for what follows.
|
|
34
|
+
if (parts.environment)
|
|
35
|
+
sections.push(parts.environment);
|
|
33
36
|
return sections.join("\n\n");
|
|
34
37
|
}
|
package/core/src/engine/run.js
CHANGED
|
@@ -16,6 +16,7 @@ import { assistantText, contentText, estimatePromptChars, foreignToolCallsAsText
|
|
|
16
16
|
import { DEFAULT_PLANNER_PROMPT, DEFAULT_SYSTEM_PROMPT, renderSystemPrompt } from "./prompts.js";
|
|
17
17
|
import { loadProjectInstructions } from "../session/project-instructions.js";
|
|
18
18
|
import { projectScope } from "../session/project-root.js";
|
|
19
|
+
import { environmentBlock } from "./environment.js";
|
|
19
20
|
import { createGatedStreamFn } from "./provider-gate.js";
|
|
20
21
|
import { Recorder } from "./recorder.js";
|
|
21
22
|
import { activePlaybook, emptyMemoryContext, playbookSection, retrieveRunMemories, skillAppendix } from "../learn/context.js";
|
|
@@ -581,11 +582,6 @@ export async function executeRun(env, input) {
|
|
|
581
582
|
};
|
|
582
583
|
for (const factory of extraToolFactories)
|
|
583
584
|
baseTools.push(...factory(toolCtx));
|
|
584
|
-
if (deps.mcp)
|
|
585
|
-
baseTools.push(...(await deps.mcp.tools()));
|
|
586
|
-
const tools = baseTools
|
|
587
|
-
.map((t) => (toolDescriptionOverrides.has(t.name) ? { ...t, description: toolDescriptionOverrides.get(t.name) } : t))
|
|
588
|
-
.map(withToolErrorCodes);
|
|
589
585
|
// Retrieval at run start (spec §8.3); the query embedding is charged to this run like any of its calls.
|
|
590
586
|
memoryDeps = {
|
|
591
587
|
env,
|
|
@@ -599,12 +595,20 @@ export async function executeRun(env, input) {
|
|
|
599
595
|
costUnknown ||= unknown;
|
|
600
596
|
},
|
|
601
597
|
};
|
|
598
|
+
// Retrieval starts before the MCP tools load, so the query embedding runs during that wait.
|
|
599
|
+
const retrieval = !input.memoryContext && !input.signal?.aborted ? retrieveRunMemories(memoryDeps, request.instruction, ["user", "global", ...(request.projectDir ? [projectScope(request.projectDir)] : []), ...(skill ? [`skill:${skill.name}`] : [])]) : null;
|
|
600
|
+
retrieval?.catch(() => undefined); // awaited below; a rejection while the MCP tools load is not unhandled
|
|
601
|
+
if (deps.mcp)
|
|
602
|
+
baseTools.push(...(await deps.mcp.tools()));
|
|
603
|
+
const tools = baseTools
|
|
604
|
+
.map((t) => (toolDescriptionOverrides.has(t.name) ? { ...t, description: toolDescriptionOverrides.get(t.name) } : t))
|
|
605
|
+
.map(withToolErrorCodes);
|
|
602
606
|
if (input.memoryContext) {
|
|
603
607
|
// load_skill adds to `ids`, so the run works on its own copy.
|
|
604
608
|
memoryContext = { ...input.memoryContext, ids: new Set(input.memoryContext.ids) };
|
|
605
609
|
}
|
|
606
|
-
else if (
|
|
607
|
-
memoryContext = await
|
|
610
|
+
else if (retrieval) {
|
|
611
|
+
memoryContext = await retrieval;
|
|
608
612
|
input.onMemoryContext?.({ ...memoryContext, ids: new Set(memoryContext.ids) });
|
|
609
613
|
}
|
|
610
614
|
// The run directory keeps the memories the run started with (spec §5.1 `context/`).
|
|
@@ -616,6 +620,7 @@ export async function executeRun(env, input) {
|
|
|
616
620
|
workspace,
|
|
617
621
|
...(rollout ? {} : { wikisDir: deps.paths.wikis }),
|
|
618
622
|
skillIndex: formatSkillIndex(deps.skills),
|
|
623
|
+
...(rollout ? {} : { environment: environmentBlock({ ...(request.projectDir ? { projectDir: request.projectDir } : {}), now: new Date(), workspaceShared: sharedWorkspace !== null, ...(process.env.SHELL ? { shell: process.env.SHELL } : {}) }) }),
|
|
619
624
|
...(request.projectDir && !rollout ? { projectInstructions: loadProjectInstructions(request.projectDir, config.instructions).block } : {}),
|
|
620
625
|
activeSkill: runSkill
|
|
621
626
|
? { name: runSkill.name, body: startPlaybookSection ? `${runSkill.body}\n\n${startPlaybookSection}` : runSkill.body }
|
package/core/src/learn/caller.js
CHANGED
|
@@ -131,7 +131,7 @@ export async function structuredLearnCall(deps, schema, request) {
|
|
|
131
131
|
* Embeds `texts` as one `embedding_call` span (tier `embedding`): the estimate is reserved from `deps.budget`, the key
|
|
132
132
|
* comes from `resolveApiKey(provider)`, and the cost comes from `pricingOverrides` (unknown without one).
|
|
133
133
|
*/
|
|
134
|
-
export async function embedCharged(deps, endpoint, texts, purpose) {
|
|
134
|
+
export async function embedCharged(deps, endpoint, texts, purpose, timeoutMs) {
|
|
135
135
|
if (texts.length === 0)
|
|
136
136
|
return [];
|
|
137
137
|
const { env, recorder, budget } = deps;
|
|
@@ -151,7 +151,7 @@ export async function embedCharged(deps, endpoint, texts, purpose) {
|
|
|
151
151
|
});
|
|
152
152
|
let settled = false;
|
|
153
153
|
try {
|
|
154
|
-
const result = await embedTexts(endpoint, texts, { apiKey: env.deps.resolveApiKey?.(endpoint.provider), fetch: deps.fetch, signal: deps.signal });
|
|
154
|
+
const result = await embedTexts(endpoint, texts, { apiKey: env.deps.resolveApiKey?.(endpoint.provider), fetch: deps.fetch, signal: deps.signal, ...(timeoutMs ? { timeoutMs } : {}) });
|
|
155
155
|
const priced = embeddingCost(config, endpoint, result.tokens);
|
|
156
156
|
settled = true;
|
|
157
157
|
if (estimate > 0)
|
|
@@ -4,6 +4,42 @@ import { getActiveArtifact, hasLiveMemories, listPlaybookItems, renderPlaybook }
|
|
|
4
4
|
import { toErrorPayload } from "../../../telemetry/src/index.js";
|
|
5
5
|
import { embedCharged } from "./caller.js";
|
|
6
6
|
import { embeddingEndpoint } from "./embedder.js";
|
|
7
|
+
/**
|
|
8
|
+
* Longest wait for the run-start query embedding. Past it, retrieval falls back to full-text relevance instead of
|
|
9
|
+
* delaying the first model call (observed average: 10s, with the 30s default as the ceiling).
|
|
10
|
+
*/
|
|
11
|
+
export const QUERY_EMBED_TIMEOUT_MS = 3_000;
|
|
12
|
+
/** Most query embeddings kept in memory; least recently used goes first. */
|
|
13
|
+
export const QUERY_EMBED_CACHE_MAX = 64;
|
|
14
|
+
const queryEmbeddings = new Map();
|
|
15
|
+
/** The cache key: the same text sent to the same endpoint, model and dimensions gives the same vector. */
|
|
16
|
+
export function queryEmbeddingKey(endpoint, text) {
|
|
17
|
+
return JSON.stringify([endpoint.model, endpoint.baseUrl, endpoint.dimensions ?? null, text]);
|
|
18
|
+
}
|
|
19
|
+
/** A cached query vector, marked most recently used; undefined on a miss. */
|
|
20
|
+
export function cachedQueryEmbedding(key) {
|
|
21
|
+
const vector = queryEmbeddings.get(key);
|
|
22
|
+
if (vector) {
|
|
23
|
+
queryEmbeddings.delete(key);
|
|
24
|
+
queryEmbeddings.set(key, vector);
|
|
25
|
+
}
|
|
26
|
+
return vector;
|
|
27
|
+
}
|
|
28
|
+
/** Stores a query vector, evicting the least recently used one past QUERY_EMBED_CACHE_MAX. */
|
|
29
|
+
export function rememberQueryEmbedding(key, vector) {
|
|
30
|
+
queryEmbeddings.delete(key);
|
|
31
|
+
queryEmbeddings.set(key, vector);
|
|
32
|
+
while (queryEmbeddings.size > QUERY_EMBED_CACHE_MAX) {
|
|
33
|
+
const oldest = queryEmbeddings.keys().next().value;
|
|
34
|
+
if (oldest === undefined)
|
|
35
|
+
break;
|
|
36
|
+
queryEmbeddings.delete(oldest);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** Empties the query-embedding cache (tests). */
|
|
40
|
+
export function clearQueryEmbeddingCache() {
|
|
41
|
+
queryEmbeddings.clear();
|
|
42
|
+
}
|
|
7
43
|
/** A fresh empty context; `ids` is mutated by load_skill, so it is never shared between runs. */
|
|
8
44
|
export function emptyMemoryContext() {
|
|
9
45
|
return { block: "", ids: new Set(), query: "", embedding: null };
|
|
@@ -23,13 +59,25 @@ export async function retrieveRunMemories(deps, instruction, scopes) {
|
|
|
23
59
|
const spanId = deps.recorder.start({ kind: "memory_read", name: "retrieve", parentSpanId: deps.parentSpanId, attrs: { scopes } });
|
|
24
60
|
let embedding = null;
|
|
25
61
|
let embeddingError = null;
|
|
62
|
+
let embeddingCached = false;
|
|
26
63
|
try {
|
|
27
64
|
const endpoint = embeddingEndpoint(config);
|
|
28
65
|
if (endpoint && instruction.trim() && hasLiveMemories(db, scopes)) {
|
|
66
|
+
const text = instruction.slice(0, 8000);
|
|
67
|
+
const key = queryEmbeddingKey(endpoint, text);
|
|
68
|
+
const cached = cachedQueryEmbedding(key);
|
|
29
69
|
try {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
70
|
+
if (cached) {
|
|
71
|
+
embedding = { model: endpoint.model, vector: cached };
|
|
72
|
+
embeddingCached = true;
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
const [vector] = await embedCharged({ ...deps, parentSpanId: spanId }, endpoint, [text], "query", QUERY_EMBED_TIMEOUT_MS);
|
|
76
|
+
if (vector) {
|
|
77
|
+
embedding = { model: endpoint.model, vector };
|
|
78
|
+
rememberQueryEmbedding(key, vector);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
33
81
|
}
|
|
34
82
|
catch (error) {
|
|
35
83
|
if (deps.signal.aborted)
|
|
@@ -47,6 +95,7 @@ export async function retrieveRunMemories(deps, instruction, scopes) {
|
|
|
47
95
|
candidates: result.candidates,
|
|
48
96
|
selected: result.selected.map((r) => ({ id: r.memory.id, score: Number(r.score.toFixed(4)), relevance: Number(r.relevance.toFixed(4)) })),
|
|
49
97
|
...(embeddingError ? { embeddingError } : {}),
|
|
98
|
+
...(embeddingCached ? { embeddingCached: true } : {}),
|
|
50
99
|
},
|
|
51
100
|
});
|
|
52
101
|
return { block: memoryBlock(memories), ids: new Set(memories.map((m) => m.id)), query: instruction, embedding };
|
package/package.json
CHANGED
package/tools/src/read-only.js
CHANGED
|
@@ -292,7 +292,25 @@ function sortReadOnly(args) {
|
|
|
292
292
|
function findReadOnly(args) {
|
|
293
293
|
return !args.some((a) => FIND_WRITES.has(a));
|
|
294
294
|
}
|
|
295
|
+
/** Dot-commands that only print; any other (`.shell`, `.output`, `.import`, `.read`) can run programs or write files. */
|
|
296
|
+
const SQLITE_SAFE_DOT = /^\.(schema|tables|indexes|indices|databases|headers|mode|timer)\b/;
|
|
297
|
+
/** SQL the sqlite3 shell runs even on a read-only connection: file writes, extensions, other databases. */
|
|
298
|
+
const SQLITE_UNSAFE_SQL = /\b(writefile|readfile|edit|load_extension|fsdir|attach)\b/i;
|
|
299
|
+
const SQLITE_SAFE_FLAGS = new Set(["-readonly", "-header", "-noheader", "-column", "-line", "-list", "-csv", "-json", "-markdown", "-table", "-batch"]);
|
|
300
|
+
/** `sqlite3 -readonly <db> [sql|.schema ...]`: the connection cannot write, and no argument reaches the shell or a file. */
|
|
301
|
+
function sqliteReadOnly(args) {
|
|
302
|
+
if (!args.includes("-readonly"))
|
|
303
|
+
return false;
|
|
304
|
+
return args.every((a) => {
|
|
305
|
+
if (a.startsWith("-"))
|
|
306
|
+
return SQLITE_SAFE_FLAGS.has(a);
|
|
307
|
+
if (a.startsWith("."))
|
|
308
|
+
return SQLITE_SAFE_DOT.test(a);
|
|
309
|
+
return !SQLITE_UNSAFE_SQL.test(a);
|
|
310
|
+
});
|
|
311
|
+
}
|
|
295
312
|
const PROGRAMS = {
|
|
313
|
+
sqlite3: sqliteReadOnly,
|
|
296
314
|
sed: sedReadOnly,
|
|
297
315
|
git: gitReadOnly,
|
|
298
316
|
gh: ghReadOnly,
|