@relay-harness/coding-agent 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -6
- package/LICENSE +22 -0
- package/README.md +12 -13
- package/dist/bundle/chunks/{chunk-XSLAMYCK.js → chunk-5H4IRYOE.js} +555 -61
- package/dist/bundle/chunks/{chunk-H6S6DQTM.js → chunk-NS67SA75.js} +1 -1
- package/dist/bundle/chunks/{virtual-modules-FNVEVB65.js → virtual-modules-ZGB2J7DY.js} +1 -1
- package/dist/bundle/cli-runtime.js +1 -1
- package/dist/bundle/index.js +1 -1
- package/dist/bundle/rpc-entry.js +1 -1
- package/dist/core/settings-manager.d.ts +5 -3
- package/dist/core/settings-manager.d.ts.map +1 -1
- package/dist/core/settings-manager.js.map +1 -1
- package/dist/extensions/laya/assessment.d.ts +7 -1
- package/dist/extensions/laya/assessment.d.ts.map +1 -1
- package/dist/extensions/laya/assessment.js +21 -1
- package/dist/extensions/laya/assessment.js.map +1 -1
- package/dist/extensions/laya/docker.d.ts +19 -0
- package/dist/extensions/laya/docker.d.ts.map +1 -0
- package/dist/extensions/laya/docker.js +44 -0
- package/dist/extensions/laya/docker.js.map +1 -0
- package/dist/extensions/laya/index.d.ts +12 -3
- package/dist/extensions/laya/index.d.ts.map +1 -1
- package/dist/extensions/laya/index.js +331 -67
- package/dist/extensions/laya/index.js.map +1 -1
- package/dist/extensions/laya/learn.d.ts +67 -0
- package/dist/extensions/laya/learn.d.ts.map +1 -0
- package/dist/extensions/laya/learn.js +263 -0
- package/dist/extensions/laya/learn.js.map +1 -0
- package/dist/extensions/laya/memory.d.ts +48 -0
- package/dist/extensions/laya/memory.d.ts.map +1 -0
- package/dist/extensions/laya/memory.js +107 -0
- package/dist/extensions/laya/memory.js.map +1 -0
- package/dist/extensions/laya/model-manifest.d.ts +1 -1
- package/dist/extensions/laya/model-manifest.js +1 -1
- package/dist/extensions/laya/model-manifest.js.map +1 -1
- package/dist/extensions/laya/routers.d.ts +8 -0
- package/dist/extensions/laya/routers.d.ts.map +1 -1
- package/dist/extensions/laya/routers.js +2 -0
- package/dist/extensions/laya/routers.js.map +1 -1
- package/dist/extensions/laya/runtime.d.ts +39 -49
- package/dist/extensions/laya/runtime.d.ts.map +1 -1
- package/dist/extensions/laya/runtime.js +93 -145
- package/dist/extensions/laya/runtime.js.map +1 -1
- package/dist/extensions/laya/server.d.ts +39 -15
- package/dist/extensions/laya/server.d.ts.map +1 -1
- package/dist/extensions/laya/server.js +212 -78
- package/dist/extensions/laya/server.js.map +1 -1
- package/dist/extensions/laya/train-script.d.ts +8 -0
- package/dist/extensions/laya/train-script.d.ts.map +1 -0
- package/dist/extensions/laya/train-script.js +422 -0
- package/dist/extensions/laya/train-script.js.map +1 -0
- package/dist/extensions/laya/training.d.ts +176 -0
- package/dist/extensions/laya/training.d.ts.map +1 -0
- package/dist/extensions/laya/training.js +337 -0
- package/dist/extensions/laya/training.js.map +1 -0
- package/dist/modes/interactive/components/config-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/config-selector.js +3 -2
- package/dist/modes/interactive/components/config-selector.js.map +1 -1
- package/dist/modes/interactive/interactive-mode.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode.js +4 -4
- package/dist/modes/interactive/interactive-mode.js.map +1 -1
- package/dist/utils/version-check.d.ts.map +1 -1
- package/dist/utils/version-check.js +10 -6
- package/dist/utils/version-check.js.map +1 -1
- package/docs/laya.md +59 -19
- package/docs/quickstart.md +19 -6
- package/docs/settings.md +4 -3
- package/examples/extensions/custom-provider-anthropic/package-lock.json +2 -2
- package/examples/extensions/custom-provider-anthropic/package.json +1 -1
- package/examples/extensions/custom-provider-gitlab-duo/package.json +1 -1
- package/examples/extensions/sandbox/package-lock.json +2 -2
- package/examples/extensions/sandbox/package.json +1 -1
- package/examples/extensions/with-deps/package-lock.json +2 -2
- package/examples/extensions/with-deps/package.json +1 -1
- package/package.json +15 -11
|
@@ -1,23 +1,29 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { dirname, join, resolve } from "node:path";
|
|
4
4
|
import { createProvider, } from "@relay-harness/ai";
|
|
5
5
|
import { typesafeSystemOneApi } from "@relay-harness/ai/api/typesafe-system-one.lazy";
|
|
6
6
|
import { getAgentDir } from "../../config.js";
|
|
7
|
-
import {
|
|
7
|
+
import { SessionManager } from "../../core/session-manager.js";
|
|
8
|
+
import { assessmentFromAnswers, assessmentFromLabels, heuristicAssessment } from "./assessment.js";
|
|
9
|
+
import { spawnDocker } from "./docker.js";
|
|
10
|
+
import { buildLearnPrompt, exercisesToRows, LEARN_TOOL_NAME, learnToolSchema, renderQuestions, renderTask, sessionTasks, } from "./learn.js";
|
|
11
|
+
import { LAYA_LESSONS_MESSAGE, LESSON_SIMILARITY, ROUTE_SIMILARITY, renderLessons, TaskMemory } from "./memory.js";
|
|
8
12
|
import { LAYA_MODEL_MANIFEST } from "./model-manifest.js";
|
|
9
13
|
import { applyPolicy, DEFAULT_MODEL_REGISTRY, nextTier, PerformanceHistory, POLICY_PROFILES, parseModelRef, QuotaManager, rankCandidates, taskKey, } from "./policy.js";
|
|
10
14
|
import { CAPABILITY_TIERS, LAYA_QUESTIONS, layaDecisionsFile } from "./questions.js";
|
|
11
|
-
import { renderPlanMessage, selectSkills, unneededTools } from "./routers.js";
|
|
12
|
-
import {
|
|
15
|
+
import { LAYA_PLAN_MESSAGE, renderPlanMessage, selectSkills, unneededTools } from "./routers.js";
|
|
16
|
+
import { layaImage } from "./runtime.js";
|
|
13
17
|
import { generateSeed } from "./seed.js";
|
|
14
18
|
import { isLocalUrl, LayaServer } from "./server.js";
|
|
15
19
|
import { datasetFromTelemetry, HARNESS_MESSAGE, historyFromTelemetry, summarizeRun, TelemetryStore, truncateRequest, } from "./telemetry.js";
|
|
20
|
+
import { accuracy, describeProgress, dockerHasGpu, readRegistry, readRows, SESSION_SOURCE, sharedTrainer, } from "./training.js";
|
|
21
|
+
export { LAYA_PLAN_MESSAGE } from "./routers.js";
|
|
16
22
|
export const LAYA_PROVIDER_ID = "laya";
|
|
17
23
|
export const LAYA_CLASSIFIER_ID = "execution-intelligence";
|
|
18
24
|
export const LAYA_VIRTUAL_MODEL_ID = "auto";
|
|
19
|
-
|
|
20
|
-
export const
|
|
25
|
+
/** Not 8000, the port laya-trainer's own server and many development servers use. */
|
|
26
|
+
export const DEFAULT_LAYA_BASE_URL = "http://127.0.0.1:8737/v1";
|
|
21
27
|
const DEFAULT_TIMEOUT_MS = 5000;
|
|
22
28
|
const DEFAULT_MIN_CONFIDENCE = 0.5;
|
|
23
29
|
const DEFAULT_ESCALATE_AFTER_FAILURES = 3;
|
|
@@ -85,17 +91,49 @@ function failuresThisTurn(messages) {
|
|
|
85
91
|
}
|
|
86
92
|
return { count, evidence };
|
|
87
93
|
}
|
|
88
|
-
|
|
94
|
+
const percent = (score) => `${(accuracy(score) * 100).toFixed(1)}%`;
|
|
95
|
+
/** What a finished training changed, for the user. */
|
|
96
|
+
export function describeTrainingOutcome({ model, activated, previous }) {
|
|
97
|
+
const session = `${model.session.candidate.correct} of ${model.session.candidate.n} answers on the session tasks right (${previous}: ${model.session.current.correct})`;
|
|
98
|
+
const minutes = Math.max(1, Math.round(model.seconds / 60));
|
|
99
|
+
return activated
|
|
100
|
+
? `Laya learned ${model.sessionTasks} session tasks in about ${minutes} min: ${model.name} now routes requests. It gets ${session}; test ${percent(model.test.candidate)} (${previous}: ${percent(model.test.current)}).`
|
|
101
|
+
: `Laya trained ${model.name}, but ${previous} keeps routing: the test score fell from ${percent(model.test.current)} to ${percent(model.test.candidate)}. It gets ${session}. /laya use ${model.name} activates it anyway.`;
|
|
102
|
+
}
|
|
103
|
+
/** Files of the Python runtime Relay used to install on the host, before Laya moved to Docker. */
|
|
104
|
+
const HOST_RUNTIME_LEFTOVERS = ["venv", "models", "serve.py", "train.py", "server.log"];
|
|
105
|
+
export default function layaExtension(relay, options = {}) {
|
|
89
106
|
const quota = new QuotaManager();
|
|
107
|
+
const docker = options.docker ?? spawnDocker;
|
|
90
108
|
const layaHome = join(getAgentDir(), "laya");
|
|
91
109
|
const store = new TelemetryStore(join(layaHome, "telemetry.jsonl"));
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
110
|
+
const baseUrl = () => settings.baseUrl ?? process.env.LAYA_BASE_URL ?? DEFAULT_LAYA_BASE_URL;
|
|
111
|
+
let variant;
|
|
112
|
+
let gpu;
|
|
113
|
+
/** The CUDA image where an NVIDIA GPU exists, the smaller CPU image otherwise. */
|
|
114
|
+
const hostVariant = async () => {
|
|
115
|
+
if (process.platform === "darwin")
|
|
116
|
+
return "cpu";
|
|
117
|
+
try {
|
|
118
|
+
return (await relay.exec("nvidia-smi", ["-L"], { timeout: 10_000 })).code === 0 ? "cuda" : "cpu";
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
return "cpu";
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
const imageVariant = () => {
|
|
125
|
+
variant ??= hostVariant();
|
|
126
|
+
return variant;
|
|
127
|
+
};
|
|
128
|
+
const image = async () => settings.image ?? layaImage(LAYA_MODEL_MANIFEST, await imageVariant());
|
|
129
|
+
const trainingGpu = () => {
|
|
130
|
+
gpu ??= (async () => (await imageVariant()) === "cuda" && dockerHasGpu(docker, await image()))();
|
|
131
|
+
return gpu;
|
|
132
|
+
};
|
|
133
|
+
const trainer = sharedTrainer({ home: layaHome, manifest: LAYA_MODEL_MANIFEST, docker, image, gpu: trainingGpu });
|
|
134
|
+
const server = new LayaServer({ docker, baseUrl, image, model: () => trainer.activeModel().dir });
|
|
135
|
+
/** Server problem last reported, so a failing Docker is not reported on every start. */
|
|
136
|
+
let reportedProblem;
|
|
99
137
|
let history = new PerformanceHistory();
|
|
100
138
|
let settings = {};
|
|
101
139
|
let profileOverride;
|
|
@@ -108,6 +146,11 @@ export default function layaExtension(relay) {
|
|
|
108
146
|
let run;
|
|
109
147
|
/** Tools active before enforced tool routing changed them. */
|
|
110
148
|
let toolBaseline;
|
|
149
|
+
/** Tasks `/laya learn` listed for the agent to label, numbered as the agent saw them. */
|
|
150
|
+
let learnTasks;
|
|
151
|
+
/** This runtime's training listener, while it is the latest one. */
|
|
152
|
+
let listener;
|
|
153
|
+
let memoryCache;
|
|
111
154
|
const refreshSettings = () => {
|
|
112
155
|
settings = relay.getSettings().laya ?? {};
|
|
113
156
|
quota.configure(settings.quota);
|
|
@@ -124,14 +167,47 @@ export default function layaExtension(relay) {
|
|
|
124
167
|
});
|
|
125
168
|
relay.registerProvider(createLayaProvider(() => settings.baseUrl));
|
|
126
169
|
/** System 1: Laya's answers, or keyword rules when the server cannot answer. */
|
|
170
|
+
/** Learned tasks, reloaded when the training dataset changes. Undefined when off or empty. */
|
|
171
|
+
function memory() {
|
|
172
|
+
if (settings.memory === false)
|
|
173
|
+
return undefined;
|
|
174
|
+
let version;
|
|
175
|
+
try {
|
|
176
|
+
const stat = statSync(trainer.workspace.dataset);
|
|
177
|
+
version = `${stat.mtimeMs}:${stat.size}`;
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
if (memoryCache?.version !== version) {
|
|
183
|
+
memoryCache = { version, memory: TaskMemory.fromRows(readRows(trainer.workspace.dataset)) };
|
|
184
|
+
}
|
|
185
|
+
return memoryCache.memory.size > 0 ? memoryCache.memory : undefined;
|
|
186
|
+
}
|
|
187
|
+
/** Lessons of learned tasks similar to a request; none for harness messages such as /laya learn's. */
|
|
188
|
+
const lessonsFor = (request) => HARNESS_MESSAGE.test(request.trimStart())
|
|
189
|
+
? undefined
|
|
190
|
+
: renderLessons(memory()?.search(request, LESSON_SIMILARITY) ?? []);
|
|
191
|
+
/** The labels of a learned task close to the request, then Laya's answers, then keyword rules. */
|
|
127
192
|
async function assess(request, ctx, signal) {
|
|
193
|
+
const learned = memory()?.search(request, ROUTE_SIMILARITY, 1)[0];
|
|
194
|
+
if (learned) {
|
|
195
|
+
try {
|
|
196
|
+
return {
|
|
197
|
+
...assessmentFromLabels(learned.task.expected),
|
|
198
|
+
source: "memory",
|
|
199
|
+
confidence: learned.similarity,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
// Labeled against questions that have changed since; Laya answers instead.
|
|
204
|
+
}
|
|
205
|
+
}
|
|
128
206
|
const classifier = ctx.modelRegistry.findOfType("classifier", LAYA_PROVIDER_ID, LAYA_CLASSIFIER_ID);
|
|
129
|
-
|
|
207
|
+
// While the container installs, or without Docker, the classifier call would only fail.
|
|
208
|
+
if (!classifier || Date.now() < layaUnavailableUntil || server.state === "installing") {
|
|
130
209
|
return heuristicAssessment(request);
|
|
131
|
-
|
|
132
|
-
// classifier call, which fails fast and lands on the keyword rules.
|
|
133
|
-
if (settings.autostart !== false)
|
|
134
|
-
await server.ensureRunning(signal);
|
|
210
|
+
}
|
|
135
211
|
const result = await ctx.modelRegistry.classify(classifier, { state: { request: truncateRequest(request) }, questions: LAYA_QUESTIONS }, { signal, timeoutMs: settings.timeoutMs ?? DEFAULT_TIMEOUT_MS, maxRetries: 0 });
|
|
136
212
|
try {
|
|
137
213
|
if (result.stopReason !== "stop")
|
|
@@ -203,7 +279,7 @@ export default function layaExtension(relay) {
|
|
|
203
279
|
alternatives: ranked.slice(1, 4).map((candidate) => candidate.ref),
|
|
204
280
|
};
|
|
205
281
|
current = state;
|
|
206
|
-
ctx.ui.setStatus("laya", `laya ${profile()}: ${base.assessment.task.type} → ${chosen.tier} ${chosen.model.id} • ${state.thinkingLevel}${
|
|
282
|
+
ctx.ui.setStatus("laya", `laya ${profile()}: ${base.assessment.task.type} → ${chosen.tier} ${chosen.model.id} • ${state.thinkingLevel}${{ laya: "", memory: " (memory)", heuristic: " (rules)" }[base.assessment.source]}`);
|
|
207
283
|
return { route: { model: chosen.model, thinkingLevel: state.thinkingLevel, state }, state };
|
|
208
284
|
}
|
|
209
285
|
async function planTask(request, ctx) {
|
|
@@ -287,33 +363,133 @@ export default function layaExtension(relay) {
|
|
|
287
363
|
name: "Auto (Laya)",
|
|
288
364
|
route,
|
|
289
365
|
});
|
|
290
|
-
/**
|
|
291
|
-
|
|
292
|
-
|
|
366
|
+
/**
|
|
367
|
+
* Pulls the image and starts or updates the server container, with progress in the status line.
|
|
368
|
+
* `explicit` is a user command: it always reports the result. Otherwise only a finished
|
|
369
|
+
* installation and a new problem are reported.
|
|
370
|
+
*/
|
|
371
|
+
async function install(ctx, explicit = false) {
|
|
372
|
+
let worked = false;
|
|
373
|
+
const ok = await server.install((message) => {
|
|
374
|
+
worked = true;
|
|
375
|
+
ctx.ui.setStatus("laya-setup", `laya: ${message}`);
|
|
376
|
+
});
|
|
377
|
+
ctx.ui.setStatus("laya-setup", undefined);
|
|
378
|
+
if (ok) {
|
|
379
|
+
layaUnavailableUntil = 0;
|
|
380
|
+
lastClassifierError = undefined;
|
|
381
|
+
reportedProblem = undefined;
|
|
382
|
+
if (explicit || worked)
|
|
383
|
+
ctx.ui.notify(`Laya is ${server.describe()}; it routes laya/auto requests.`);
|
|
384
|
+
return true;
|
|
385
|
+
}
|
|
386
|
+
const problem = server.describe();
|
|
387
|
+
if (explicit || problem !== reportedProblem) {
|
|
388
|
+
reportedProblem = problem;
|
|
389
|
+
ctx.ui.notify(`Laya ${problem}. Keyword rules route laya/auto requests until it runs.`, "warning");
|
|
390
|
+
}
|
|
391
|
+
return false;
|
|
392
|
+
}
|
|
393
|
+
/** Deletes the Python environment and model that Relay installed on the host before Laya moved to Docker. */
|
|
394
|
+
function removeHostRuntime() {
|
|
395
|
+
for (const name of HOST_RUNTIME_LEFTOVERS) {
|
|
293
396
|
try {
|
|
294
|
-
|
|
295
|
-
exec: (command, args, options) => relay.exec(command, args, options),
|
|
296
|
-
home: layaHome,
|
|
297
|
-
manifest: LAYA_MODEL_MANIFEST,
|
|
298
|
-
python: settings.python,
|
|
299
|
-
signal: ctx.signal,
|
|
300
|
-
report: (message) => ctx.ui.setStatus("laya-setup", `laya setup: ${message}`),
|
|
301
|
-
});
|
|
302
|
-
ctx.ui.notify("Laya is installed. Starting the server on the next request.");
|
|
397
|
+
rmSync(join(layaHome, name), { recursive: true, force: true });
|
|
303
398
|
}
|
|
304
|
-
catch
|
|
305
|
-
|
|
399
|
+
catch {
|
|
400
|
+
// In use or not ours to delete; it does no harm.
|
|
306
401
|
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
/** Capability tier of a `provider/model` in the Laya model registry. */
|
|
405
|
+
const tierOf = (ref) => {
|
|
406
|
+
const registry = { ...DEFAULT_MODEL_REGISTRY, ...settings.models };
|
|
407
|
+
return CAPABILITY_TIERS.find((tier) => registry[tier]?.includes(ref));
|
|
408
|
+
};
|
|
409
|
+
/**
|
|
410
|
+
* Starts training on the collected exercises. It runs in the background in interactive modes and
|
|
411
|
+
* reports through the listener; print and JSON mode wait for it, since the process ends with the turn.
|
|
412
|
+
*/
|
|
413
|
+
async function startTraining(ctx) {
|
|
414
|
+
if (trainer.running)
|
|
415
|
+
return "Laya is already training; run /laya train after it ends to include these tasks.";
|
|
416
|
+
if (server.state !== "ready" && !(await server.install())) {
|
|
417
|
+
return `Laya ${server.describe()}, so it cannot train yet. Run /laya setup, then /laya train.`;
|
|
418
|
+
}
|
|
419
|
+
if (!(await trainingGpu())) {
|
|
420
|
+
const accepted = !ctx.hasUI ||
|
|
421
|
+
(await ctx.ui.confirm("Train Laya on the CPU?", "Docker gives Laya no GPU on this computer. Training still works, but can take an hour or more. Laya keeps routing requests meanwhile."));
|
|
422
|
+
if (!accepted)
|
|
423
|
+
return "Training was not started. Run /laya train when you want it.";
|
|
424
|
+
}
|
|
425
|
+
// The server runs on the CPU, so it keeps routing while the training container has the GPU.
|
|
426
|
+
const job = trainer.train();
|
|
427
|
+
if (ctx.mode === "print" || ctx.mode === "json") {
|
|
428
|
+
try {
|
|
429
|
+
return describeTrainingOutcome(await job);
|
|
310
430
|
}
|
|
311
|
-
|
|
312
|
-
|
|
431
|
+
catch (error) {
|
|
432
|
+
return `Laya training failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
// The listener reports the result.
|
|
436
|
+
job.catch(() => { });
|
|
437
|
+
return "Laya is training in a Docker container (a few minutes with a GPU). The current model keeps routing until it ends; then the new model routes requests if it passes the test.";
|
|
313
438
|
}
|
|
314
|
-
relay.on("session_start", () => {
|
|
439
|
+
relay.on("session_start", (_event, ctx) => {
|
|
315
440
|
refreshSettings();
|
|
316
441
|
history = settings.telemetry === false ? new PerformanceHistory() : historyFromTelemetry(store.read());
|
|
442
|
+
removeHostRuntime();
|
|
443
|
+
// Install right away in the background: pull the image on first use, then keep the container
|
|
444
|
+
// running. Sessions without a UI (print mode, SDK, tests) leave Docker alone.
|
|
445
|
+
if (settings.autostart !== false && ctx.hasUI && isLocalUrl(baseUrl()))
|
|
446
|
+
void install(ctx);
|
|
447
|
+
listener = {
|
|
448
|
+
progress: (progress) => ctx.ui.setStatus("laya-train", `laya training: ${describeProgress(progress)}`),
|
|
449
|
+
finished: (result) => {
|
|
450
|
+
ctx.ui.setStatus("laya-train", undefined);
|
|
451
|
+
if ("error" in result) {
|
|
452
|
+
ctx.ui.notify(`Laya training failed: ${result.error.message}`, "error");
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
ctx.ui.notify(describeTrainingOutcome(result.outcome), result.outcome.activated ? "info" : "warning");
|
|
456
|
+
// The server container is replaced by one serving the new model.
|
|
457
|
+
if (result.outcome.activated)
|
|
458
|
+
void install(ctx);
|
|
459
|
+
},
|
|
460
|
+
};
|
|
461
|
+
trainer.listener = listener;
|
|
462
|
+
});
|
|
463
|
+
relay.registerTool({
|
|
464
|
+
name: LEARN_TOOL_NAME,
|
|
465
|
+
label: "Laya learn",
|
|
466
|
+
description: "Save labeled tasks of this session as training exercises for Laya, Relay's routing model, and train it so similar requests get the right model, effort and tools. Use after /laya learn lists the tasks, or when the user asks Relay to learn from this session or conversation.",
|
|
467
|
+
parameters: learnToolSchema,
|
|
468
|
+
// Loaded by /laya learn, or found by tool search when the user asks in their own words.
|
|
469
|
+
exposure: "deferred",
|
|
470
|
+
executionMode: "sequential",
|
|
471
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
472
|
+
if (!learnTasks) {
|
|
473
|
+
// Called without /laya learn: number this session's tasks and ask for labels against them.
|
|
474
|
+
const tasks = sessionTasks(ctx.sessionManager.getBranch(), tierOf);
|
|
475
|
+
learnTasks = { tasks, session: ctx.sessionManager.getSessionId() };
|
|
476
|
+
throw new Error([
|
|
477
|
+
`Nothing was saved: label these tasks of the session by their numbers, then call ${LEARN_TOOL_NAME} again. Leave out entries that are not coding tasks, such as this request to learn.`,
|
|
478
|
+
`Questions:\n${renderQuestions()}`,
|
|
479
|
+
...tasks.map(renderTask),
|
|
480
|
+
].join("\n\n"));
|
|
481
|
+
}
|
|
482
|
+
const rows = exercisesToRows(learnTasks.tasks, params.exercises, {
|
|
483
|
+
session: learnTasks.session,
|
|
484
|
+
created: new Date().toISOString(),
|
|
485
|
+
});
|
|
486
|
+
const { added, updated } = trainer.addExercises(rows);
|
|
487
|
+
learnTasks = undefined;
|
|
488
|
+
const sessionRows = trainer.counts()[SESSION_SOURCE] ?? 0;
|
|
489
|
+
const saved = `Saved ${rows.length} exercises (${added} new, ${updated} updated); ${sessionRows} session tasks in total, in ${trainer.workspace.dataset}. Requests similar to these tasks use their labels and lessons from the next one.`;
|
|
490
|
+
const training = params.train === false ? "Training was not started (run /laya train)." : await startTraining(ctx);
|
|
491
|
+
return { content: [{ type: "text", text: `${saved} ${training}` }], details: { added, updated, training } };
|
|
492
|
+
},
|
|
317
493
|
});
|
|
318
494
|
/** The user's own tool set: the active tools, or the set enforcement replaced while it is unchanged. */
|
|
319
495
|
const userTools = () => {
|
|
@@ -330,19 +506,18 @@ export default function layaExtension(relay) {
|
|
|
330
506
|
relay.setActiveTools(userTools());
|
|
331
507
|
toolBaseline = undefined;
|
|
332
508
|
}
|
|
333
|
-
if (!isSelected(ctx))
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
await setupRuntime(ctx);
|
|
509
|
+
if (!isSelected(ctx)) {
|
|
510
|
+
// Lessons help whichever model runs the request.
|
|
511
|
+
const lessons = lessonsFor(event.prompt);
|
|
512
|
+
if (!lessons)
|
|
513
|
+
return;
|
|
514
|
+
return {
|
|
515
|
+
message: {
|
|
516
|
+
customType: LAYA_LESSONS_MESSAGE,
|
|
517
|
+
content: `[laya:lessons] ${lessons}`,
|
|
518
|
+
display: false,
|
|
519
|
+
},
|
|
520
|
+
};
|
|
346
521
|
}
|
|
347
522
|
const assessment = await assess(event.prompt, ctx, ctx.signal);
|
|
348
523
|
pending = { request: event.prompt, assessment };
|
|
@@ -358,17 +533,24 @@ export default function layaExtension(relay) {
|
|
|
358
533
|
toolBaseline = { tools: baseline, applied };
|
|
359
534
|
relay.setActiveTools(applied);
|
|
360
535
|
}
|
|
536
|
+
const lessons = lessonsFor(event.prompt);
|
|
537
|
+
const plan = renderPlanMessage(assessment, policy, skills, deactivated);
|
|
361
538
|
return {
|
|
362
539
|
message: {
|
|
363
540
|
customType: LAYA_PLAN_MESSAGE,
|
|
364
|
-
content:
|
|
541
|
+
content: lessons ? `${plan}\n- ${lessons}` : plan,
|
|
365
542
|
display: false,
|
|
366
543
|
details: { assessment, policy, skills, deactivated },
|
|
367
544
|
},
|
|
368
545
|
};
|
|
369
546
|
});
|
|
370
|
-
relay.on("session_shutdown", () => {
|
|
371
|
-
server.
|
|
547
|
+
relay.on("session_shutdown", (event) => {
|
|
548
|
+
// The server container keeps running for the next session.
|
|
549
|
+
if (trainer.listener === listener)
|
|
550
|
+
trainer.listener = undefined;
|
|
551
|
+
// Training belongs to the process: it goes on into the next session, and stops when Relay quits.
|
|
552
|
+
if (event.reason === "quit")
|
|
553
|
+
trainer.stop();
|
|
372
554
|
});
|
|
373
555
|
relay.on("agent_start", () => {
|
|
374
556
|
run = undefined;
|
|
@@ -425,8 +607,21 @@ export default function layaExtension(relay) {
|
|
|
425
607
|
writeFileSync(path, content, "utf8");
|
|
426
608
|
};
|
|
427
609
|
relay.registerCommand("laya", {
|
|
428
|
-
description: "Laya execution router: status,
|
|
429
|
-
getArgumentCompletions: (prefix) => [
|
|
610
|
+
description: "Laya execution router: status, setup, learn from sessions, train, models, policy",
|
|
611
|
+
getArgumentCompletions: (prefix) => [
|
|
612
|
+
"status",
|
|
613
|
+
"setup",
|
|
614
|
+
"start",
|
|
615
|
+
"stop",
|
|
616
|
+
"learn",
|
|
617
|
+
"train",
|
|
618
|
+
"models",
|
|
619
|
+
"use ",
|
|
620
|
+
"policy ",
|
|
621
|
+
"decisions",
|
|
622
|
+
"seed",
|
|
623
|
+
"export",
|
|
624
|
+
]
|
|
430
625
|
.filter((item) => item.startsWith(prefix))
|
|
431
626
|
.map((item) => ({ value: item, label: item.trim() })),
|
|
432
627
|
handler: async (args, ctx) => {
|
|
@@ -469,26 +664,91 @@ export default function layaExtension(relay) {
|
|
|
469
664
|
ctx.ui.notify(`Exported ${rows.length} evidence-labeled exercises to ${path}`);
|
|
470
665
|
return;
|
|
471
666
|
}
|
|
472
|
-
case "
|
|
473
|
-
|
|
667
|
+
case "learn": {
|
|
668
|
+
const file = rest.join(" ");
|
|
669
|
+
let entries = ctx.sessionManager.getBranch();
|
|
670
|
+
let session = ctx.sessionManager.getSessionId();
|
|
671
|
+
if (file) {
|
|
672
|
+
const path = resolve(ctx.cwd, file);
|
|
673
|
+
if (!existsSync(path)) {
|
|
674
|
+
ctx.ui.notify(`No session file at ${path}`, "warning");
|
|
675
|
+
return;
|
|
676
|
+
}
|
|
677
|
+
const manager = SessionManager.open(path);
|
|
678
|
+
entries = manager.getBranch();
|
|
679
|
+
session = manager.getSessionId();
|
|
680
|
+
}
|
|
681
|
+
const tasks = sessionTasks(entries, tierOf);
|
|
682
|
+
if (tasks.length === 0) {
|
|
683
|
+
ctx.ui.notify("The session has no requests to learn from yet.", "warning");
|
|
684
|
+
return;
|
|
685
|
+
}
|
|
686
|
+
learnTasks = { tasks, session };
|
|
687
|
+
const active = relay.getActiveTools();
|
|
688
|
+
if (!active.includes(LEARN_TOOL_NAME))
|
|
689
|
+
relay.setActiveTools([...active, LEARN_TOOL_NAME]);
|
|
690
|
+
const prompt = buildLearnPrompt(tasks, { source: file ? resolve(ctx.cwd, file) : undefined });
|
|
691
|
+
if (ctx.isIdle())
|
|
692
|
+
relay.sendUserMessage(prompt);
|
|
693
|
+
else
|
|
694
|
+
relay.sendUserMessage(prompt, { deliverAs: "followUp" });
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
case "train": {
|
|
698
|
+
ctx.ui.notify(await startTraining(ctx));
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
701
|
+
case "models": {
|
|
702
|
+
const active = trainer.activeModel().name;
|
|
703
|
+
const registry = readRegistry(trainer.workspace);
|
|
704
|
+
const mark = (name) => (name === active ? " (routing)" : "");
|
|
705
|
+
const lines = [`${LAYA_MODEL_MANIFEST.version}: shipped with Relay${mark(LAYA_MODEL_MANIFEST.version)}`];
|
|
706
|
+
for (const model of registry.models) {
|
|
707
|
+
lines.push(`${model.name}: ${model.createdAt.slice(0, 16).replace("T", " ")}, from ${model.basedOn}, ${model.sessionTasks} session tasks, test ${percent(model.test.candidate)} (${model.basedOn}: ${percent(model.test.current)}), session answers ${model.session.candidate.correct}/${model.session.candidate.n}${mark(model.name)}`);
|
|
708
|
+
}
|
|
709
|
+
const counts = trainer.counts();
|
|
710
|
+
lines.push(`Exercises: ${Object.entries(counts)
|
|
711
|
+
.map(([source, count]) => `${count} ${source}`)
|
|
712
|
+
.join(", ") || "none yet"} (${trainer.workspace.dataset})`);
|
|
713
|
+
if (trainer.running) {
|
|
714
|
+
lines.push(`Training: ${trainer.progress ? describeProgress(trainer.progress) : "starting"}`);
|
|
715
|
+
}
|
|
716
|
+
ctx.ui.notify(lines.join("\n"));
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
case "use": {
|
|
720
|
+
const name = rest[0];
|
|
721
|
+
if (!name) {
|
|
722
|
+
ctx.ui.notify(`Usage: /laya use <model> (current: ${trainer.activeModel().name}; see /laya models)`, "warning");
|
|
723
|
+
return;
|
|
724
|
+
}
|
|
725
|
+
try {
|
|
726
|
+
trainer.use(name);
|
|
727
|
+
}
|
|
728
|
+
catch (error) {
|
|
729
|
+
ctx.ui.notify(error instanceof Error ? error.message : String(error), "warning");
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
732
|
+
ctx.ui.notify(`${trainer.activeModel().name} routes requests once the Laya container restarts with it`);
|
|
733
|
+
// Replaces the server container with one serving the selected model.
|
|
734
|
+
await install(ctx, true);
|
|
474
735
|
return;
|
|
475
736
|
}
|
|
737
|
+
case "setup":
|
|
476
738
|
case "start": {
|
|
477
|
-
|
|
478
|
-
? "Laya server is running"
|
|
479
|
-
: "Laya server did not start (see /laya status)", "info");
|
|
739
|
+
await install(ctx, true);
|
|
480
740
|
return;
|
|
481
741
|
}
|
|
482
742
|
case "stop": {
|
|
483
|
-
server.stop();
|
|
484
|
-
ctx.ui.notify("Stopped the Laya
|
|
743
|
+
await server.stop();
|
|
744
|
+
ctx.ui.notify("Stopped the Laya container; /laya start or the next Relay start runs it again");
|
|
485
745
|
return;
|
|
486
746
|
}
|
|
487
747
|
case "status": {
|
|
488
748
|
const providers = new Set(candidates(ctx).map((candidate) => candidate.provider));
|
|
489
749
|
const lines = [
|
|
490
750
|
`Profile: ${profile()}${isSelected(ctx) ? "" : " (select laya/auto to route with Laya)"}`,
|
|
491
|
-
`Laya
|
|
751
|
+
`Laya container: ${server.describe()}; server ${(await server.isUp()) ? "answering" : "not answering"} at ${baseUrl()}`,
|
|
492
752
|
`Laya server: ${lastClassifierError ? `unavailable, using keyword rules (${lastClassifierError})` : "ok or not yet asked"}`,
|
|
493
753
|
`Models with credentials: ${candidates(ctx)
|
|
494
754
|
.map((candidate) => `${candidate.tier}:${candidate.ref}`)
|
|
@@ -496,6 +756,10 @@ export default function layaExtension(relay) {
|
|
|
496
756
|
`Quota: ${Object.entries(quota.snapshot(providers))
|
|
497
757
|
.map(([name, value]) => `${name} ${value}`)
|
|
498
758
|
.join(", ") || "none"}`,
|
|
759
|
+
`Routing model: ${trainer.activeModel().name}${trainer.running
|
|
760
|
+
? `; training ${trainer.progress ? describeProgress(trainer.progress) : "starting"}`
|
|
761
|
+
: ""}`,
|
|
762
|
+
`Memory: ${settings.memory === false ? "off" : `${memory()?.size ?? 0} learned tasks route similar requests and share their lessons`}`,
|
|
499
763
|
`Telemetry: ${settings.telemetry === false ? "off" : store.path}`,
|
|
500
764
|
];
|
|
501
765
|
if (current) {
|
|
@@ -508,7 +772,7 @@ export default function layaExtension(relay) {
|
|
|
508
772
|
return;
|
|
509
773
|
}
|
|
510
774
|
default:
|
|
511
|
-
ctx.ui.notify("Usage: /laya [status|setup|start|stop|policy <profile>|decisions [path]|seed [count] [path]|export [path]]", "warning");
|
|
775
|
+
ctx.ui.notify("Usage: /laya [status|setup|start|stop|learn [session file]|train|models|use <model>|policy <profile>|decisions [path]|seed [count] [path]|export [path]]", "warning");
|
|
512
776
|
}
|
|
513
777
|
},
|
|
514
778
|
});
|