@ddwang/magnitude-core 0.3.1-ddwang.2 → 0.3.1-ddwang.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/browserAgent.js +2 -2
- package/dist/agent/errors.d.ts +2 -0
- package/dist/agent/errors.js +1 -1
- package/dist/agent/index.d.ts +16 -3
- package/dist/agent/index.js +150 -79
- package/dist/ai/modelHarness.js +4 -1
- package/dist/ai/multiModelHarness.js +4 -2
- package/dist/common/events.d.ts +3 -0
- package/dist/common/operation.d.ts +53 -2
- package/dist/common/operation.js +136 -8
- package/dist/common/operation.test.js +37 -1
- package/dist/common/retry.js +2 -2
- package/dist/common/util.js +2 -2
- package/dist/connectors/browserConnector.js +2 -2
- package/dist/index.cjs +325 -125
- package/dist/index.d.cts +52 -9
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +324 -124
- package/dist/memory/image.d.ts +1 -1
- package/dist/memory/image.js +3 -2
- package/dist/memory/rendering/renderJsonParts.js +2 -2
- package/dist/memory/rendering/renderXmlParts.d.ts +1 -1
- package/dist/memory/rendering/renderXmlParts.js +3 -3
- package/dist/web/harness.js +27 -39
- package/dist/web/recovery.d.ts +2 -0
- package/dist/web/visualizer/cursor.d.ts +2 -1
- package/dist/web/visualizer/cursor.js +15 -14
- package/dist/web/visualizer/cursor.test.d.ts +1 -0
- package/dist/web/visualizer/cursor.test.js +63 -0
- package/dist/web/visualizer/index.d.ts +2 -0
- package/dist/web/visualizer/index.js +10 -6
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { setLogLevel } from '@boundaryml/baml/logging';
|
|
2
|
-
import { BamlRuntime, BamlCtxManager,
|
|
2
|
+
import baml, { BamlRuntime, BamlCtxManager, ClientRegistry, toBamlError, BamlAbortError, Collector, BamlStream, ThrowIfVersionMismatch, BamlValidationError } from '@boundaryml/baml';
|
|
3
3
|
import pino from 'pino';
|
|
4
4
|
import EventEmitter from 'eventemitter3';
|
|
5
5
|
import z$1, { z, ZodObject } from 'zod';
|
|
6
6
|
export { z } from 'zod';
|
|
7
7
|
import sharp from 'sharp';
|
|
8
8
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
9
|
+
import crypto$1, { randomUUID, createHash as createHash$1 } from 'node:crypto';
|
|
10
|
+
import { performance } from 'node:perf_hooks';
|
|
9
11
|
import * as fs from 'node:fs';
|
|
10
12
|
import * as path from 'node:path';
|
|
11
13
|
import * as os from 'node:os';
|
|
@@ -23,7 +25,6 @@ import { TypeBuilder as TypeBuilder$1 } from '@boundaryml/baml/type_builder';
|
|
|
23
25
|
import { zodToJsonSchema } from 'zod-to-json-schema';
|
|
24
26
|
import { chromium } from 'playwright';
|
|
25
27
|
import objectHash from 'object-hash';
|
|
26
|
-
import crypto$1, { createHash as createHash$1 } from 'node:crypto';
|
|
27
28
|
import { partitionHtml, serializeToMarkdown } from 'magnitude-extract';
|
|
28
29
|
|
|
29
30
|
const fileMap = {
|
|
@@ -735,7 +736,7 @@ class Image {
|
|
|
735
736
|
}
|
|
736
737
|
async toBaml() {
|
|
737
738
|
const { format, base64 } = await this.toJson();
|
|
738
|
-
return Image
|
|
739
|
+
return baml.Image.fromBase64(`image/${format}`, base64);
|
|
739
740
|
}
|
|
740
741
|
async saveToFile(filepath) {
|
|
741
742
|
await this.img.clone().toFile(filepath);
|
|
@@ -820,7 +821,7 @@ async function buildXmlPartsRecursive(data, indentLevel, partsList, isInsideList
|
|
|
820
821
|
}
|
|
821
822
|
return;
|
|
822
823
|
}
|
|
823
|
-
if (data instanceof Image
|
|
824
|
+
if (data instanceof baml.Image) {
|
|
824
825
|
partsList.push(data);
|
|
825
826
|
return;
|
|
826
827
|
}
|
|
@@ -858,7 +859,7 @@ async function buildXmlPartsRecursive(data, indentLevel, partsList, isInsideList
|
|
|
858
859
|
}
|
|
859
860
|
}
|
|
860
861
|
if (currentStr) mergedValueParts.push(currentStr);
|
|
861
|
-
if (mergedValueParts.length === 1 && mergedValueParts[0] instanceof Image
|
|
862
|
+
if (mergedValueParts.length === 1 && mergedValueParts[0] instanceof baml.Image) {
|
|
862
863
|
partsList.push(`${indent}<${tagName}>`);
|
|
863
864
|
partsList.push(mergedValueParts[0]);
|
|
864
865
|
partsList.push(`</${tagName}>`);
|
|
@@ -924,7 +925,7 @@ async function buildJsonPartsRecursive(data, partsList, indent, currentLevel = 0
|
|
|
924
925
|
}
|
|
925
926
|
return;
|
|
926
927
|
}
|
|
927
|
-
if (data instanceof Image
|
|
928
|
+
if (data instanceof baml.Image) {
|
|
928
929
|
partsList.push(data);
|
|
929
930
|
return;
|
|
930
931
|
}
|
|
@@ -1122,20 +1123,35 @@ class OperationDeadlineError extends AgentError {
|
|
|
1122
1123
|
}
|
|
1123
1124
|
class AgentBusyError extends AgentError {
|
|
1124
1125
|
constructor() {
|
|
1125
|
-
super("Agent has
|
|
1126
|
+
super("Agent has pending work; await whenIdle() before reusing it", { variant: "busy" });
|
|
1126
1127
|
this.name = "AgentBusyError";
|
|
1127
1128
|
}
|
|
1128
1129
|
}
|
|
1129
1130
|
|
|
1131
|
+
function attachOperationDiagnostics(error, diagnostics) {
|
|
1132
|
+
if (!(error instanceof Error)) return;
|
|
1133
|
+
try {
|
|
1134
|
+
if (!("operation" in error)) Object.defineProperty(error, "operation", { value: diagnostics, enumerable: true });
|
|
1135
|
+
} catch {
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1130
1138
|
const operations = new AsyncLocalStorage();
|
|
1131
1139
|
class Operation {
|
|
1132
|
-
constructor(owner, options) {
|
|
1140
|
+
constructor(owner, options, kind = "exec", onUpdate) {
|
|
1133
1141
|
this.owner = owner;
|
|
1134
|
-
this.
|
|
1142
|
+
this.kind = kind;
|
|
1143
|
+
this.onUpdate = onUpdate;
|
|
1135
1144
|
if (options.deadline !== void 0 && !Number.isFinite(options.deadline)) {
|
|
1136
1145
|
throw new TypeError("deadline must be a finite Unix timestamp in milliseconds");
|
|
1137
1146
|
}
|
|
1138
1147
|
this.deadline = options.deadline;
|
|
1148
|
+
this.externalSignal = options.signal;
|
|
1149
|
+
this.signal.addEventListener("abort", () => {
|
|
1150
|
+
this.cancelled = performance.now();
|
|
1151
|
+
this.outcome = this.signal.reason instanceof OperationDeadlineError ? "deadline" : "cancelled";
|
|
1152
|
+
attachOperationDiagnostics(this.signal.reason, this.snapshot());
|
|
1153
|
+
this.publish();
|
|
1154
|
+
}, { once: true });
|
|
1139
1155
|
options.signal?.addEventListener("abort", this.externalAbort, { once: true });
|
|
1140
1156
|
if (options.signal?.aborted) this.externalAbort();
|
|
1141
1157
|
this.scheduleDeadline();
|
|
@@ -1145,7 +1161,19 @@ class Operation {
|
|
|
1145
1161
|
closed = false;
|
|
1146
1162
|
signal = this.controller.signal;
|
|
1147
1163
|
deadline;
|
|
1148
|
-
|
|
1164
|
+
externalSignal;
|
|
1165
|
+
externalAbort = () => this.cancel(this.externalSignal?.reason);
|
|
1166
|
+
id = randomUUID();
|
|
1167
|
+
startedAt = Date.now();
|
|
1168
|
+
started = performance.now();
|
|
1169
|
+
finished;
|
|
1170
|
+
cancelled;
|
|
1171
|
+
idle;
|
|
1172
|
+
announced = false;
|
|
1173
|
+
outcome;
|
|
1174
|
+
lastAction;
|
|
1175
|
+
timings = {};
|
|
1176
|
+
spans = /* @__PURE__ */ new Set();
|
|
1149
1177
|
scheduleDeadline() {
|
|
1150
1178
|
if (this.deadline === void 0 || this.signal.aborted) return;
|
|
1151
1179
|
const remaining = this.deadline - Date.now();
|
|
@@ -1153,28 +1181,106 @@ class Operation {
|
|
|
1153
1181
|
else this.timer = setTimeout(() => this.scheduleDeadline(), Math.min(remaining, 2147483647));
|
|
1154
1182
|
}
|
|
1155
1183
|
cancel(reason) {
|
|
1184
|
+
if (this.closed) return;
|
|
1156
1185
|
clearTimeout(this.timer);
|
|
1157
1186
|
if (!this.signal.aborted) this.controller.abort(new OperationCancelledError(reason));
|
|
1158
1187
|
}
|
|
1159
1188
|
check() {
|
|
1160
|
-
if (
|
|
1189
|
+
if (this.signal.aborted) throw this.signal.reason;
|
|
1190
|
+
if (this.closed) throw new OperationCancelledError("Operation already finished");
|
|
1191
|
+
if (this.deadline !== void 0 && Date.now() >= this.deadline) {
|
|
1161
1192
|
this.controller.abort(new OperationDeadlineError(this.deadline));
|
|
1162
1193
|
}
|
|
1163
1194
|
if (this.signal.aborted) throw this.signal.reason;
|
|
1164
|
-
if (this.closed) throw new OperationCancelledError("Operation already finished");
|
|
1165
1195
|
}
|
|
1166
1196
|
run(fn) {
|
|
1167
1197
|
return operations.run(this, fn);
|
|
1168
1198
|
}
|
|
1199
|
+
announce() {
|
|
1200
|
+
this.announced = true;
|
|
1201
|
+
this.publish();
|
|
1202
|
+
}
|
|
1203
|
+
publish() {
|
|
1204
|
+
if (!this.announced) return;
|
|
1205
|
+
try {
|
|
1206
|
+
this.onUpdate?.(this.snapshot());
|
|
1207
|
+
} catch {
|
|
1208
|
+
logger.warn("Operation diagnostic listener failed");
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
snapshot() {
|
|
1212
|
+
const now = this.finished ?? performance.now();
|
|
1213
|
+
const timings = {};
|
|
1214
|
+
for (const [phase2, timing] of Object.entries(this.timings)) timings[phase2] = { ...timing };
|
|
1215
|
+
let phase = "preparing";
|
|
1216
|
+
for (const span of this.spans) {
|
|
1217
|
+
phase = span.phase;
|
|
1218
|
+
const timing = timings[phase] ??= { count: 0, totalMs: 0 };
|
|
1219
|
+
timing.count++;
|
|
1220
|
+
timing.totalMs += now - span.started;
|
|
1221
|
+
}
|
|
1222
|
+
return {
|
|
1223
|
+
id: this.id,
|
|
1224
|
+
kind: this.kind,
|
|
1225
|
+
status: this.finished !== void 0 ? "finished" : this.cancelled !== void 0 ? "draining" : "running",
|
|
1226
|
+
outcome: this.outcome,
|
|
1227
|
+
phase,
|
|
1228
|
+
startedAt: this.startedAt,
|
|
1229
|
+
elapsedMs: now - this.started,
|
|
1230
|
+
...this.cancelled !== void 0 && this.finished !== void 0 ? { cancellationToDrainMs: this.finished - this.cancelled } : {},
|
|
1231
|
+
...this.cancelled !== void 0 && this.idle !== void 0 ? { cancellationToIdleMs: this.idle - this.cancelled } : {},
|
|
1232
|
+
...this.lastAction ? { lastAction: { ...this.lastAction } } : {},
|
|
1233
|
+
timings
|
|
1234
|
+
};
|
|
1235
|
+
}
|
|
1236
|
+
beginPhase(phase) {
|
|
1237
|
+
if (this.closed) return () => {
|
|
1238
|
+
};
|
|
1239
|
+
const span = { phase, started: performance.now() };
|
|
1240
|
+
this.spans.add(span);
|
|
1241
|
+
this.publish();
|
|
1242
|
+
return () => {
|
|
1243
|
+
if (!this.spans.delete(span)) return;
|
|
1244
|
+
const timing = this.timings[phase] ??= { count: 0, totalMs: 0 };
|
|
1245
|
+
timing.count++;
|
|
1246
|
+
timing.totalMs += performance.now() - span.started;
|
|
1247
|
+
this.publish();
|
|
1248
|
+
};
|
|
1249
|
+
}
|
|
1250
|
+
prepareAction(name) {
|
|
1251
|
+
this.lastAction = { index: (this.lastAction?.index ?? 0) + 1, name, state: "pending" };
|
|
1252
|
+
this.publish();
|
|
1253
|
+
}
|
|
1254
|
+
actionState(state) {
|
|
1255
|
+
if (this.lastAction) this.lastAction.state = state;
|
|
1256
|
+
}
|
|
1257
|
+
fail(error) {
|
|
1258
|
+
this.outcome ??= "failed";
|
|
1259
|
+
attachOperationDiagnostics(error, this.snapshot());
|
|
1260
|
+
}
|
|
1261
|
+
finish() {
|
|
1262
|
+
this.finished = performance.now();
|
|
1263
|
+
this.outcome ??= "succeeded";
|
|
1264
|
+
this.dispose();
|
|
1265
|
+
this.publish();
|
|
1266
|
+
}
|
|
1267
|
+
markIdle() {
|
|
1268
|
+
if (this.idle !== void 0) return;
|
|
1269
|
+
this.idle = performance.now();
|
|
1270
|
+
this.publish();
|
|
1271
|
+
}
|
|
1169
1272
|
dispose() {
|
|
1170
1273
|
this.closed = true;
|
|
1171
1274
|
clearTimeout(this.timer);
|
|
1172
|
-
this.
|
|
1275
|
+
this.externalSignal?.removeEventListener("abort", this.externalAbort);
|
|
1173
1276
|
}
|
|
1174
1277
|
}
|
|
1175
1278
|
function currentOperation() {
|
|
1176
1279
|
return operations.getStore();
|
|
1177
1280
|
}
|
|
1281
|
+
function withoutOperation(fn) {
|
|
1282
|
+
return operations.exit(fn);
|
|
1283
|
+
}
|
|
1178
1284
|
function checkOperation() {
|
|
1179
1285
|
currentOperation()?.check();
|
|
1180
1286
|
}
|
|
@@ -1183,6 +1289,19 @@ function operationOptions() {
|
|
|
1183
1289
|
const operation = currentOperation();
|
|
1184
1290
|
return { signal: operation?.signal, deadline: operation?.deadline };
|
|
1185
1291
|
}
|
|
1292
|
+
function beginOperationPhase(phase) {
|
|
1293
|
+
return currentOperation()?.beginPhase(phase) ?? (() => {
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
async function measureOperation(phase, fn) {
|
|
1297
|
+
const finish = beginOperationPhase(phase);
|
|
1298
|
+
try {
|
|
1299
|
+
checkOperation();
|
|
1300
|
+
return await fn();
|
|
1301
|
+
} finally {
|
|
1302
|
+
finish();
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1186
1305
|
async function drainAll(promises) {
|
|
1187
1306
|
const results = await Promise.allSettled(promises);
|
|
1188
1307
|
const failure = results.find((result) => result.status === "rejected");
|
|
@@ -2801,7 +2920,7 @@ async function retryOnError(fnToRetry, retryOptions) {
|
|
|
2801
2920
|
}
|
|
2802
2921
|
}
|
|
2803
2922
|
}
|
|
2804
|
-
if (attempt < options.retryLimit) await operationSleep(options.delayMs);
|
|
2923
|
+
if (attempt < options.retryLimit) await measureOperation("retry", () => operationSleep(options.delayMs));
|
|
2805
2924
|
}
|
|
2806
2925
|
throw lastError;
|
|
2807
2926
|
}
|
|
@@ -3432,7 +3551,9 @@ class ModelHarness {
|
|
|
3432
3551
|
async _withUsage(invoke) {
|
|
3433
3552
|
checkOperation();
|
|
3434
3553
|
const collector = new Collector("model-call");
|
|
3554
|
+
const finish = beginOperationPhase("model");
|
|
3435
3555
|
try {
|
|
3556
|
+
checkOperation();
|
|
3436
3557
|
try {
|
|
3437
3558
|
return await invoke(collector);
|
|
3438
3559
|
} finally {
|
|
@@ -3455,6 +3576,7 @@ class ModelHarness {
|
|
|
3455
3576
|
}
|
|
3456
3577
|
}
|
|
3457
3578
|
} finally {
|
|
3579
|
+
finish();
|
|
3458
3580
|
for (const log of collector.logs) {
|
|
3459
3581
|
for (const call of log.calls) {
|
|
3460
3582
|
try {
|
|
@@ -3735,7 +3857,7 @@ class MultiModelHarness {
|
|
|
3735
3857
|
uniqueModels = [];
|
|
3736
3858
|
events = new EventEmitter();
|
|
3737
3859
|
constructor(clients) {
|
|
3738
|
-
const sortedClients = clients.
|
|
3860
|
+
const sortedClients = [...clients].sort((a, b) => (b.roles ? b.roles.length : 9999) - (a.roles ? a.roles.length : 9999));
|
|
3739
3861
|
for (const client of sortedClients) {
|
|
3740
3862
|
const harness = new ModelHarness({ llm: client });
|
|
3741
3863
|
this.uniqueModels.push(harness);
|
|
@@ -3754,7 +3876,7 @@ class MultiModelHarness {
|
|
|
3754
3876
|
}
|
|
3755
3877
|
}
|
|
3756
3878
|
async setup() {
|
|
3757
|
-
await
|
|
3879
|
+
await drainAll(this.uniqueModels.map((model) => model.setup()));
|
|
3758
3880
|
}
|
|
3759
3881
|
describe() {
|
|
3760
3882
|
return this.uniqueModels[0].describeModel();
|
|
@@ -3808,8 +3930,14 @@ class Agent {
|
|
|
3808
3930
|
_paused = false;
|
|
3809
3931
|
_pauseResolve = null;
|
|
3810
3932
|
activeOperation;
|
|
3933
|
+
latestOperation;
|
|
3811
3934
|
idle = Promise.resolve();
|
|
3812
|
-
|
|
3935
|
+
resolveIdle;
|
|
3936
|
+
pendingWork = 0;
|
|
3937
|
+
lifecycleTail = Promise.resolve();
|
|
3938
|
+
lifecycleRequest;
|
|
3939
|
+
lifecycleState = "new";
|
|
3940
|
+
telemetryStarted = false;
|
|
3813
3941
|
latestTaskMemory;
|
|
3814
3942
|
// | null = null;
|
|
3815
3943
|
constructor(baseConfig = {}) {
|
|
@@ -3864,17 +3992,57 @@ class Agent {
|
|
|
3864
3992
|
return connector;
|
|
3865
3993
|
}
|
|
3866
3994
|
async start() {
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3995
|
+
return this.scheduleLifecycle("start", async () => {
|
|
3996
|
+
if (this.lifecycleState === "ready") return;
|
|
3997
|
+
this.lifecycleState = "starting";
|
|
3998
|
+
if (this.options.telemetry && !this.telemetryStarted) {
|
|
3999
|
+
telemetrifyAgent(this);
|
|
4000
|
+
this.telemetryStarted = true;
|
|
4001
|
+
}
|
|
4002
|
+
try {
|
|
4003
|
+
await this.models.setup();
|
|
4004
|
+
for (const connector of this.connectors) await connector.onStart?.();
|
|
4005
|
+
} catch (error) {
|
|
4006
|
+
await this.stopConnectors();
|
|
4007
|
+
this.lifecycleState = "stopped";
|
|
4008
|
+
throw error;
|
|
4009
|
+
}
|
|
4010
|
+
this.lifecycleState = "ready";
|
|
4011
|
+
this.events.emit("start");
|
|
4012
|
+
});
|
|
4013
|
+
}
|
|
4014
|
+
get lifecycle() {
|
|
4015
|
+
return this.lifecycleState;
|
|
4016
|
+
}
|
|
4017
|
+
beginWork() {
|
|
4018
|
+
if (this.pendingWork++ === 0) this.idle = new Promise((resolve) => {
|
|
4019
|
+
this.resolveIdle = resolve;
|
|
4020
|
+
});
|
|
4021
|
+
}
|
|
4022
|
+
endWork() {
|
|
4023
|
+
if (--this.pendingWork === 0) {
|
|
4024
|
+
const resolve = this.resolveIdle;
|
|
4025
|
+
this.resolveIdle = void 0;
|
|
4026
|
+
withoutOperation(() => this.latestOperation?.markIdle());
|
|
4027
|
+
resolve?.();
|
|
3874
4028
|
}
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
4029
|
+
}
|
|
4030
|
+
scheduleLifecycle(kind, fn) {
|
|
4031
|
+
checkOperation();
|
|
4032
|
+
if (kind === "start" && this.activeOperation) throw new AgentBusyError();
|
|
4033
|
+
if (this.lifecycleRequest?.kind === kind) return this.lifecycleRequest.promise;
|
|
4034
|
+
this.beginWork();
|
|
4035
|
+
return withoutOperation(() => {
|
|
4036
|
+
const promise = this.lifecycleTail.then(fn).finally(() => {
|
|
4037
|
+
if (this.lifecycleRequest?.promise === promise) this.lifecycleRequest = void 0;
|
|
4038
|
+
this.endWork();
|
|
4039
|
+
});
|
|
4040
|
+
this.lifecycleRequest = { kind, promise };
|
|
4041
|
+
this.lifecycleTail = promise.catch(() => {
|
|
4042
|
+
});
|
|
4043
|
+
if (kind === "stop") this.activeOperation?.cancel("Agent stopped");
|
|
4044
|
+
return promise;
|
|
4045
|
+
});
|
|
3878
4046
|
}
|
|
3879
4047
|
identifyAction(action) {
|
|
3880
4048
|
const actionDefinition = this.actions.find((def) => def.name === action.variant);
|
|
@@ -3885,22 +4053,31 @@ class Agent {
|
|
|
3885
4053
|
}
|
|
3886
4054
|
/** True until underlying work settles, including after a cancelled caller returns. */
|
|
3887
4055
|
get busy() {
|
|
3888
|
-
return this.
|
|
4056
|
+
return this.pendingWork > 0;
|
|
3889
4057
|
}
|
|
3890
4058
|
whenIdle() {
|
|
3891
4059
|
return this.idle;
|
|
3892
4060
|
}
|
|
3893
|
-
|
|
4061
|
+
/** A payload-free snapshot of the active or most recent operation. */
|
|
4062
|
+
get operation() {
|
|
4063
|
+
return this.latestOperation?.snapshot();
|
|
4064
|
+
}
|
|
4065
|
+
async runOperation(options, fn, kind = "exec") {
|
|
3894
4066
|
const inherited = currentOperation();
|
|
3895
4067
|
if (inherited?.owner === this) inherited.check();
|
|
3896
|
-
if (this.busy)
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
this.
|
|
3902
|
-
|
|
4068
|
+
if (this.busy) {
|
|
4069
|
+
const error = new AgentBusyError();
|
|
4070
|
+
if (this.activeOperation) attachOperationDiagnostics(error, this.activeOperation.snapshot());
|
|
4071
|
+
throw error;
|
|
4072
|
+
}
|
|
4073
|
+
if (this.lifecycleState === "stopped") throw new AgentError("Agent is stopped; call start() before using it");
|
|
4074
|
+
const operation = new Operation(this, options, kind, (snapshot) => {
|
|
4075
|
+
this.events.emit("operation", snapshot);
|
|
3903
4076
|
});
|
|
4077
|
+
this.activeOperation = operation;
|
|
4078
|
+
this.latestOperation = operation;
|
|
4079
|
+
this.beginWork();
|
|
4080
|
+
operation.announce();
|
|
3904
4081
|
const worker = operation.run(async () => {
|
|
3905
4082
|
operation.check();
|
|
3906
4083
|
try {
|
|
@@ -3909,10 +4086,13 @@ class Agent {
|
|
|
3909
4086
|
operation.check();
|
|
3910
4087
|
}
|
|
3911
4088
|
});
|
|
3912
|
-
const settled = worker.
|
|
3913
|
-
operation.
|
|
4089
|
+
const settled = worker.catch((error) => {
|
|
4090
|
+
operation.fail(error);
|
|
4091
|
+
throw error;
|
|
4092
|
+
}).finally(() => {
|
|
4093
|
+
operation.finish();
|
|
3914
4094
|
this.activeOperation = void 0;
|
|
3915
|
-
|
|
4095
|
+
this.endWork();
|
|
3916
4096
|
});
|
|
3917
4097
|
return untilAborted(settled, operation.signal);
|
|
3918
4098
|
}
|
|
@@ -3933,6 +4113,9 @@ class Agent {
|
|
|
3933
4113
|
if (!parsed.success) {
|
|
3934
4114
|
throw new AgentError(`Generated action '${action.variant}' violates input schema: ${parsed.error.message}`, { adaptable: true });
|
|
3935
4115
|
}
|
|
4116
|
+
const operation = currentOperation();
|
|
4117
|
+
operation?.prepareAction(actionDefinition.name);
|
|
4118
|
+
checkOperation();
|
|
3936
4119
|
const memoryOnly = memoryActions.includes(actionDefinition);
|
|
3937
4120
|
if (!memoryOnly) for (const connector of this.connectors) {
|
|
3938
4121
|
await connector.beforeAction?.(action, operationOptions());
|
|
@@ -3940,9 +4123,18 @@ class Agent {
|
|
|
3940
4123
|
}
|
|
3941
4124
|
this.events.emit("actionStarted", action);
|
|
3942
4125
|
checkOperation();
|
|
3943
|
-
const data = await
|
|
3944
|
-
|
|
3945
|
-
|
|
4126
|
+
const data = await measureOperation("action", async () => {
|
|
4127
|
+
const options = operationOptions();
|
|
4128
|
+
operation?.actionState("started");
|
|
4129
|
+
try {
|
|
4130
|
+
const result = await actionDefinition.resolver({ input: parsed.data, agent: this, memory, ...options });
|
|
4131
|
+
operation?.actionState("completed");
|
|
4132
|
+
return result;
|
|
4133
|
+
} catch (error) {
|
|
4134
|
+
operation?.actionState("failed");
|
|
4135
|
+
throw error;
|
|
4136
|
+
}
|
|
4137
|
+
});
|
|
3946
4138
|
checkOperation();
|
|
3947
4139
|
this.events.emit("actionDone", action);
|
|
3948
4140
|
checkOperation();
|
|
@@ -3965,21 +4157,22 @@ class Agent {
|
|
|
3965
4157
|
return data;
|
|
3966
4158
|
}
|
|
3967
4159
|
async _recordConnectorObservations(memory) {
|
|
3968
|
-
|
|
3969
|
-
|
|
3970
|
-
|
|
3971
|
-
|
|
3972
|
-
|
|
3973
|
-
|
|
4160
|
+
return measureOperation("observations", async () => {
|
|
4161
|
+
for (const connector of this.connectors) {
|
|
4162
|
+
const connObservations = connector.collectObservations ? await connector.collectObservations(operationOptions()) : [];
|
|
4163
|
+
checkOperation();
|
|
4164
|
+
for (const obs of connObservations) {
|
|
4165
|
+
memory.recordObservation(obs);
|
|
4166
|
+
}
|
|
3974
4167
|
}
|
|
3975
|
-
|
|
3976
|
-
|
|
4168
|
+
this.events.emit("observationsRecorded");
|
|
4169
|
+
});
|
|
3977
4170
|
}
|
|
3978
4171
|
get memory() {
|
|
3979
4172
|
return this.latestTaskMemory;
|
|
3980
4173
|
}
|
|
3981
4174
|
async act(taskOrSteps, options = {}) {
|
|
3982
|
-
return this.runOperation(options, () => this._runAct(taskOrSteps, options));
|
|
4175
|
+
return this.runOperation(options, () => this._runAct(taskOrSteps, options), "act");
|
|
3983
4176
|
}
|
|
3984
4177
|
async _runAct(taskOrSteps, options) {
|
|
3985
4178
|
const instructions = [
|
|
@@ -4012,28 +4205,25 @@ class Agent {
|
|
|
4012
4205
|
})(task);
|
|
4013
4206
|
}
|
|
4014
4207
|
async _buildContext(memory, options) {
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4019
|
-
|
|
4020
|
-
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
|
|
4024
|
-
|
|
4025
|
-
|
|
4026
|
-
instructions
|
|
4027
|
-
});
|
|
4208
|
+
return measureOperation("context", async () => {
|
|
4209
|
+
const messages = await memory.render(options);
|
|
4210
|
+
checkOperation();
|
|
4211
|
+
const connectorInstructions = [];
|
|
4212
|
+
for (const connector of this.connectors) {
|
|
4213
|
+
if (connector.getInstructions) {
|
|
4214
|
+
const instructions = await connector.getInstructions(operationOptions());
|
|
4215
|
+
checkOperation();
|
|
4216
|
+
if (instructions) {
|
|
4217
|
+
connectorInstructions.push({ connectorId: connector.id, instructions });
|
|
4218
|
+
}
|
|
4028
4219
|
}
|
|
4029
4220
|
}
|
|
4030
|
-
|
|
4031
|
-
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
};
|
|
4221
|
+
return {
|
|
4222
|
+
instructions: memory.instructions,
|
|
4223
|
+
observationContent: messages,
|
|
4224
|
+
connectorInstructions
|
|
4225
|
+
};
|
|
4226
|
+
});
|
|
4037
4227
|
}
|
|
4038
4228
|
async _act(description, memory, options = {}) {
|
|
4039
4229
|
checkOperation();
|
|
@@ -4122,7 +4312,7 @@ class Agent {
|
|
|
4122
4312
|
await this._recordConnectorObservations(this.latestTaskMemory);
|
|
4123
4313
|
const memoryContext = await this._buildContext(this.memory, options);
|
|
4124
4314
|
return await this.models.query(memoryContext, query, schema);
|
|
4125
|
-
});
|
|
4315
|
+
}, "query");
|
|
4126
4316
|
}
|
|
4127
4317
|
async queueDone() {
|
|
4128
4318
|
checkOperation();
|
|
@@ -4136,9 +4326,12 @@ class Agent {
|
|
|
4136
4326
|
if (!this._paused) return;
|
|
4137
4327
|
logger.info("Agent: Paused");
|
|
4138
4328
|
try {
|
|
4139
|
-
await
|
|
4140
|
-
this.
|
|
4141
|
-
|
|
4329
|
+
await measureOperation("paused", async () => {
|
|
4330
|
+
if (!this._paused) return;
|
|
4331
|
+
await untilAborted(new Promise((resolve) => {
|
|
4332
|
+
this._pauseResolve = resolve;
|
|
4333
|
+
}), currentOperation()?.signal);
|
|
4334
|
+
});
|
|
4142
4335
|
} finally {
|
|
4143
4336
|
this._pauseResolve = null;
|
|
4144
4337
|
}
|
|
@@ -4161,15 +4354,19 @@ class Agent {
|
|
|
4161
4354
|
return this._paused;
|
|
4162
4355
|
}
|
|
4163
4356
|
async stop() {
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
if (this._paused) {
|
|
4357
|
+
return this.scheduleLifecycle("stop", async () => {
|
|
4358
|
+
if (this.lifecycleState === "stopped") return;
|
|
4359
|
+
this.lifecycleState = "stopping";
|
|
4360
|
+
this.doneActing = true;
|
|
4169
4361
|
this._paused = false;
|
|
4170
4362
|
this._pauseResolve?.();
|
|
4171
4363
|
this._pauseResolve = null;
|
|
4172
|
-
|
|
4364
|
+
await this.stopConnectors();
|
|
4365
|
+
this.lifecycleState = "stopped";
|
|
4366
|
+
this.events.emit("stop");
|
|
4367
|
+
});
|
|
4368
|
+
}
|
|
4369
|
+
async stopConnectors() {
|
|
4173
4370
|
logger.info("Agent: Stopping connectors...");
|
|
4174
4371
|
for (const connector of this.connectors) {
|
|
4175
4372
|
try {
|
|
@@ -4178,7 +4375,6 @@ class Agent {
|
|
|
4178
4375
|
logger.warn(`Agent: Error stopping connector ${connector.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
4179
4376
|
}
|
|
4180
4377
|
}
|
|
4181
|
-
this.events.emit("stop");
|
|
4182
4378
|
logger.info("Agent: All connectors stopped.");
|
|
4183
4379
|
logger.info("Agent: Stopped successfully.");
|
|
4184
4380
|
}
|
|
@@ -4485,14 +4681,15 @@ function parseTypeContent(content) {
|
|
|
4485
4681
|
}
|
|
4486
4682
|
|
|
4487
4683
|
class CursorVisual {
|
|
4684
|
+
constructor(animate = true) {
|
|
4685
|
+
this.animate = animate;
|
|
4686
|
+
}
|
|
4488
4687
|
/**
|
|
4489
4688
|
* Manages the visual indicator for actions on a page
|
|
4490
4689
|
*/
|
|
4491
4690
|
page;
|
|
4492
4691
|
visualElementId = "action-visual-indicator";
|
|
4493
4692
|
lastPosition = null;
|
|
4494
|
-
constructor() {
|
|
4495
|
-
}
|
|
4496
4693
|
async setActivePage(page) {
|
|
4497
4694
|
this.page = page;
|
|
4498
4695
|
page.on("load", async () => {
|
|
@@ -4507,9 +4704,11 @@ class CursorVisual {
|
|
|
4507
4704
|
);
|
|
4508
4705
|
}
|
|
4509
4706
|
async move(x, y) {
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4707
|
+
await measureOperation("cursor", async () => {
|
|
4708
|
+
this.lastPosition = { x, y };
|
|
4709
|
+
await this._drawVisual(x, y, false);
|
|
4710
|
+
if (this.animate) await operationSleep(300);
|
|
4711
|
+
});
|
|
4513
4712
|
}
|
|
4514
4713
|
async setupOnPage() {
|
|
4515
4714
|
if (this.lastPosition) {
|
|
@@ -4520,7 +4719,7 @@ class CursorVisual {
|
|
|
4520
4719
|
async _drawVisual(x, y, showClickEffect) {
|
|
4521
4720
|
try {
|
|
4522
4721
|
await this.page.evaluate(
|
|
4523
|
-
({ x: x2, y: y2, id, showClickEffect: showClickEffect2 }) => {
|
|
4722
|
+
({ x: x2, y: y2, id, showClickEffect: showClickEffect2, animate }) => {
|
|
4524
4723
|
const viewportX = x2;
|
|
4525
4724
|
const viewportY = y2;
|
|
4526
4725
|
const docX = x2 + window.scrollX;
|
|
@@ -4562,7 +4761,6 @@ class CursorVisual {
|
|
|
4562
4761
|
pointerElement.style.height = "32px";
|
|
4563
4762
|
pointerElement.style.zIndex = "2147483647";
|
|
4564
4763
|
pointerElement.style.pointerEvents = "none";
|
|
4565
|
-
pointerElement.style.transition = "left 0.3s cubic-bezier(0.25, 0.1, 0.25, 1), top 0.3s cubic-bezier(0.25, 0.1, 0.25, 1)";
|
|
4566
4764
|
const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
|
|
4567
4765
|
svg.setAttribute("width", "32");
|
|
4568
4766
|
svg.setAttribute("height", "32");
|
|
@@ -4585,11 +4783,12 @@ class CursorVisual {
|
|
|
4585
4783
|
pointerElement.appendChild(svg);
|
|
4586
4784
|
document.body.appendChild(pointerElement);
|
|
4587
4785
|
}
|
|
4786
|
+
pointerElement.style.transition = animate ? "left 0.3s cubic-bezier(0.25, 0.1, 0.25, 1), top 0.3s cubic-bezier(0.25, 0.1, 0.25, 1)" : "none";
|
|
4588
4787
|
pointerElement.style.left = `${viewportX}px`;
|
|
4589
4788
|
pointerElement.style.top = `${viewportY}px`;
|
|
4590
4789
|
pointerElement.style.transform = "translate(-1px, -3px)";
|
|
4591
4790
|
},
|
|
4592
|
-
{ x, y, id: this.visualElementId, showClickEffect }
|
|
4791
|
+
{ x, y, id: this.visualElementId, showClickEffect, animate: this.animate }
|
|
4593
4792
|
);
|
|
4594
4793
|
} catch (error) {
|
|
4595
4794
|
logger.trace(`Failed to draw visual: ${error.message}`);
|
|
@@ -5067,12 +5266,13 @@ class ActionVisualizer {
|
|
|
5067
5266
|
this.context = context;
|
|
5068
5267
|
this.options = {
|
|
5069
5268
|
showCursor: options.showCursor ?? true,
|
|
5269
|
+
animateCursor: options.animateCursor ?? true,
|
|
5070
5270
|
showHoverCircle: options.showHoverCircle ?? false,
|
|
5071
5271
|
showClickRipple: options.showClickRipple ?? true,
|
|
5072
5272
|
showDragLine: options.showDragLine ?? false,
|
|
5073
5273
|
showTypeEffects: options.showTypeEffects ?? true
|
|
5074
5274
|
};
|
|
5075
|
-
this.cursor = new CursorVisual();
|
|
5275
|
+
this.cursor = new CursorVisual(this.options.animateCursor);
|
|
5076
5276
|
this.mouseEffects = new MouseEffectVisual(this.options);
|
|
5077
5277
|
this.typeEffects = new TypeEffectVisual();
|
|
5078
5278
|
}
|
|
@@ -5080,18 +5280,18 @@ class ActionVisualizer {
|
|
|
5080
5280
|
}
|
|
5081
5281
|
async setActivePage(page) {
|
|
5082
5282
|
this.page = page;
|
|
5083
|
-
await this.cursor.setActivePage(page);
|
|
5283
|
+
if (this.options.showCursor) await this.cursor.setActivePage(page);
|
|
5084
5284
|
await this.mouseEffects.setActivePage(page);
|
|
5085
5285
|
await this.typeEffects.setActivePage(page);
|
|
5086
5286
|
}
|
|
5087
5287
|
async moveVirtualCursor(x, y) {
|
|
5088
|
-
await this.cursor.move(x, y);
|
|
5288
|
+
if (this.options.showCursor) await this.cursor.move(x, y);
|
|
5089
5289
|
}
|
|
5090
5290
|
async hideAll() {
|
|
5091
|
-
await this.cursor.hide();
|
|
5291
|
+
if (this.options.showCursor) await this.cursor.hide();
|
|
5092
5292
|
}
|
|
5093
5293
|
async showAll() {
|
|
5094
|
-
await this.cursor.show();
|
|
5294
|
+
if (this.options.showCursor) await this.cursor.show();
|
|
5095
5295
|
}
|
|
5096
5296
|
}
|
|
5097
5297
|
|
|
@@ -5921,33 +6121,33 @@ class WebHarness {
|
|
|
5921
6121
|
return this.tabs.getActivePage();
|
|
5922
6122
|
}
|
|
5923
6123
|
async screenshot(options = {}) {
|
|
5924
|
-
|
|
5925
|
-
|
|
5926
|
-
|
|
5927
|
-
|
|
5928
|
-
|
|
5929
|
-
try {
|
|
5930
|
-
dpr = await this.page.evaluate(() => window.devicePixelRatio);
|
|
5931
|
-
checkOperation();
|
|
5932
|
-
buffer = await this.page.screenshot({ type: "png", ...options });
|
|
6124
|
+
return measureOperation("screenshot", async () => {
|
|
6125
|
+
let dpr;
|
|
6126
|
+
let buffer;
|
|
6127
|
+
const retries = 3;
|
|
6128
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
5933
6129
|
checkOperation();
|
|
5934
|
-
|
|
5935
|
-
|
|
5936
|
-
|
|
5937
|
-
|
|
5938
|
-
|
|
5939
|
-
|
|
5940
|
-
}
|
|
5941
|
-
|
|
5942
|
-
|
|
6130
|
+
try {
|
|
6131
|
+
dpr = await this.page.evaluate(() => window.devicePixelRatio);
|
|
6132
|
+
checkOperation();
|
|
6133
|
+
buffer = await this.page.screenshot({ type: "png", ...options });
|
|
6134
|
+
checkOperation();
|
|
6135
|
+
break;
|
|
6136
|
+
} catch (err) {
|
|
6137
|
+
checkOperation();
|
|
6138
|
+
const error = err;
|
|
6139
|
+
if (error.message.includes("Target page, context or browser has been closed")) {
|
|
6140
|
+
throw new Error("Attempted to take screenshot but page, context or browser is closed");
|
|
6141
|
+
}
|
|
6142
|
+
if (attempt >= retries) {
|
|
6143
|
+
throw new Error(`Unable to capture screenshot after retries, error: ${error.message}`);
|
|
6144
|
+
}
|
|
5943
6145
|
}
|
|
5944
6146
|
}
|
|
5945
|
-
|
|
5946
|
-
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
const rescaledImage = await image.resize(width / dpr, height / dpr);
|
|
5950
|
-
return rescaledImage;
|
|
6147
|
+
const image = Image.fromBase64(buffer.toString("base64"));
|
|
6148
|
+
const { width, height } = await image.getDimensions();
|
|
6149
|
+
return await image.resize(width / dpr, height / dpr);
|
|
6150
|
+
});
|
|
5951
6151
|
}
|
|
5952
6152
|
// async goto(url: string) {
|
|
5953
6153
|
// // No need to redraw here anymore, the 'load' event listener handles it
|
|
@@ -6146,7 +6346,7 @@ class WebHarness {
|
|
|
6146
6346
|
}
|
|
6147
6347
|
async waitForStability(timeout) {
|
|
6148
6348
|
checkOperation();
|
|
6149
|
-
await this.stability.waitForStability(timeout);
|
|
6349
|
+
await measureOperation("stability", () => this.stability.waitForStability(timeout));
|
|
6150
6350
|
checkOperation();
|
|
6151
6351
|
}
|
|
6152
6352
|
// async applyTransformations() {
|
|
@@ -6665,7 +6865,7 @@ async function retry(fn, options = {}) {
|
|
|
6665
6865
|
delay * Math.pow(multiplier, attempt),
|
|
6666
6866
|
maxDelay
|
|
6667
6867
|
);
|
|
6668
|
-
await operationSleep(currentDelay);
|
|
6868
|
+
await measureOperation("retry", () => operationSleep(currentDelay));
|
|
6669
6869
|
}
|
|
6670
6870
|
}
|
|
6671
6871
|
}
|
|
@@ -6799,7 +6999,7 @@ class BrowserConnector {
|
|
|
6799
6999
|
this.cancelWait = () => controller.abort(new OperationCancelledError("Browser stopped"));
|
|
6800
7000
|
this.recovery.waitUntil = Date.now() + duration;
|
|
6801
7001
|
try {
|
|
6802
|
-
await operationSleep(duration, controller.signal);
|
|
7002
|
+
await measureOperation("cooldown", () => operationSleep(duration, controller.signal));
|
|
6803
7003
|
} finally {
|
|
6804
7004
|
operation?.signal.removeEventListener("abort", abort);
|
|
6805
7005
|
this.cancelWait = void 0;
|
|
@@ -7036,10 +7236,10 @@ class BrowserAgent extends Agent {
|
|
|
7036
7236
|
this.browserAgentEvents.emit("nav", url);
|
|
7037
7237
|
checkOperation();
|
|
7038
7238
|
await this.require(BrowserConnector).getHarness().navigate(url);
|
|
7039
|
-
});
|
|
7239
|
+
}, "nav");
|
|
7040
7240
|
}
|
|
7041
7241
|
async extract(instructions, schema, options = {}) {
|
|
7042
|
-
return this.runOperation(options, () => this._extract(instructions, schema));
|
|
7242
|
+
return this.runOperation(options, () => this._extract(instructions, schema), "extract");
|
|
7043
7243
|
}
|
|
7044
7244
|
async _extract(instructions, schema) {
|
|
7045
7245
|
this.browserAgentEvents.emit("extractStarted", instructions, schema);
|