create-walle 0.9.26 → 0.9.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/template/claude-task-manager/api-prompts.js +11 -6
- package/template/claude-task-manager/lib/session-messages-projection.js +30 -1
- package/template/claude-task-manager/public/index.html +50 -2
- package/template/claude-task-manager/server.js +125 -1
- package/template/claude-task-manager/workers/read-pool-worker.js +10 -0
- package/template/package.json +1 -1
- package/template/wall-e/agent.js +77 -24
- package/template/wall-e/brain.js +258 -1
- package/template/wall-e/chat.js +30 -25
- package/template/wall-e/coding/session-plan.js +79 -0
- package/template/wall-e/coding-orchestrator.js +9 -3
- package/template/wall-e/coding-prompts.js +10 -3
- package/template/wall-e/lib/scheduler.js +154 -8
- package/template/wall-e/lib/worker-thread-pool.js +9 -1
- package/template/wall-e/loops/think.js +26 -3
- package/template/wall-e/mcp-server.js +20 -4
- package/template/wall-e/sources/jsonl-utils.js +84 -11
- package/template/wall-e/tools/local-tools.js +16 -0
- package/template/wall-e/workers/runtime-worker.js +24 -0
|
@@ -29,6 +29,30 @@ const DEFAULT_MAX_MISSED_JOBS_PER_RESTART = 5;
|
|
|
29
29
|
const DEFAULT_SLOW_TICK_MS = 250;
|
|
30
30
|
const DEFAULT_SLOW_JOB_MS = 2_000;
|
|
31
31
|
const PERF_LOG_THROTTLE_MS = 30_000;
|
|
32
|
+
const DEFAULT_NON_BLOCKING_CATCHUP_MAX_WORK_MS = 5_000;
|
|
33
|
+
const DEFAULT_NON_BLOCKING_CATCHUP_DELAY_MS = 2 * 60 * 1000;
|
|
34
|
+
const DEFAULT_RESTART_DEFER_JOB_NAMES = [
|
|
35
|
+
'harvest',
|
|
36
|
+
'training',
|
|
37
|
+
'replay',
|
|
38
|
+
'agent-benchmark',
|
|
39
|
+
'weekly-eval-loop',
|
|
40
|
+
'brain-optimize',
|
|
41
|
+
'dedup',
|
|
42
|
+
'daily-backup',
|
|
43
|
+
'brain-retention',
|
|
44
|
+
'question-digest',
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
function _normalizeLane(value, priority = 5) {
|
|
48
|
+
const n = Number(value);
|
|
49
|
+
if (Number.isFinite(n)) return Math.max(0, Math.trunc(n));
|
|
50
|
+
const p = Number(priority);
|
|
51
|
+
if (!Number.isFinite(p)) return 2;
|
|
52
|
+
if (p <= 2) return 0;
|
|
53
|
+
if (p <= 5) return 1;
|
|
54
|
+
return 2;
|
|
55
|
+
}
|
|
32
56
|
|
|
33
57
|
/** Default cascade check: true if any numeric value in result is > 0 */
|
|
34
58
|
function _defaultCascadeCheck(result) {
|
|
@@ -130,6 +154,7 @@ class Scheduler extends EventEmitter {
|
|
|
130
154
|
* @param {number} desc.intervalMs
|
|
131
155
|
* @param {string} [desc.pool] - Resource pool name
|
|
132
156
|
* @param {number} [desc.priority=5] - 1 (highest) to 10 (lowest)
|
|
157
|
+
* @param {number} [desc.lane] - 0 foreground, 1 normal, 2 background
|
|
133
158
|
* @param {string[]} [desc.dependsOn] - Jobs that must complete once before this runs
|
|
134
159
|
* @param {string[]} [desc.cascadeFrom] - Jobs whose results trigger immediate eligibility
|
|
135
160
|
* @param {number} [desc.startDelayMs=0]
|
|
@@ -155,6 +180,7 @@ class Scheduler extends EventEmitter {
|
|
|
155
180
|
worker: this._workerJobNames.has(desc.name),
|
|
156
181
|
...desc,
|
|
157
182
|
};
|
|
183
|
+
desc.lane = _normalizeLane(desc.lane, desc.priority);
|
|
158
184
|
if (desc.worker && !desc.timeoutMs && this._defaultWorkerJobTimeoutMs > 0) {
|
|
159
185
|
desc.timeoutMs = this._defaultWorkerJobTimeoutMs;
|
|
160
186
|
}
|
|
@@ -298,7 +324,7 @@ class Scheduler extends EventEmitter {
|
|
|
298
324
|
getJobs() {
|
|
299
325
|
return Object.entries(this.getState())
|
|
300
326
|
.map(([name, state]) => ({ name, ...state }))
|
|
301
|
-
.sort((a, b) => a.priority - b.priority || a.name.localeCompare(b.name));
|
|
327
|
+
.sort((a, b) => a.lane - b.lane || a.priority - b.priority || a.name.localeCompare(b.name));
|
|
302
328
|
}
|
|
303
329
|
|
|
304
330
|
/**
|
|
@@ -374,6 +400,7 @@ class Scheduler extends EventEmitter {
|
|
|
374
400
|
return {
|
|
375
401
|
enabled: desc.enabled,
|
|
376
402
|
pool: desc.pool || null,
|
|
403
|
+
lane: desc.lane,
|
|
377
404
|
priority: desc.priority,
|
|
378
405
|
intervalMs: desc.intervalMs,
|
|
379
406
|
timeoutMs: desc.timeoutMs || null,
|
|
@@ -681,8 +708,12 @@ class Scheduler extends EventEmitter {
|
|
|
681
708
|
}
|
|
682
709
|
summary.candidates = candidates.length;
|
|
683
710
|
|
|
684
|
-
// 3. Sort by
|
|
685
|
-
candidates.sort((a, b) =>
|
|
711
|
+
// 3. Sort by lane first (foreground before background), then priority.
|
|
712
|
+
candidates.sort((a, b) =>
|
|
713
|
+
a.descriptor.lane - b.descriptor.lane
|
|
714
|
+
|| a.descriptor.priority - b.descriptor.priority
|
|
715
|
+
|| a.descriptor.name.localeCompare(b.descriptor.name)
|
|
716
|
+
);
|
|
686
717
|
|
|
687
718
|
// 4. Dispatch respecting pool limits. Worker-thread requests register
|
|
688
719
|
// asynchronously inside WorkerThreadSlot.request(), so reserve the slots we
|
|
@@ -802,6 +833,38 @@ class Scheduler extends EventEmitter {
|
|
|
802
833
|
}
|
|
803
834
|
}
|
|
804
835
|
|
|
836
|
+
_completeSchedulerCatchupWorkItem(job, result) {
|
|
837
|
+
const id = job && job._runtimeCatchupWorkItemId;
|
|
838
|
+
if (!id || !this._persistBrain || typeof this._persistBrain.completeRuntimeWorkItem !== 'function') return;
|
|
839
|
+
try {
|
|
840
|
+
this._persistBrain.completeRuntimeWorkItem(id, {
|
|
841
|
+
cursor: {
|
|
842
|
+
completedBy: 'scheduler',
|
|
843
|
+
runCount: result && typeof result === 'object' ? result.count : undefined,
|
|
844
|
+
},
|
|
845
|
+
});
|
|
846
|
+
job._runtimeCatchupWorkItemId = null;
|
|
847
|
+
} catch (e) {
|
|
848
|
+
const name = job?.descriptor?.name || id;
|
|
849
|
+
console.warn(`[scheduler] Could not complete catch-up work item for "${name}": ${e.message}`);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
_failSchedulerCatchupWorkItem(job, err) {
|
|
854
|
+
const id = job && job._runtimeCatchupWorkItemId;
|
|
855
|
+
if (!id || !this._persistBrain || typeof this._persistBrain.failRuntimeWorkItem !== 'function') return;
|
|
856
|
+
try {
|
|
857
|
+
this._persistBrain.failRuntimeWorkItem(id, {
|
|
858
|
+
error: err,
|
|
859
|
+
retryAfterMs: this._minRefireGapMs,
|
|
860
|
+
});
|
|
861
|
+
job._runtimeCatchupWorkItemId = null;
|
|
862
|
+
} catch (e) {
|
|
863
|
+
const name = job?.descriptor?.name || id;
|
|
864
|
+
console.warn(`[scheduler] Could not update failed catch-up work item for "${name}": ${e.message}`);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
|
|
805
868
|
_dispatch(job) {
|
|
806
869
|
const { name, pool, run, onResult, onError } = job.descriptor;
|
|
807
870
|
job.state.status = 'running';
|
|
@@ -857,6 +920,7 @@ class Scheduler extends EventEmitter {
|
|
|
857
920
|
job.state.lastError = null;
|
|
858
921
|
job.state.runCount++;
|
|
859
922
|
job.state.completedOnce = true;
|
|
923
|
+
this._completeSchedulerCatchupWorkItem(job, result);
|
|
860
924
|
|
|
861
925
|
// Recovery: clear consecutive-error tracking + dismiss any active
|
|
862
926
|
// scheduler:job_failure:<name> alert. Failure-alert behavior is
|
|
@@ -889,6 +953,7 @@ class Scheduler extends EventEmitter {
|
|
|
889
953
|
failure = err;
|
|
890
954
|
job.state.lastError = err.message;
|
|
891
955
|
job.state.errorCount++;
|
|
956
|
+
this._failSchedulerCatchupWorkItem(job, err);
|
|
892
957
|
|
|
893
958
|
// Failure-alert hook: bumps consecutiveErrors, emits service alert
|
|
894
959
|
// when threshold (default 2) hit and outside cooldown (default 1h).
|
|
@@ -950,6 +1015,8 @@ class Scheduler extends EventEmitter {
|
|
|
950
1015
|
const result = await this._workerPool.request('scheduler.runJob', payload, {
|
|
951
1016
|
timeoutMs,
|
|
952
1017
|
heavy: true,
|
|
1018
|
+
lane: desc.lane,
|
|
1019
|
+
priority: desc.priority,
|
|
953
1020
|
terminateOnTimeout: timeoutMs > 0,
|
|
954
1021
|
});
|
|
955
1022
|
workerMeta.worker_used = true;
|
|
@@ -1053,9 +1120,19 @@ class Scheduler extends EventEmitter {
|
|
|
1053
1120
|
async runMissedJobs(opts = {}) {
|
|
1054
1121
|
const max = opts.max ?? this._maxMissedJobsPerRestart;
|
|
1055
1122
|
const staggerMs = opts.staggerMs ?? this._missedJobStaggerMs;
|
|
1123
|
+
const nonBlocking = opts.nonBlocking === true;
|
|
1124
|
+
const deferNames = new Set(opts.deferNames || DEFAULT_RESTART_DEFER_JOB_NAMES);
|
|
1125
|
+
const deferDelayMs = Number.isFinite(Number(opts.deferDelayMs))
|
|
1126
|
+
? Math.max(0, Math.trunc(Number(opts.deferDelayMs)))
|
|
1127
|
+
: DEFAULT_NON_BLOCKING_CATCHUP_DELAY_MS;
|
|
1128
|
+
const maxWorkMs = Number.isFinite(Number(opts.maxWorkMs))
|
|
1129
|
+
? Math.max(0, Math.trunc(Number(opts.maxWorkMs)))
|
|
1130
|
+
: DEFAULT_NON_BLOCKING_CATCHUP_MAX_WORK_MS;
|
|
1131
|
+
const enqueueWorkItem = typeof opts.enqueueWorkItem === 'function' ? opts.enqueueWorkItem : null;
|
|
1056
1132
|
const op = runtimeHealth.beginOperation('scheduler.runMissedJobs', {
|
|
1057
1133
|
max,
|
|
1058
1134
|
staggerMs,
|
|
1135
|
+
nonBlocking,
|
|
1059
1136
|
});
|
|
1060
1137
|
if (!this._persistJobState || !this._persistBrain) {
|
|
1061
1138
|
const summary = { ran: 0, deferred: 0, reason: 'persistence-disabled' };
|
|
@@ -1104,8 +1181,55 @@ class Scheduler extends EventEmitter {
|
|
|
1104
1181
|
candidates.sort((a, b) =>
|
|
1105
1182
|
(a.stored.next_eligible_at || 0) - (b.stored.next_eligible_at || 0)
|
|
1106
1183
|
);
|
|
1107
|
-
|
|
1108
|
-
const
|
|
1184
|
+
let eligibleCandidates = candidates;
|
|
1185
|
+
const policyDeferred = [];
|
|
1186
|
+
if (nonBlocking) {
|
|
1187
|
+
eligibleCandidates = [];
|
|
1188
|
+
for (const candidate of candidates) {
|
|
1189
|
+
const policy = candidate.job.descriptor.restartCatchupPolicy
|
|
1190
|
+
|| (deferNames.has(candidate.stored.job_name) ? 'defer' : 'run');
|
|
1191
|
+
if (policy === 'skip' || policy === 'defer') {
|
|
1192
|
+
policyDeferred.push({ ...candidate, reason: policy === 'skip' ? 'restart-skip' : 'restart-defer-policy' });
|
|
1193
|
+
} else {
|
|
1194
|
+
eligibleCandidates.push(candidate);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
const immediate = eligibleCandidates.slice(0, max);
|
|
1200
|
+
const deferred = [
|
|
1201
|
+
...eligibleCandidates.slice(max).map((candidate) => ({ ...candidate, reason: 'restart-catchup-cap' })),
|
|
1202
|
+
...policyDeferred,
|
|
1203
|
+
];
|
|
1204
|
+
|
|
1205
|
+
if (nonBlocking && deferred.length > 0) {
|
|
1206
|
+
const deferredAt = Date.now();
|
|
1207
|
+
deferred.forEach((candidate, index) => {
|
|
1208
|
+
const { job } = candidate;
|
|
1209
|
+
job.state.nextEligibleAt = deferredAt + deferDelayMs + (index * Math.min(5000, this._minRefireGapMs));
|
|
1210
|
+
this._persistState(job.descriptor.name, job);
|
|
1211
|
+
if (enqueueWorkItem) {
|
|
1212
|
+
const workItemId = `scheduler-catchup:${job.descriptor.name}`;
|
|
1213
|
+
try {
|
|
1214
|
+
enqueueWorkItem({
|
|
1215
|
+
id: workItemId,
|
|
1216
|
+
kind: 'scheduler.catchup',
|
|
1217
|
+
lane: job.descriptor.lane,
|
|
1218
|
+
priority: job.descriptor.priority,
|
|
1219
|
+
payload: {
|
|
1220
|
+
jobName: job.descriptor.name,
|
|
1221
|
+
reason: candidate.reason,
|
|
1222
|
+
trigger: 'restart-catchup',
|
|
1223
|
+
},
|
|
1224
|
+
not_before: job.state.nextEligibleAt,
|
|
1225
|
+
});
|
|
1226
|
+
job._runtimeCatchupWorkItemId = workItemId;
|
|
1227
|
+
} catch (e) {
|
|
1228
|
+
console.warn(`[scheduler] Could not enqueue catch-up work item for "${job.descriptor.name}": ${e.message}`);
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
});
|
|
1232
|
+
}
|
|
1109
1233
|
|
|
1110
1234
|
if (deferred.length > 0) {
|
|
1111
1235
|
console.log(
|
|
@@ -1130,8 +1254,14 @@ class Scheduler extends EventEmitter {
|
|
|
1130
1254
|
let ran = 0;
|
|
1131
1255
|
let workMs = 0;
|
|
1132
1256
|
let staggerWaitMs = 0;
|
|
1257
|
+
let budgetDeferred = [];
|
|
1258
|
+
const jobsRan = [];
|
|
1133
1259
|
for (let i = 0; i < immediate.length; i++) {
|
|
1134
1260
|
const { job } = immediate[i];
|
|
1261
|
+
if (nonBlocking && workMs >= maxWorkMs && ran > 0) {
|
|
1262
|
+
budgetDeferred = immediate.slice(i).map((candidate) => ({ ...candidate, reason: 'restart-catchup-budget' }));
|
|
1263
|
+
break;
|
|
1264
|
+
}
|
|
1135
1265
|
// Mark eligible RIGHT NOW so _dispatch picks it up
|
|
1136
1266
|
job.state.nextEligibleAt = Date.now();
|
|
1137
1267
|
job._wakeTrigger = 'restart-catchup';
|
|
@@ -1141,9 +1271,10 @@ class Scheduler extends EventEmitter {
|
|
|
1141
1271
|
try {
|
|
1142
1272
|
const workStartedAt = Date.now();
|
|
1143
1273
|
this._dispatch(job);
|
|
1144
|
-
if (job.runPromise) await job.runPromise;
|
|
1274
|
+
if (!nonBlocking && job.runPromise) await job.runPromise;
|
|
1145
1275
|
workMs += Date.now() - workStartedAt;
|
|
1146
1276
|
ran++;
|
|
1277
|
+
jobsRan.push(job.descriptor.name);
|
|
1147
1278
|
} catch (e) {
|
|
1148
1279
|
console.warn(`[scheduler] Missed-job replay for "${job.descriptor.name}" threw: ${e.message}`);
|
|
1149
1280
|
}
|
|
@@ -1154,14 +1285,25 @@ class Scheduler extends EventEmitter {
|
|
|
1154
1285
|
}
|
|
1155
1286
|
}
|
|
1156
1287
|
|
|
1288
|
+
if (nonBlocking && budgetDeferred.length > 0) {
|
|
1289
|
+
const deferredAt = Date.now();
|
|
1290
|
+
budgetDeferred.forEach((candidate, index) => {
|
|
1291
|
+
const { job } = candidate;
|
|
1292
|
+
job.state.nextEligibleAt = deferredAt + deferDelayMs + (index * Math.min(5000, this._minRefireGapMs));
|
|
1293
|
+
this._persistState(job.descriptor.name, job);
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1157
1297
|
const summary = {
|
|
1158
1298
|
ran,
|
|
1159
|
-
deferred: deferred.length + skipped.length,
|
|
1160
|
-
jobs_ran:
|
|
1299
|
+
deferred: deferred.length + skipped.length + budgetDeferred.length,
|
|
1300
|
+
jobs_ran: jobsRan,
|
|
1161
1301
|
jobs_deferred: [
|
|
1162
1302
|
...deferred.map((c) => c.stored.job_name),
|
|
1303
|
+
...budgetDeferred.map((c) => c.stored.job_name),
|
|
1163
1304
|
...skipped.map((c) => c.job.descriptor.name),
|
|
1164
1305
|
],
|
|
1306
|
+
non_blocking: nonBlocking,
|
|
1165
1307
|
jobs_skipped: skipped.map((c) => ({ name: c.job.descriptor.name, reason: c.reason })),
|
|
1166
1308
|
work_ms: workMs,
|
|
1167
1309
|
stagger_wait_ms: staggerWaitMs,
|
|
@@ -1186,4 +1328,8 @@ module.exports = {
|
|
|
1186
1328
|
MIN_REFIRE_GAP_MS,
|
|
1187
1329
|
DEFAULT_MISSED_JOB_STAGGER_MS,
|
|
1188
1330
|
DEFAULT_MAX_MISSED_JOBS_PER_RESTART,
|
|
1331
|
+
DEFAULT_NON_BLOCKING_CATCHUP_MAX_WORK_MS,
|
|
1332
|
+
DEFAULT_NON_BLOCKING_CATCHUP_DELAY_MS,
|
|
1333
|
+
DEFAULT_RESTART_DEFER_JOB_NAMES,
|
|
1334
|
+
_normalizeLane,
|
|
1189
1335
|
};
|
|
@@ -59,6 +59,7 @@ class WorkerThreadSlot {
|
|
|
59
59
|
failed: 0,
|
|
60
60
|
active: null,
|
|
61
61
|
pending: 0,
|
|
62
|
+
pendingByLane: {},
|
|
62
63
|
timedOut: 0,
|
|
63
64
|
lastTimeoutAt: null,
|
|
64
65
|
lastTimeoutOp: null,
|
|
@@ -184,7 +185,14 @@ class WorkerThreadSlot {
|
|
|
184
185
|
this._pending.set(requestId, { op, resolve, reject, timer });
|
|
185
186
|
this._status.pendingClientRequests = this._pending.size;
|
|
186
187
|
try {
|
|
187
|
-
this._worker.postMessage({
|
|
188
|
+
this._worker.postMessage({
|
|
189
|
+
type: 'request',
|
|
190
|
+
requestId,
|
|
191
|
+
op,
|
|
192
|
+
payload,
|
|
193
|
+
lane: options.lane,
|
|
194
|
+
priority: options.priority,
|
|
195
|
+
});
|
|
188
196
|
} catch (err) {
|
|
189
197
|
this._finishRequest(requestId, _serializeError(err));
|
|
190
198
|
}
|
|
@@ -8,9 +8,31 @@ const { perfLogsEnabled } = require('../lib/diagnostics-flags');
|
|
|
8
8
|
let embeddings;
|
|
9
9
|
try { embeddings = require('../embeddings'); } catch { embeddings = null; }
|
|
10
10
|
|
|
11
|
-
const
|
|
11
|
+
const DEFAULT_BATCH_SIZE = 20;
|
|
12
|
+
const LOCAL_MODEL_BATCH_SIZE = 5;
|
|
12
13
|
const THINK_PHASE_LOG_THRESHOLD_MS = Math.max(0, Number(process.env.WALLE_THINK_PHASE_LOG_THRESHOLD_MS || 1000));
|
|
13
14
|
|
|
15
|
+
function resolveThinkBatchSize(opts = {}, env = process.env) {
|
|
16
|
+
const explicit = Number(opts.batchSize ?? env.WALL_E_THINK_BATCH_SIZE ?? env.WALLE_THINK_BATCH_SIZE);
|
|
17
|
+
if (Number.isFinite(explicit) && explicit > 0) return Math.max(1, Math.min(100, Math.trunc(explicit)));
|
|
18
|
+
|
|
19
|
+
const hint = [
|
|
20
|
+
opts.model,
|
|
21
|
+
opts.provider,
|
|
22
|
+
env.WALL_E_PROVIDER,
|
|
23
|
+
env.WALLE_PROVIDER,
|
|
24
|
+
env.WALL_E_MODEL_PROVIDER,
|
|
25
|
+
env.WALLE_MODEL_PROVIDER,
|
|
26
|
+
env.WALL_E_MODEL,
|
|
27
|
+
env.WALLE_MODEL,
|
|
28
|
+
].filter(Boolean).join(' ').toLowerCase();
|
|
29
|
+
|
|
30
|
+
if (/(^|[\s/@:_-])(ollama|local|llama|gemma|qwen|mistral|phi)(?=$|[\s/@:_-]|\d)/.test(hint)) {
|
|
31
|
+
return LOCAL_MODEL_BATCH_SIZE;
|
|
32
|
+
}
|
|
33
|
+
return DEFAULT_BATCH_SIZE;
|
|
34
|
+
}
|
|
35
|
+
|
|
14
36
|
function _phaseMeta(meta = {}) {
|
|
15
37
|
const out = {};
|
|
16
38
|
for (const [key, value] of Object.entries(meta || {})) {
|
|
@@ -90,7 +112,8 @@ async function runOnce(opts = {}) {
|
|
|
90
112
|
const extractFn = opts.extractFn || extractKnowledge;
|
|
91
113
|
const ownerName = brain.getOwnerName();
|
|
92
114
|
|
|
93
|
-
const
|
|
115
|
+
const batchSize = resolveThinkBatchSize(opts);
|
|
116
|
+
const pending = brain.listMemories({ extractionStatus: 'pending', limit: batchSize });
|
|
94
117
|
if (pending.length === 0) {
|
|
95
118
|
// Still run self-resolution even with no new memories
|
|
96
119
|
await selfResolveQuestions(ownerName, opts);
|
|
@@ -281,4 +304,4 @@ async function selfResolveQuestions(ownerName, opts = {}) {
|
|
|
281
304
|
}
|
|
282
305
|
}
|
|
283
306
|
|
|
284
|
-
module.exports = { runOnce, _shouldAskAboutContradiction };
|
|
307
|
+
module.exports = { runOnce, resolveThinkBatchSize, _shouldAskAboutContradiction };
|
|
@@ -22,6 +22,25 @@ const DEFAULT_MCP_LOOKUP_CONTEXT_PACK_TIMEOUT_MS = 1200;
|
|
|
22
22
|
const DEFAULT_MCP_LOOKUP_EMBEDDING_TIMEOUT_MS = 700;
|
|
23
23
|
const MAX_MCP_LOOKUP_TIMEOUT_MS = 10_000;
|
|
24
24
|
|
|
25
|
+
function _envFlag(name) {
|
|
26
|
+
const raw = process.env[name];
|
|
27
|
+
if (raw == null || raw === '') return null;
|
|
28
|
+
if (/^(1|true|yes|on)$/i.test(String(raw).trim())) return true;
|
|
29
|
+
if (/^(0|false|no|off)$/i.test(String(raw).trim())) return false;
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function canOffloadMcpLookupToWorker() {
|
|
34
|
+
const explicit = _envFlag('WALL_E_MCP_LOOKUP_WORKER');
|
|
35
|
+
if (explicit != null) return explicit;
|
|
36
|
+
const role = process.env.WALL_E_PROCESS_ROLE || '';
|
|
37
|
+
if (role === 'wall-e-runtime-worker') return false;
|
|
38
|
+
// Runtime workers open the configured persistent brain DB. Embedded MCP imports
|
|
39
|
+
// such as tests, CTM, or stdio repair mode may have already initialized brain.js
|
|
40
|
+
// against a custom DB path, so keep those lookups in-process.
|
|
41
|
+
return role === 'walle-daemon';
|
|
42
|
+
}
|
|
43
|
+
|
|
25
44
|
const MCP_RESOURCES = [
|
|
26
45
|
{
|
|
27
46
|
uri: 'walle://status/session-memory',
|
|
@@ -722,10 +741,7 @@ async function executeTool(name, args) {
|
|
|
722
741
|
return routeWallEQuery(args.query, { context: args.context });
|
|
723
742
|
}
|
|
724
743
|
case 'walle_lookup_context': {
|
|
725
|
-
|
|
726
|
-
process.env.WALL_E_PROCESS_ROLE !== 'wall-e-runtime-worker' &&
|
|
727
|
-
(process.env.NODE_ENV !== 'test' || process.env.WALL_E_RUNTIME_WORKER_POOL_TEST === '1');
|
|
728
|
-
if (canOffload) {
|
|
744
|
+
if (canOffloadMcpLookupToWorker()) {
|
|
729
745
|
try {
|
|
730
746
|
return await require('./lib/runtime-worker-pool').requestRuntimeWorker(
|
|
731
747
|
'mcp.lookupWallEContext',
|
|
@@ -3,27 +3,98 @@
|
|
|
3
3
|
const fs = require('node:fs');
|
|
4
4
|
const path = require('node:path');
|
|
5
5
|
const crypto = require('node:crypto');
|
|
6
|
+
const { StringDecoder } = require('node:string_decoder');
|
|
6
7
|
|
|
7
8
|
const { applyTransforms } = require('./transforms');
|
|
8
9
|
|
|
9
|
-
|
|
10
|
+
const DEFAULT_JSONL_READ_CHUNK_BYTES = 256 * 1024;
|
|
11
|
+
const DEFAULT_JSONL_MAX_LINE_CHARS = 4 * 1024 * 1024;
|
|
12
|
+
|
|
13
|
+
function jsonlMaxLineChars(env = process.env) {
|
|
14
|
+
const n = Number(env.WALL_E_SOURCE_JSONL_MAX_LINE_CHARS ?? env.WALLE_SOURCE_JSONL_MAX_LINE_CHARS);
|
|
15
|
+
if (!Number.isFinite(n) || n <= 0) return DEFAULT_JSONL_MAX_LINE_CHARS;
|
|
16
|
+
return Math.max(4096, Math.trunc(n));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function jsonlReadChunkBytes(env = process.env) {
|
|
20
|
+
const n = Number(env.WALL_E_SOURCE_JSONL_READ_CHUNK_BYTES ?? env.WALLE_SOURCE_JSONL_READ_CHUNK_BYTES);
|
|
21
|
+
if (!Number.isFinite(n) || n <= 0) return DEFAULT_JSONL_READ_CHUNK_BYTES;
|
|
22
|
+
return Math.max(4096, Math.min(2 * 1024 * 1024, Math.trunc(n)));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function _processJsonlLine({ line, lineNo, events, diagnostics, filePath, maxLineChars }) {
|
|
26
|
+
if (!line || !line.trim()) return;
|
|
27
|
+
if (line.length > maxLineChars) {
|
|
28
|
+
diagnostics.push({
|
|
29
|
+
level: 'warn',
|
|
30
|
+
message: `Skipping oversized JSONL line ${lineNo}: ${line.length} chars exceeds ${maxLineChars}`,
|
|
31
|
+
filePath,
|
|
32
|
+
line: lineNo,
|
|
33
|
+
code: 'jsonl_line_too_large',
|
|
34
|
+
});
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
events.push({ line: lineNo, event: JSON.parse(line) });
|
|
39
|
+
} catch (err) {
|
|
40
|
+
diagnostics.push({ level: 'warn', message: `Malformed JSONL line ${lineNo}: ${err.message}`, filePath, line: lineNo });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function readJsonlFile(filePath, opts = {}) {
|
|
10
45
|
const events = [];
|
|
11
46
|
const diagnostics = [];
|
|
12
|
-
|
|
47
|
+
const maxLineChars = opts.maxLineChars || jsonlMaxLineChars(opts.env || process.env);
|
|
48
|
+
const chunkBytes = opts.chunkBytes || jsonlReadChunkBytes(opts.env || process.env);
|
|
49
|
+
let fd = null;
|
|
13
50
|
try {
|
|
14
|
-
|
|
51
|
+
fd = fs.openSync(filePath, 'r');
|
|
15
52
|
} catch (err) {
|
|
16
53
|
return { events, diagnostics: [{ level: 'error', message: err.message, filePath }] };
|
|
17
54
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
55
|
+
|
|
56
|
+
const decoder = new StringDecoder('utf8');
|
|
57
|
+
const buffer = Buffer.allocUnsafe(chunkBytes);
|
|
58
|
+
let pending = '';
|
|
59
|
+
let lineNo = 1;
|
|
60
|
+
let skippingOversize = false;
|
|
61
|
+
try {
|
|
62
|
+
while (true) {
|
|
63
|
+
const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, null);
|
|
64
|
+
if (bytesRead <= 0) break;
|
|
65
|
+
pending += decoder.write(buffer.subarray(0, bytesRead));
|
|
66
|
+
let newlineIndex;
|
|
67
|
+
while ((newlineIndex = pending.indexOf('\n')) !== -1) {
|
|
68
|
+
const line = pending.slice(0, newlineIndex).replace(/\r$/, '');
|
|
69
|
+
pending = pending.slice(newlineIndex + 1);
|
|
70
|
+
if (skippingOversize) {
|
|
71
|
+
skippingOversize = false;
|
|
72
|
+
lineNo += 1;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
_processJsonlLine({ line, lineNo, events, diagnostics, filePath, maxLineChars });
|
|
76
|
+
lineNo += 1;
|
|
77
|
+
}
|
|
78
|
+
if (pending.length > maxLineChars) {
|
|
79
|
+
diagnostics.push({
|
|
80
|
+
level: 'warn',
|
|
81
|
+
message: `Skipping oversized JSONL line ${lineNo}: exceeds ${maxLineChars} chars while streaming`,
|
|
82
|
+
filePath,
|
|
83
|
+
line: lineNo,
|
|
84
|
+
code: 'jsonl_line_too_large',
|
|
85
|
+
});
|
|
86
|
+
pending = '';
|
|
87
|
+
skippingOversize = true;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
pending += decoder.end();
|
|
91
|
+
if (!skippingOversize) {
|
|
92
|
+
_processJsonlLine({ line: pending.replace(/\r$/, ''), lineNo, events, diagnostics, filePath, maxLineChars });
|
|
26
93
|
}
|
|
94
|
+
} catch (err) {
|
|
95
|
+
diagnostics.push({ level: 'error', message: err.message, filePath });
|
|
96
|
+
} finally {
|
|
97
|
+
try { fs.closeSync(fd); } catch {}
|
|
27
98
|
}
|
|
28
99
|
return { events, diagnostics };
|
|
29
100
|
}
|
|
@@ -174,6 +245,8 @@ module.exports = {
|
|
|
174
245
|
looksLikeToolResult,
|
|
175
246
|
parseToolInput,
|
|
176
247
|
readJsonlFile,
|
|
248
|
+
jsonlMaxLineChars,
|
|
249
|
+
jsonlReadChunkBytes,
|
|
177
250
|
sourceIdFromFile,
|
|
178
251
|
summarizeToolInput,
|
|
179
252
|
toIso,
|
|
@@ -1531,6 +1531,18 @@ function hasShellControlArg(args) {
|
|
|
1531
1531
|
return args.some((arg) => /^(?:[<>]|<<|>>|\||&&|\|\||;)$/.test(String(arg || '').trim()));
|
|
1532
1532
|
}
|
|
1533
1533
|
|
|
1534
|
+
// Cloning a project/site into a working copy is the scatter antipattern (session
|
|
1535
|
+
// c3f3af97: `cp -R` the source 34× into site-v2-fixed-* throwaway trees the user
|
|
1536
|
+
// never opened, then edited those instead of the real file). `cp` has legitimate
|
|
1537
|
+
// uses, so we don't hard-block it — we surface a non-fatal hint steering the agent
|
|
1538
|
+
// back to editing the target in place. Matches recursive copies (cp -R/-r/-a,
|
|
1539
|
+
// --recursive/--archive) and rsync, not plain single-file copies.
|
|
1540
|
+
const PROJECT_CLONE_RE = /\bcp\s+(?:-[a-zA-Z]*[rRa][a-zA-Z]*|--(?:recursive|archive))\b|\brsync\b/;
|
|
1541
|
+
const CLONE_ADVISORY = 'Recursive directory copy detected. Do not clone a project/site into a copy or variant directory (e.g. *-fixed, *-v2, *-backup, a timestamped dir) as a place to work — that strands your edits in a throwaway tree the user never opens and forces re-investigation next turn. Edit the target files in place at their real path. If you were told to keep your version in a specific folder, create that ONE folder once and keep editing it; reuse a destination you already made instead of copying the source again.';
|
|
1542
|
+
function looksLikeProjectClone(commandStr) {
|
|
1543
|
+
return typeof commandStr === 'string' && PROJECT_CLONE_RE.test(commandStr);
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1534
1546
|
/**
|
|
1535
1547
|
* Run a shell command with tree-sitter analysis gating.
|
|
1536
1548
|
*
|
|
@@ -1655,6 +1667,9 @@ async function runShell(commandOrOpts, legacyArgs, legacyOpts) {
|
|
|
1655
1667
|
if (looksLikeSerialLoop && elapsedMs >= SHELL_PROGRESS_AFTER_MS) {
|
|
1656
1668
|
result.hint = `This serial loop took ${Math.round(elapsedMs / 1000)}s. For many similar operations, prefer ONE batched command (most CLIs accept multiple paths) or run it with background:true and poll bg_output, instead of a loop of separate run_shell calls.`;
|
|
1657
1669
|
}
|
|
1670
|
+
if (looksLikeProjectClone(commandStr)) {
|
|
1671
|
+
result.hint = result.hint ? `${result.hint} ${CLONE_ADVISORY}` : CLONE_ADVISORY;
|
|
1672
|
+
}
|
|
1658
1673
|
return result;
|
|
1659
1674
|
} catch (err) {
|
|
1660
1675
|
// User Stop / turn abort: the child was killed via the AbortSignal. Return a
|
|
@@ -5433,6 +5448,7 @@ module.exports = {
|
|
|
5433
5448
|
MAIL_AUTOMATION_LOCK_PATH, MAIL_AUTOMATION_COOLDOWN_PATH,
|
|
5434
5449
|
getSystemInfo, webFetch,
|
|
5435
5450
|
globFiles, grepFiles, listDirectory, isBinaryFile,
|
|
5451
|
+
looksLikeProjectClone,
|
|
5436
5452
|
SHELL_ALLOWLIST, SHELL_DENYLIST,
|
|
5437
5453
|
isProtectedPath, resolveProjectPath,
|
|
5438
5454
|
_collectGmailAttachments, _decodeBase64UrlToBuffer, _sanitizeAttachmentFilename,
|
|
@@ -10,6 +10,20 @@ let completed = 0;
|
|
|
10
10
|
let failed = 0;
|
|
11
11
|
const queue = [];
|
|
12
12
|
|
|
13
|
+
function _priorityValue(value, fallback) {
|
|
14
|
+
const n = Number(value);
|
|
15
|
+
return Number.isFinite(n) ? Math.max(0, Math.trunc(n)) : fallback;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function _queuedByLane() {
|
|
19
|
+
const byLane = {};
|
|
20
|
+
for (const task of queue) {
|
|
21
|
+
const lane = String(task.lane);
|
|
22
|
+
byLane[lane] = (byLane[lane] || 0) + 1;
|
|
23
|
+
}
|
|
24
|
+
return byLane;
|
|
25
|
+
}
|
|
26
|
+
|
|
13
27
|
function _serializeError(err) {
|
|
14
28
|
return {
|
|
15
29
|
message: err && err.message ? err.message : String(err || 'unknown error'),
|
|
@@ -24,6 +38,7 @@ function _status() {
|
|
|
24
38
|
active,
|
|
25
39
|
running: !!active,
|
|
26
40
|
pending: queue.length,
|
|
41
|
+
pendingByLane: _queuedByLane(),
|
|
27
42
|
enqueued,
|
|
28
43
|
completed,
|
|
29
44
|
failed,
|
|
@@ -94,7 +109,16 @@ function _enqueue(msg) {
|
|
|
94
109
|
requestId: msg.requestId,
|
|
95
110
|
op: msg.op,
|
|
96
111
|
payload: msg.payload || {},
|
|
112
|
+
lane: _priorityValue(msg.lane, 2),
|
|
113
|
+
priority: _priorityValue(msg.priority, 5),
|
|
114
|
+
enqueuedAt: Date.now(),
|
|
97
115
|
});
|
|
116
|
+
queue.sort((a, b) => (
|
|
117
|
+
a.lane - b.lane
|
|
118
|
+
|| a.priority - b.priority
|
|
119
|
+
|| a.enqueuedAt - b.enqueuedAt
|
|
120
|
+
|| a.requestId - b.requestId
|
|
121
|
+
));
|
|
98
122
|
_pump();
|
|
99
123
|
}
|
|
100
124
|
|