create-walle 0.9.26 → 0.9.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,6 +29,30 @@ const DEFAULT_MAX_MISSED_JOBS_PER_RESTART = 5;
29
29
  const DEFAULT_SLOW_TICK_MS = 250;
30
30
  const DEFAULT_SLOW_JOB_MS = 2_000;
31
31
  const PERF_LOG_THROTTLE_MS = 30_000;
32
+ const DEFAULT_NON_BLOCKING_CATCHUP_MAX_WORK_MS = 5_000;
33
+ const DEFAULT_NON_BLOCKING_CATCHUP_DELAY_MS = 2 * 60 * 1000;
34
+ const DEFAULT_RESTART_DEFER_JOB_NAMES = [
35
+ 'harvest',
36
+ 'training',
37
+ 'replay',
38
+ 'agent-benchmark',
39
+ 'weekly-eval-loop',
40
+ 'brain-optimize',
41
+ 'dedup',
42
+ 'daily-backup',
43
+ 'brain-retention',
44
+ 'question-digest',
45
+ ];
46
+
47
+ function _normalizeLane(value, priority = 5) {
48
+ const n = Number(value);
49
+ if (Number.isFinite(n)) return Math.max(0, Math.trunc(n));
50
+ const p = Number(priority);
51
+ if (!Number.isFinite(p)) return 2;
52
+ if (p <= 2) return 0;
53
+ if (p <= 5) return 1;
54
+ return 2;
55
+ }
32
56
 
33
57
  /** Default cascade check: true if any numeric value in result is > 0 */
34
58
  function _defaultCascadeCheck(result) {
@@ -130,6 +154,7 @@ class Scheduler extends EventEmitter {
130
154
  * @param {number} desc.intervalMs
131
155
  * @param {string} [desc.pool] - Resource pool name
132
156
  * @param {number} [desc.priority=5] - 1 (highest) to 10 (lowest)
157
+ * @param {number} [desc.lane] - 0 foreground, 1 normal, 2 background
133
158
  * @param {string[]} [desc.dependsOn] - Jobs that must complete once before this runs
134
159
  * @param {string[]} [desc.cascadeFrom] - Jobs whose results trigger immediate eligibility
135
160
  * @param {number} [desc.startDelayMs=0]
@@ -155,6 +180,7 @@ class Scheduler extends EventEmitter {
155
180
  worker: this._workerJobNames.has(desc.name),
156
181
  ...desc,
157
182
  };
183
+ desc.lane = _normalizeLane(desc.lane, desc.priority);
158
184
  if (desc.worker && !desc.timeoutMs && this._defaultWorkerJobTimeoutMs > 0) {
159
185
  desc.timeoutMs = this._defaultWorkerJobTimeoutMs;
160
186
  }
@@ -298,7 +324,7 @@ class Scheduler extends EventEmitter {
298
324
  getJobs() {
299
325
  return Object.entries(this.getState())
300
326
  .map(([name, state]) => ({ name, ...state }))
301
- .sort((a, b) => a.priority - b.priority || a.name.localeCompare(b.name));
327
+ .sort((a, b) => a.lane - b.lane || a.priority - b.priority || a.name.localeCompare(b.name));
302
328
  }
303
329
 
304
330
  /**
@@ -374,6 +400,7 @@ class Scheduler extends EventEmitter {
374
400
  return {
375
401
  enabled: desc.enabled,
376
402
  pool: desc.pool || null,
403
+ lane: desc.lane,
377
404
  priority: desc.priority,
378
405
  intervalMs: desc.intervalMs,
379
406
  timeoutMs: desc.timeoutMs || null,
@@ -681,8 +708,12 @@ class Scheduler extends EventEmitter {
681
708
  }
682
709
  summary.candidates = candidates.length;
683
710
 
684
- // 3. Sort by priority (lower = higher priority)
685
- candidates.sort((a, b) => a.descriptor.priority - b.descriptor.priority);
711
+ // 3. Sort by lane first (foreground before background), then priority.
712
+ candidates.sort((a, b) =>
713
+ a.descriptor.lane - b.descriptor.lane
714
+ || a.descriptor.priority - b.descriptor.priority
715
+ || a.descriptor.name.localeCompare(b.descriptor.name)
716
+ );
686
717
 
687
718
  // 4. Dispatch respecting pool limits. Worker-thread requests register
688
719
  // asynchronously inside WorkerThreadSlot.request(), so reserve the slots we
@@ -802,6 +833,38 @@ class Scheduler extends EventEmitter {
802
833
  }
803
834
  }
804
835
 
836
+ _completeSchedulerCatchupWorkItem(job, result) {
837
+ const id = job && job._runtimeCatchupWorkItemId;
838
+ if (!id || !this._persistBrain || typeof this._persistBrain.completeRuntimeWorkItem !== 'function') return;
839
+ try {
840
+ this._persistBrain.completeRuntimeWorkItem(id, {
841
+ cursor: {
842
+ completedBy: 'scheduler',
843
+ runCount: result && typeof result === 'object' ? result.count : undefined,
844
+ },
845
+ });
846
+ job._runtimeCatchupWorkItemId = null;
847
+ } catch (e) {
848
+ const name = job?.descriptor?.name || id;
849
+ console.warn(`[scheduler] Could not complete catch-up work item for "${name}": ${e.message}`);
850
+ }
851
+ }
852
+
853
+ _failSchedulerCatchupWorkItem(job, err) {
854
+ const id = job && job._runtimeCatchupWorkItemId;
855
+ if (!id || !this._persistBrain || typeof this._persistBrain.failRuntimeWorkItem !== 'function') return;
856
+ try {
857
+ this._persistBrain.failRuntimeWorkItem(id, {
858
+ error: err,
859
+ retryAfterMs: this._minRefireGapMs,
860
+ });
861
+ job._runtimeCatchupWorkItemId = null;
862
+ } catch (e) {
863
+ const name = job?.descriptor?.name || id;
864
+ console.warn(`[scheduler] Could not update failed catch-up work item for "${name}": ${e.message}`);
865
+ }
866
+ }
867
+
805
868
  _dispatch(job) {
806
869
  const { name, pool, run, onResult, onError } = job.descriptor;
807
870
  job.state.status = 'running';
@@ -857,6 +920,7 @@ class Scheduler extends EventEmitter {
857
920
  job.state.lastError = null;
858
921
  job.state.runCount++;
859
922
  job.state.completedOnce = true;
923
+ this._completeSchedulerCatchupWorkItem(job, result);
860
924
 
861
925
  // Recovery: clear consecutive-error tracking + dismiss any active
862
926
  // scheduler:job_failure:<name> alert. Failure-alert behavior is
@@ -889,6 +953,7 @@ class Scheduler extends EventEmitter {
889
953
  failure = err;
890
954
  job.state.lastError = err.message;
891
955
  job.state.errorCount++;
956
+ this._failSchedulerCatchupWorkItem(job, err);
892
957
 
893
958
  // Failure-alert hook: bumps consecutiveErrors, emits service alert
894
959
  // when threshold (default 2) hit and outside cooldown (default 1h).
@@ -950,6 +1015,8 @@ class Scheduler extends EventEmitter {
950
1015
  const result = await this._workerPool.request('scheduler.runJob', payload, {
951
1016
  timeoutMs,
952
1017
  heavy: true,
1018
+ lane: desc.lane,
1019
+ priority: desc.priority,
953
1020
  terminateOnTimeout: timeoutMs > 0,
954
1021
  });
955
1022
  workerMeta.worker_used = true;
@@ -1053,9 +1120,19 @@ class Scheduler extends EventEmitter {
1053
1120
  async runMissedJobs(opts = {}) {
1054
1121
  const max = opts.max ?? this._maxMissedJobsPerRestart;
1055
1122
  const staggerMs = opts.staggerMs ?? this._missedJobStaggerMs;
1123
+ const nonBlocking = opts.nonBlocking === true;
1124
+ const deferNames = new Set(opts.deferNames || DEFAULT_RESTART_DEFER_JOB_NAMES);
1125
+ const deferDelayMs = Number.isFinite(Number(opts.deferDelayMs))
1126
+ ? Math.max(0, Math.trunc(Number(opts.deferDelayMs)))
1127
+ : DEFAULT_NON_BLOCKING_CATCHUP_DELAY_MS;
1128
+ const maxWorkMs = Number.isFinite(Number(opts.maxWorkMs))
1129
+ ? Math.max(0, Math.trunc(Number(opts.maxWorkMs)))
1130
+ : DEFAULT_NON_BLOCKING_CATCHUP_MAX_WORK_MS;
1131
+ const enqueueWorkItem = typeof opts.enqueueWorkItem === 'function' ? opts.enqueueWorkItem : null;
1056
1132
  const op = runtimeHealth.beginOperation('scheduler.runMissedJobs', {
1057
1133
  max,
1058
1134
  staggerMs,
1135
+ nonBlocking,
1059
1136
  });
1060
1137
  if (!this._persistJobState || !this._persistBrain) {
1061
1138
  const summary = { ran: 0, deferred: 0, reason: 'persistence-disabled' };
@@ -1104,8 +1181,55 @@ class Scheduler extends EventEmitter {
1104
1181
  candidates.sort((a, b) =>
1105
1182
  (a.stored.next_eligible_at || 0) - (b.stored.next_eligible_at || 0)
1106
1183
  );
1107
- const immediate = candidates.slice(0, max);
1108
- const deferred = candidates.slice(max);
1184
+ let eligibleCandidates = candidates;
1185
+ const policyDeferred = [];
1186
+ if (nonBlocking) {
1187
+ eligibleCandidates = [];
1188
+ for (const candidate of candidates) {
1189
+ const policy = candidate.job.descriptor.restartCatchupPolicy
1190
+ || (deferNames.has(candidate.stored.job_name) ? 'defer' : 'run');
1191
+ if (policy === 'skip' || policy === 'defer') {
1192
+ policyDeferred.push({ ...candidate, reason: policy === 'skip' ? 'restart-skip' : 'restart-defer-policy' });
1193
+ } else {
1194
+ eligibleCandidates.push(candidate);
1195
+ }
1196
+ }
1197
+ }
1198
+
1199
+ const immediate = eligibleCandidates.slice(0, max);
1200
+ const deferred = [
1201
+ ...eligibleCandidates.slice(max).map((candidate) => ({ ...candidate, reason: 'restart-catchup-cap' })),
1202
+ ...policyDeferred,
1203
+ ];
1204
+
1205
+ if (nonBlocking && deferred.length > 0) {
1206
+ const deferredAt = Date.now();
1207
+ deferred.forEach((candidate, index) => {
1208
+ const { job } = candidate;
1209
+ job.state.nextEligibleAt = deferredAt + deferDelayMs + (index * Math.min(5000, this._minRefireGapMs));
1210
+ this._persistState(job.descriptor.name, job);
1211
+ if (enqueueWorkItem) {
1212
+ const workItemId = `scheduler-catchup:${job.descriptor.name}`;
1213
+ try {
1214
+ enqueueWorkItem({
1215
+ id: workItemId,
1216
+ kind: 'scheduler.catchup',
1217
+ lane: job.descriptor.lane,
1218
+ priority: job.descriptor.priority,
1219
+ payload: {
1220
+ jobName: job.descriptor.name,
1221
+ reason: candidate.reason,
1222
+ trigger: 'restart-catchup',
1223
+ },
1224
+ not_before: job.state.nextEligibleAt,
1225
+ });
1226
+ job._runtimeCatchupWorkItemId = workItemId;
1227
+ } catch (e) {
1228
+ console.warn(`[scheduler] Could not enqueue catch-up work item for "${job.descriptor.name}": ${e.message}`);
1229
+ }
1230
+ }
1231
+ });
1232
+ }
1109
1233
 
1110
1234
  if (deferred.length > 0) {
1111
1235
  console.log(
@@ -1130,8 +1254,14 @@ class Scheduler extends EventEmitter {
1130
1254
  let ran = 0;
1131
1255
  let workMs = 0;
1132
1256
  let staggerWaitMs = 0;
1257
+ let budgetDeferred = [];
1258
+ const jobsRan = [];
1133
1259
  for (let i = 0; i < immediate.length; i++) {
1134
1260
  const { job } = immediate[i];
1261
+ if (nonBlocking && workMs >= maxWorkMs && ran > 0) {
1262
+ budgetDeferred = immediate.slice(i).map((candidate) => ({ ...candidate, reason: 'restart-catchup-budget' }));
1263
+ break;
1264
+ }
1135
1265
  // Mark eligible RIGHT NOW so _dispatch picks it up
1136
1266
  job.state.nextEligibleAt = Date.now();
1137
1267
  job._wakeTrigger = 'restart-catchup';
@@ -1141,9 +1271,10 @@ class Scheduler extends EventEmitter {
1141
1271
  try {
1142
1272
  const workStartedAt = Date.now();
1143
1273
  this._dispatch(job);
1144
- if (job.runPromise) await job.runPromise;
1274
+ if (!nonBlocking && job.runPromise) await job.runPromise;
1145
1275
  workMs += Date.now() - workStartedAt;
1146
1276
  ran++;
1277
+ jobsRan.push(job.descriptor.name);
1147
1278
  } catch (e) {
1148
1279
  console.warn(`[scheduler] Missed-job replay for "${job.descriptor.name}" threw: ${e.message}`);
1149
1280
  }
@@ -1154,14 +1285,25 @@ class Scheduler extends EventEmitter {
1154
1285
  }
1155
1286
  }
1156
1287
 
1288
+ if (nonBlocking && budgetDeferred.length > 0) {
1289
+ const deferredAt = Date.now();
1290
+ budgetDeferred.forEach((candidate, index) => {
1291
+ const { job } = candidate;
1292
+ job.state.nextEligibleAt = deferredAt + deferDelayMs + (index * Math.min(5000, this._minRefireGapMs));
1293
+ this._persistState(job.descriptor.name, job);
1294
+ });
1295
+ }
1296
+
1157
1297
  const summary = {
1158
1298
  ran,
1159
- deferred: deferred.length + skipped.length,
1160
- jobs_ran: immediate.map((c) => c.stored.job_name),
1299
+ deferred: deferred.length + skipped.length + budgetDeferred.length,
1300
+ jobs_ran: jobsRan,
1161
1301
  jobs_deferred: [
1162
1302
  ...deferred.map((c) => c.stored.job_name),
1303
+ ...budgetDeferred.map((c) => c.stored.job_name),
1163
1304
  ...skipped.map((c) => c.job.descriptor.name),
1164
1305
  ],
1306
+ non_blocking: nonBlocking,
1165
1307
  jobs_skipped: skipped.map((c) => ({ name: c.job.descriptor.name, reason: c.reason })),
1166
1308
  work_ms: workMs,
1167
1309
  stagger_wait_ms: staggerWaitMs,
@@ -1186,4 +1328,8 @@ module.exports = {
1186
1328
  MIN_REFIRE_GAP_MS,
1187
1329
  DEFAULT_MISSED_JOB_STAGGER_MS,
1188
1330
  DEFAULT_MAX_MISSED_JOBS_PER_RESTART,
1331
+ DEFAULT_NON_BLOCKING_CATCHUP_MAX_WORK_MS,
1332
+ DEFAULT_NON_BLOCKING_CATCHUP_DELAY_MS,
1333
+ DEFAULT_RESTART_DEFER_JOB_NAMES,
1334
+ _normalizeLane,
1189
1335
  };
@@ -59,6 +59,7 @@ class WorkerThreadSlot {
59
59
  failed: 0,
60
60
  active: null,
61
61
  pending: 0,
62
+ pendingByLane: {},
62
63
  timedOut: 0,
63
64
  lastTimeoutAt: null,
64
65
  lastTimeoutOp: null,
@@ -184,7 +185,14 @@ class WorkerThreadSlot {
184
185
  this._pending.set(requestId, { op, resolve, reject, timer });
185
186
  this._status.pendingClientRequests = this._pending.size;
186
187
  try {
187
- this._worker.postMessage({ type: 'request', requestId, op, payload });
188
+ this._worker.postMessage({
189
+ type: 'request',
190
+ requestId,
191
+ op,
192
+ payload,
193
+ lane: options.lane,
194
+ priority: options.priority,
195
+ });
188
196
  } catch (err) {
189
197
  this._finishRequest(requestId, _serializeError(err));
190
198
  }
@@ -8,9 +8,31 @@ const { perfLogsEnabled } = require('../lib/diagnostics-flags');
8
8
  let embeddings;
9
9
  try { embeddings = require('../embeddings'); } catch { embeddings = null; }
10
10
 
11
- const BATCH_SIZE = 20;
11
+ const DEFAULT_BATCH_SIZE = 20;
12
+ const LOCAL_MODEL_BATCH_SIZE = 5;
12
13
  const THINK_PHASE_LOG_THRESHOLD_MS = Math.max(0, Number(process.env.WALLE_THINK_PHASE_LOG_THRESHOLD_MS || 1000));
13
14
 
15
+ function resolveThinkBatchSize(opts = {}, env = process.env) {
16
+ const explicit = Number(opts.batchSize ?? env.WALL_E_THINK_BATCH_SIZE ?? env.WALLE_THINK_BATCH_SIZE);
17
+ if (Number.isFinite(explicit) && explicit > 0) return Math.max(1, Math.min(100, Math.trunc(explicit)));
18
+
19
+ const hint = [
20
+ opts.model,
21
+ opts.provider,
22
+ env.WALL_E_PROVIDER,
23
+ env.WALLE_PROVIDER,
24
+ env.WALL_E_MODEL_PROVIDER,
25
+ env.WALLE_MODEL_PROVIDER,
26
+ env.WALL_E_MODEL,
27
+ env.WALLE_MODEL,
28
+ ].filter(Boolean).join(' ').toLowerCase();
29
+
30
+ if (/(^|[\s/@:_-])(ollama|local|llama|gemma|qwen|mistral|phi)(?=$|[\s/@:_-]|\d)/.test(hint)) {
31
+ return LOCAL_MODEL_BATCH_SIZE;
32
+ }
33
+ return DEFAULT_BATCH_SIZE;
34
+ }
35
+
14
36
  function _phaseMeta(meta = {}) {
15
37
  const out = {};
16
38
  for (const [key, value] of Object.entries(meta || {})) {
@@ -90,7 +112,8 @@ async function runOnce(opts = {}) {
90
112
  const extractFn = opts.extractFn || extractKnowledge;
91
113
  const ownerName = brain.getOwnerName();
92
114
 
93
- const pending = brain.listMemories({ extractionStatus: 'pending', limit: BATCH_SIZE });
115
+ const batchSize = resolveThinkBatchSize(opts);
116
+ const pending = brain.listMemories({ extractionStatus: 'pending', limit: batchSize });
94
117
  if (pending.length === 0) {
95
118
  // Still run self-resolution even with no new memories
96
119
  await selfResolveQuestions(ownerName, opts);
@@ -281,4 +304,4 @@ async function selfResolveQuestions(ownerName, opts = {}) {
281
304
  }
282
305
  }
283
306
 
284
- module.exports = { runOnce, _shouldAskAboutContradiction };
307
+ module.exports = { runOnce, resolveThinkBatchSize, _shouldAskAboutContradiction };
@@ -22,6 +22,25 @@ const DEFAULT_MCP_LOOKUP_CONTEXT_PACK_TIMEOUT_MS = 1200;
22
22
  const DEFAULT_MCP_LOOKUP_EMBEDDING_TIMEOUT_MS = 700;
23
23
  const MAX_MCP_LOOKUP_TIMEOUT_MS = 10_000;
24
24
 
25
+ function _envFlag(name) {
26
+ const raw = process.env[name];
27
+ if (raw == null || raw === '') return null;
28
+ if (/^(1|true|yes|on)$/i.test(String(raw).trim())) return true;
29
+ if (/^(0|false|no|off)$/i.test(String(raw).trim())) return false;
30
+ return null;
31
+ }
32
+
33
+ function canOffloadMcpLookupToWorker() {
34
+ const explicit = _envFlag('WALL_E_MCP_LOOKUP_WORKER');
35
+ if (explicit != null) return explicit;
36
+ const role = process.env.WALL_E_PROCESS_ROLE || '';
37
+ if (role === 'wall-e-runtime-worker') return false;
38
+ // Runtime workers open the configured persistent brain DB. Embedded MCP imports
39
+ // such as tests, CTM, or stdio repair mode may have already initialized brain.js
40
+ // against a custom DB path, so keep those lookups in-process.
41
+ return role === 'walle-daemon';
42
+ }
43
+
25
44
  const MCP_RESOURCES = [
26
45
  {
27
46
  uri: 'walle://status/session-memory',
@@ -722,10 +741,7 @@ async function executeTool(name, args) {
722
741
  return routeWallEQuery(args.query, { context: args.context });
723
742
  }
724
743
  case 'walle_lookup_context': {
725
- const canOffload =
726
- process.env.WALL_E_PROCESS_ROLE !== 'wall-e-runtime-worker' &&
727
- (process.env.NODE_ENV !== 'test' || process.env.WALL_E_RUNTIME_WORKER_POOL_TEST === '1');
728
- if (canOffload) {
744
+ if (canOffloadMcpLookupToWorker()) {
729
745
  try {
730
746
  return await require('./lib/runtime-worker-pool').requestRuntimeWorker(
731
747
  'mcp.lookupWallEContext',
@@ -3,27 +3,98 @@
3
3
  const fs = require('node:fs');
4
4
  const path = require('node:path');
5
5
  const crypto = require('node:crypto');
6
+ const { StringDecoder } = require('node:string_decoder');
6
7
 
7
8
  const { applyTransforms } = require('./transforms');
8
9
 
9
- function readJsonlFile(filePath) {
10
+ const DEFAULT_JSONL_READ_CHUNK_BYTES = 256 * 1024;
11
+ const DEFAULT_JSONL_MAX_LINE_CHARS = 4 * 1024 * 1024;
12
+
13
+ function jsonlMaxLineChars(env = process.env) {
14
+ const n = Number(env.WALL_E_SOURCE_JSONL_MAX_LINE_CHARS ?? env.WALLE_SOURCE_JSONL_MAX_LINE_CHARS);
15
+ if (!Number.isFinite(n) || n <= 0) return DEFAULT_JSONL_MAX_LINE_CHARS;
16
+ return Math.max(4096, Math.trunc(n));
17
+ }
18
+
19
+ function jsonlReadChunkBytes(env = process.env) {
20
+ const n = Number(env.WALL_E_SOURCE_JSONL_READ_CHUNK_BYTES ?? env.WALLE_SOURCE_JSONL_READ_CHUNK_BYTES);
21
+ if (!Number.isFinite(n) || n <= 0) return DEFAULT_JSONL_READ_CHUNK_BYTES;
22
+ return Math.max(4096, Math.min(2 * 1024 * 1024, Math.trunc(n)));
23
+ }
24
+
25
+ function _processJsonlLine({ line, lineNo, events, diagnostics, filePath, maxLineChars }) {
26
+ if (!line || !line.trim()) return;
27
+ if (line.length > maxLineChars) {
28
+ diagnostics.push({
29
+ level: 'warn',
30
+ message: `Skipping oversized JSONL line ${lineNo}: ${line.length} chars exceeds ${maxLineChars}`,
31
+ filePath,
32
+ line: lineNo,
33
+ code: 'jsonl_line_too_large',
34
+ });
35
+ return;
36
+ }
37
+ try {
38
+ events.push({ line: lineNo, event: JSON.parse(line) });
39
+ } catch (err) {
40
+ diagnostics.push({ level: 'warn', message: `Malformed JSONL line ${lineNo}: ${err.message}`, filePath, line: lineNo });
41
+ }
42
+ }
43
+
44
+ function readJsonlFile(filePath, opts = {}) {
10
45
  const events = [];
11
46
  const diagnostics = [];
12
- let raw = '';
47
+ const maxLineChars = opts.maxLineChars || jsonlMaxLineChars(opts.env || process.env);
48
+ const chunkBytes = opts.chunkBytes || jsonlReadChunkBytes(opts.env || process.env);
49
+ let fd = null;
13
50
  try {
14
- raw = fs.readFileSync(filePath, 'utf8');
51
+ fd = fs.openSync(filePath, 'r');
15
52
  } catch (err) {
16
53
  return { events, diagnostics: [{ level: 'error', message: err.message, filePath }] };
17
54
  }
18
- const lines = raw.split('\n');
19
- for (let i = 0; i < lines.length; i++) {
20
- const line = lines[i];
21
- if (!line.trim()) continue;
22
- try {
23
- events.push({ line: i + 1, event: JSON.parse(line) });
24
- } catch (err) {
25
- diagnostics.push({ level: 'warn', message: `Malformed JSONL line ${i + 1}: ${err.message}`, filePath, line: i + 1 });
55
+
56
+ const decoder = new StringDecoder('utf8');
57
+ const buffer = Buffer.allocUnsafe(chunkBytes);
58
+ let pending = '';
59
+ let lineNo = 1;
60
+ let skippingOversize = false;
61
+ try {
62
+ while (true) {
63
+ const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, null);
64
+ if (bytesRead <= 0) break;
65
+ pending += decoder.write(buffer.subarray(0, bytesRead));
66
+ let newlineIndex;
67
+ while ((newlineIndex = pending.indexOf('\n')) !== -1) {
68
+ const line = pending.slice(0, newlineIndex).replace(/\r$/, '');
69
+ pending = pending.slice(newlineIndex + 1);
70
+ if (skippingOversize) {
71
+ skippingOversize = false;
72
+ lineNo += 1;
73
+ continue;
74
+ }
75
+ _processJsonlLine({ line, lineNo, events, diagnostics, filePath, maxLineChars });
76
+ lineNo += 1;
77
+ }
78
+ if (pending.length > maxLineChars) {
79
+ diagnostics.push({
80
+ level: 'warn',
81
+ message: `Skipping oversized JSONL line ${lineNo}: exceeds ${maxLineChars} chars while streaming`,
82
+ filePath,
83
+ line: lineNo,
84
+ code: 'jsonl_line_too_large',
85
+ });
86
+ pending = '';
87
+ skippingOversize = true;
88
+ }
89
+ }
90
+ pending += decoder.end();
91
+ if (!skippingOversize) {
92
+ _processJsonlLine({ line: pending.replace(/\r$/, ''), lineNo, events, diagnostics, filePath, maxLineChars });
26
93
  }
94
+ } catch (err) {
95
+ diagnostics.push({ level: 'error', message: err.message, filePath });
96
+ } finally {
97
+ try { fs.closeSync(fd); } catch {}
27
98
  }
28
99
  return { events, diagnostics };
29
100
  }
@@ -174,6 +245,8 @@ module.exports = {
174
245
  looksLikeToolResult,
175
246
  parseToolInput,
176
247
  readJsonlFile,
248
+ jsonlMaxLineChars,
249
+ jsonlReadChunkBytes,
177
250
  sourceIdFromFile,
178
251
  summarizeToolInput,
179
252
  toIso,
@@ -1531,6 +1531,18 @@ function hasShellControlArg(args) {
1531
1531
  return args.some((arg) => /^(?:[<>]|<<|>>|\||&&|\|\||;)$/.test(String(arg || '').trim()));
1532
1532
  }
1533
1533
 
1534
+ // Cloning a project/site into a working copy is the scatter antipattern (session
1535
+ // c3f3af97: `cp -R` the source 34× into site-v2-fixed-* throwaway trees the user
1536
+ // never opened, then edited those instead of the real file). `cp` has legitimate
1537
+ // uses, so we don't hard-block it — we surface a non-fatal hint steering the agent
1538
+ // back to editing the target in place. Matches recursive copies (cp -R/-r/-a,
1539
+ // --recursive/--archive) and rsync, not plain single-file copies.
1540
+ const PROJECT_CLONE_RE = /\bcp\s+(?:-[a-zA-Z]*[rRa][a-zA-Z]*|--(?:recursive|archive))\b|\brsync\b/;
1541
+ const CLONE_ADVISORY = 'Recursive directory copy detected. Do not clone a project/site into a copy or variant directory (e.g. *-fixed, *-v2, *-backup, a timestamped dir) as a place to work — that strands your edits in a throwaway tree the user never opens and forces re-investigation next turn. Edit the target files in place at their real path. If you were told to keep your version in a specific folder, create that ONE folder once and keep editing it; reuse a destination you already made instead of copying the source again.';
1542
+ function looksLikeProjectClone(commandStr) {
1543
+ return typeof commandStr === 'string' && PROJECT_CLONE_RE.test(commandStr);
1544
+ }
1545
+
1534
1546
  /**
1535
1547
  * Run a shell command with tree-sitter analysis gating.
1536
1548
  *
@@ -1655,6 +1667,9 @@ async function runShell(commandOrOpts, legacyArgs, legacyOpts) {
1655
1667
  if (looksLikeSerialLoop && elapsedMs >= SHELL_PROGRESS_AFTER_MS) {
1656
1668
  result.hint = `This serial loop took ${Math.round(elapsedMs / 1000)}s. For many similar operations, prefer ONE batched command (most CLIs accept multiple paths) or run it with background:true and poll bg_output, instead of a loop of separate run_shell calls.`;
1657
1669
  }
1670
+ if (looksLikeProjectClone(commandStr)) {
1671
+ result.hint = result.hint ? `${result.hint} ${CLONE_ADVISORY}` : CLONE_ADVISORY;
1672
+ }
1658
1673
  return result;
1659
1674
  } catch (err) {
1660
1675
  // User Stop / turn abort: the child was killed via the AbortSignal. Return a
@@ -5433,6 +5448,7 @@ module.exports = {
5433
5448
  MAIL_AUTOMATION_LOCK_PATH, MAIL_AUTOMATION_COOLDOWN_PATH,
5434
5449
  getSystemInfo, webFetch,
5435
5450
  globFiles, grepFiles, listDirectory, isBinaryFile,
5451
+ looksLikeProjectClone,
5436
5452
  SHELL_ALLOWLIST, SHELL_DENYLIST,
5437
5453
  isProtectedPath, resolveProjectPath,
5438
5454
  _collectGmailAttachments, _decodeBase64UrlToBuffer, _sanitizeAttachmentFilename,
@@ -10,6 +10,20 @@ let completed = 0;
10
10
  let failed = 0;
11
11
  const queue = [];
12
12
 
13
+ function _priorityValue(value, fallback) {
14
+ const n = Number(value);
15
+ return Number.isFinite(n) ? Math.max(0, Math.trunc(n)) : fallback;
16
+ }
17
+
18
+ function _queuedByLane() {
19
+ const byLane = {};
20
+ for (const task of queue) {
21
+ const lane = String(task.lane);
22
+ byLane[lane] = (byLane[lane] || 0) + 1;
23
+ }
24
+ return byLane;
25
+ }
26
+
13
27
  function _serializeError(err) {
14
28
  return {
15
29
  message: err && err.message ? err.message : String(err || 'unknown error'),
@@ -24,6 +38,7 @@ function _status() {
24
38
  active,
25
39
  running: !!active,
26
40
  pending: queue.length,
41
+ pendingByLane: _queuedByLane(),
27
42
  enqueued,
28
43
  completed,
29
44
  failed,
@@ -94,7 +109,16 @@ function _enqueue(msg) {
94
109
  requestId: msg.requestId,
95
110
  op: msg.op,
96
111
  payload: msg.payload || {},
112
+ lane: _priorityValue(msg.lane, 2),
113
+ priority: _priorityValue(msg.priority, 5),
114
+ enqueuedAt: Date.now(),
97
115
  });
116
+ queue.sort((a, b) => (
117
+ a.lane - b.lane
118
+ || a.priority - b.priority
119
+ || a.enqueuedAt - b.enqueuedAt
120
+ || a.requestId - b.requestId
121
+ ));
98
122
  _pump();
99
123
  }
100
124