@yemi33/minions 0.1.2345 → 0.1.2347
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard.js +87 -10
- package/docs/README.md +1 -0
- package/docs/command-center.md +3 -1
- package/docs/completion-reports.md +23 -0
- package/docs/kb-sweep.md +57 -24
- package/docs/live-checkout-mode.md +27 -5
- package/engine/dispatch.js +1 -0
- package/engine/kb-sweep.js +293 -10
- package/engine/live-checkout.js +157 -24
- package/engine/metrics-store.js +0 -0
- package/engine/shared.js +28 -2
- package/engine.js +77 -0
- package/package.json +1 -1
package/engine/kb-sweep.js
CHANGED
|
@@ -121,6 +121,11 @@ function _entryAgeMs(entry, now) {
|
|
|
121
121
|
* (absent from the TTL map) are never touched. Pinned entries are already
|
|
122
122
|
* excluded upstream (they never enter the manifest), and knowledge/agents/ is
|
|
123
123
|
* never scanned into the manifest at all — so per-agent memory is inherently safe.
|
|
124
|
+
* Consolidation digests are also exempt — like the rewrite pass (`_isDigestEntry`
|
|
125
|
+
* guard below), a digest's `date` frontmatter is inherited from the old
|
|
126
|
+
* period-bucket of the boilerplate notes it rolled up, not its own write time, so
|
|
127
|
+
* age-based TTL would silently archive a freshly-written digest on the very next
|
|
128
|
+
* sweep if it were not excluded here.
|
|
124
129
|
*
|
|
125
130
|
* @returns {{ expired:number, survivors:object[] }}
|
|
126
131
|
*/
|
|
@@ -132,6 +137,7 @@ function _expireByAge(manifest, opts = {}) {
|
|
|
132
137
|
let expired = 0;
|
|
133
138
|
const survivors = [];
|
|
134
139
|
for (const e of manifest) {
|
|
140
|
+
if (_isDigestEntry(e)) { survivors.push(e); continue; }
|
|
135
141
|
const ttlDays = ttlByCategory[e.category];
|
|
136
142
|
if (!ttlDays) { survivors.push(e); continue; }
|
|
137
143
|
if (_entryAgeMs(e, now) <= ttlDays * DAY_MS) { survivors.push(e); continue; }
|
|
@@ -168,6 +174,252 @@ function _hashDedup(manifest, opts = {}) {
|
|
|
168
174
|
return { survivors, archived };
|
|
169
175
|
}
|
|
170
176
|
|
|
177
|
+
// ── Structural consolidation (W-mr973knu) ───────────────────────────────────
|
|
178
|
+
// The primary fix for the ballooning KB. Hash-dedup only catches near-exact
|
|
179
|
+
// duplicates and the LLM batch pass only compares entries *within* a 30-entry
|
|
180
|
+
// batch, so the dominant cost — hundreds of one-off boilerplate notes (agent
|
|
181
|
+
// failure dumps, no-op / merge-conflict fix reports) that differ only by
|
|
182
|
+
// WI/date/dispatch id — is never collapsed: each is unique by PR#/date and
|
|
183
|
+
// spread across dozens of batches. This pass groups those boilerplate notes by
|
|
184
|
+
// a *structural* signature (category + agent + kind + time bucket) rather than
|
|
185
|
+
// by content, and rolls each large group into a single recoverable digest,
|
|
186
|
+
// archiving the originals to knowledge/_swept/. It runs on RECENT entries too,
|
|
187
|
+
// so it shrinks the KB continuously instead of merely aging files out.
|
|
188
|
+
|
|
189
|
+
const CONSOLIDATION_DIGEST_PREFIX = '_digest-';
|
|
190
|
+
const CONSOLIDATION_DIGEST_FLAG = '_digest'; // frontmatter marker on a digest (never re-consolidated / re-written)
|
|
191
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Resolve the effective structural-consolidation config, merging
|
|
195
|
+
* `opts.engineConfig.kbConsolidation` over `ENGINE_DEFAULTS.kbConsolidation`.
|
|
196
|
+
* `enabled` is false when no categories are configured or an explicit
|
|
197
|
+
* `enabled: false` override is present.
|
|
198
|
+
*/
|
|
199
|
+
function _resolveConsolidationConfig(opts = {}) {
|
|
200
|
+
const defaults = (shared.ENGINE_DEFAULTS && shared.ENGINE_DEFAULTS.kbConsolidation) || {};
|
|
201
|
+
const override = (opts.engineConfig && opts.engineConfig.kbConsolidation) || {};
|
|
202
|
+
const merged = { ...defaults, ...override };
|
|
203
|
+
const categories = Array.isArray(merged.categories)
|
|
204
|
+
? merged.categories.filter(c => typeof c === 'string' && c)
|
|
205
|
+
: [];
|
|
206
|
+
const minGroupSize = Number(merged.minGroupSize);
|
|
207
|
+
const periodDays = Number(merged.periodDays);
|
|
208
|
+
return {
|
|
209
|
+
enabled: merged.enabled !== false && categories.length > 0,
|
|
210
|
+
categories: new Set(categories),
|
|
211
|
+
minGroupSize: Number.isFinite(minGroupSize) && minGroupSize >= 2 ? Math.floor(minGroupSize) : 5,
|
|
212
|
+
periodDays: Number.isFinite(periodDays) && periodDays > 0 ? periodDays : 7,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** First `^key: value$` match anywhere in the entry content (handles the
|
|
217
|
+
* double-frontmatter shape of agent-authored notes). '' when absent. */
|
|
218
|
+
function _extractHeadField(content, key) {
|
|
219
|
+
const re = new RegExp('^' + key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + ':\\s*(.+)$', 'm');
|
|
220
|
+
const m = String(content || '').match(re);
|
|
221
|
+
return m ? m[1].trim() : '';
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** True when an entry is itself a consolidation digest — such entries are never
|
|
225
|
+
* re-grouped and never rewritten. */
|
|
226
|
+
function _isDigestEntry(entry) {
|
|
227
|
+
if (!entry) return false;
|
|
228
|
+
if (typeof entry.file === 'string' && entry.file.startsWith(CONSOLIDATION_DIGEST_PREFIX)) return true;
|
|
229
|
+
if (entry._digest) return true;
|
|
230
|
+
const content = entry.content || '';
|
|
231
|
+
if (_extractHeadField(content, CONSOLIDATION_DIGEST_FLAG)) return true;
|
|
232
|
+
if (_extractHeadField(content, 'source') === 'kb-consolidation') return true;
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Classify an entry into a consolidatable "kind", or null when it is not
|
|
238
|
+
* boilerplate we roll up (unique/durable notes are left alone). Kinds target
|
|
239
|
+
* the two dominant balloon sources: agent-failure dumps (keyed by failureClass)
|
|
240
|
+
* and no-op / merge-conflict fix reports.
|
|
241
|
+
* @returns {{ key:string, label:string }|null}
|
|
242
|
+
*/
|
|
243
|
+
function _consolidationKind(entry) {
|
|
244
|
+
const content = entry.content || '';
|
|
245
|
+
const failureClass = _extractHeadField(content, 'failureClass');
|
|
246
|
+
if (failureClass) return { key: `failure-${failureClass}`, label: `${failureClass} failures` };
|
|
247
|
+
const result = _extractHeadField(content, 'result');
|
|
248
|
+
const title = String(entry.title || '').toLowerCase();
|
|
249
|
+
if (/\berror\b/i.test(result) || /agent fail/.test(title)) {
|
|
250
|
+
return { key: 'failure-unknown', label: 'agent failures' };
|
|
251
|
+
}
|
|
252
|
+
if (/\bno-?op\b/.test(title)) return { key: 'noop', label: 'no-op dispatches' };
|
|
253
|
+
if (/merge.?conflict/.test(title)) return { key: 'merge-conflict', label: 'merge-conflict fixes' };
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** ISO date (YYYY-MM-DD) of the start of the `periodDays` bucket an entry falls
|
|
258
|
+
* in, from its authored date (mtime fallback). 'undated' when neither known. */
|
|
259
|
+
function _periodBucket(entry, periodDays) {
|
|
260
|
+
const parsed = typeof entry.date === 'string' && entry.date ? Date.parse(entry.date) : NaN;
|
|
261
|
+
const ms = Number.isFinite(parsed)
|
|
262
|
+
? parsed
|
|
263
|
+
: (Number.isFinite(entry.mtimeMs) && entry.mtimeMs > 0 ? entry.mtimeMs : NaN);
|
|
264
|
+
if (!Number.isFinite(ms)) return 'undated';
|
|
265
|
+
const dayIdx = Math.floor(ms / DAY_MS);
|
|
266
|
+
const startMs = Math.floor(dayIdx / periodDays) * periodDays * DAY_MS;
|
|
267
|
+
return new Date(startMs).toISOString().slice(0, 10);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function _slugify(s) {
|
|
271
|
+
return String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 48) || 'x';
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function _digestFileName(agent, kindKey, period) {
|
|
275
|
+
return `${CONSOLIDATION_DIGEST_PREFIX}${_slugify(agent)}-${_slugify(kindKey)}-${period}.md`;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/** Extract the compact per-note fields that become one digest table row. */
|
|
279
|
+
function _digestRow(entry) {
|
|
280
|
+
const content = entry.content || '';
|
|
281
|
+
const wi = _extractHeadField(content, 'sourceItem')
|
|
282
|
+
|| (content.match(/\bW-[a-z0-9]{6,}/i) || [''])[0]
|
|
283
|
+
|| '';
|
|
284
|
+
const dispatch = _extractHeadField(content, 'dispatchId');
|
|
285
|
+
const failureClass = _extractHeadField(content, 'failureClass');
|
|
286
|
+
let reason = _extractHeadField(content, 'Reason') || _extractHeadField(content, 'reason');
|
|
287
|
+
if (!reason) {
|
|
288
|
+
const rm = content.match(/(?:^|\n)\s*(?:\*\*)?Reason:(?:\*\*)?\s*(.+)/i);
|
|
289
|
+
if (rm) reason = rm[1];
|
|
290
|
+
}
|
|
291
|
+
if (!reason) {
|
|
292
|
+
// First prose line after the last frontmatter block / heading.
|
|
293
|
+
const body = content.replace(/^---\n[\s\S]*?\n---\n?/g, '').replace(/^#.*$/m, '');
|
|
294
|
+
const line = (body.split('\n').find(l => l.trim() && !l.trim().startsWith('#')) || '').trim();
|
|
295
|
+
reason = line;
|
|
296
|
+
}
|
|
297
|
+
reason = String(reason || '').replace(/\|/g, '\\|').replace(/\s+/g, ' ').trim().slice(0, 160);
|
|
298
|
+
return {
|
|
299
|
+
date: entry.date || '',
|
|
300
|
+
wi,
|
|
301
|
+
dispatch,
|
|
302
|
+
failureClass,
|
|
303
|
+
reason: reason || '—',
|
|
304
|
+
file: entry.file,
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** Parse the `source file` column out of an existing digest so re-runs merge
|
|
309
|
+
* rather than clobber. Returns a Map<sourceFile, rowLine>. */
|
|
310
|
+
function _readExistingDigestRows(digestPath) {
|
|
311
|
+
const rows = new Map();
|
|
312
|
+
const content = safeReadOrNull(digestPath);
|
|
313
|
+
if (!content) return rows;
|
|
314
|
+
for (const line of content.split('\n')) {
|
|
315
|
+
if (!line.startsWith('| ')) continue;
|
|
316
|
+
const cells = line.split('|').map(c => c.trim());
|
|
317
|
+
// columns: '' | date | work item | dispatch | failureClass | reason | source file | ''
|
|
318
|
+
const src = cells[cells.length - 2];
|
|
319
|
+
if (!src || src === 'source file' || /^-+$/.test(src)) continue;
|
|
320
|
+
rows.set(src, line);
|
|
321
|
+
}
|
|
322
|
+
return rows;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function _formatDigestRowLine(r) {
|
|
326
|
+
return `| ${r.date} | ${r.wi || '—'} | ${r.dispatch || '—'} | ${r.failureClass || '—'} | ${r.reason} | ${r.file} |`;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function _buildDigestContent(group, rowLines, count) {
|
|
330
|
+
const fm = [
|
|
331
|
+
`category: ${group.category}`,
|
|
332
|
+
`agent: ${group.agent}`,
|
|
333
|
+
`date: ${group.period}`,
|
|
334
|
+
`${CONSOLIDATION_DIGEST_FLAG}: ${new Date().toISOString()}`,
|
|
335
|
+
'source: kb-consolidation',
|
|
336
|
+
`_consolidatedCount: ${count}`,
|
|
337
|
+
].join('\n');
|
|
338
|
+
const header = `# Consolidated: ${group.agent} — ${group.kind.label} (from ${group.period})`;
|
|
339
|
+
const intro = `Rolled up ${count} near-identical ${group.kind.label} note(s) into this single digest. `
|
|
340
|
+
+ 'Originals were archived to `knowledge/_swept/` (recoverable for 30 days). '
|
|
341
|
+
+ 'Each row is one collapsed note; recover a specific one from `_swept/` by its `source file`.';
|
|
342
|
+
const table = [
|
|
343
|
+
'| date | work item | dispatch | failureClass | reason | source file |',
|
|
344
|
+
'|------|-----------|----------|--------------|--------|-------------|',
|
|
345
|
+
...rowLines,
|
|
346
|
+
].join('\n');
|
|
347
|
+
return `---\n${fm}\n---\n\n${header}\n\n${intro}\n\n${table}\n`;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Structural consolidation pass. Groups boilerplate transient-category notes by
|
|
352
|
+
* (category, agent, kind, time-bucket) and collapses any group of at least
|
|
353
|
+
* `minGroupSize` into a single recoverable digest, archiving the originals.
|
|
354
|
+
* Digests, durable categories, pinned entries (already filtered upstream), and
|
|
355
|
+
* unclassifiable/unique notes are left untouched.
|
|
356
|
+
*
|
|
357
|
+
* @returns {{ survivors:object[], consolidated:number, groupsCollapsed:number, digestsWritten:number }}
|
|
358
|
+
*/
|
|
359
|
+
function _structuralConsolidate(manifest, opts = {}) {
|
|
360
|
+
const cfg = _resolveConsolidationConfig(opts);
|
|
361
|
+
if (!cfg.enabled) {
|
|
362
|
+
return { survivors: manifest.slice(), consolidated: 0, groupsCollapsed: 0, digestsWritten: 0 };
|
|
363
|
+
}
|
|
364
|
+
const groups = new Map(); // groupKey → { agent, category, kind, period, entries[] }
|
|
365
|
+
const survivors = [];
|
|
366
|
+
for (const e of manifest) {
|
|
367
|
+
if (!cfg.categories.has(e.category) || _isDigestEntry(e)) { survivors.push(e); continue; }
|
|
368
|
+
const kind = _consolidationKind(e);
|
|
369
|
+
if (!kind) { survivors.push(e); continue; }
|
|
370
|
+
const agent = e.agent || 'unknown';
|
|
371
|
+
const period = _periodBucket(e, cfg.periodDays);
|
|
372
|
+
const groupKey = `${e.category}::${agent}::${kind.key}::${period}`;
|
|
373
|
+
if (!groups.has(groupKey)) groups.set(groupKey, { agent, category: e.category, kind, period, entries: [] });
|
|
374
|
+
groups.get(groupKey).entries.push(e);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
let consolidated = 0, groupsCollapsed = 0, digestsWritten = 0;
|
|
378
|
+
for (const g of groups.values()) {
|
|
379
|
+
if (g.entries.length < cfg.minGroupSize) { survivors.push(...g.entries); continue; }
|
|
380
|
+
g.entries.sort((a, b) => (b.date || '').localeCompare(a.date || '') || (b.mtimeMs - a.mtimeMs));
|
|
381
|
+
|
|
382
|
+
if (opts.dryRun) {
|
|
383
|
+
consolidated += g.entries.length;
|
|
384
|
+
groupsCollapsed++;
|
|
385
|
+
digestsWritten++;
|
|
386
|
+
// Nothing archived in dryRun — the originals remain on disk.
|
|
387
|
+
survivors.push(...g.entries);
|
|
388
|
+
continue;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
const digestFile = _digestFileName(g.agent, g.kind.key, g.period);
|
|
392
|
+
const digestPath = path.join(KB_DIR, g.category, digestFile);
|
|
393
|
+
// Merge with any prior digest for this exact group so repeated sweeps
|
|
394
|
+
// accumulate rows instead of clobbering earlier ones.
|
|
395
|
+
const merged = _readExistingDigestRows(digestPath);
|
|
396
|
+
for (const e of g.entries) {
|
|
397
|
+
const r = _digestRow(e);
|
|
398
|
+
merged.set(r.file, _formatDigestRowLine(r));
|
|
399
|
+
}
|
|
400
|
+
const rowLines = [...merged.values()];
|
|
401
|
+
const digestContent = _buildDigestContent(g, rowLines, rowLines.length);
|
|
402
|
+
try {
|
|
403
|
+
const dir = path.dirname(digestPath);
|
|
404
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
405
|
+
safeWrite(digestPath, digestContent);
|
|
406
|
+
digestsWritten++;
|
|
407
|
+
} catch (err) {
|
|
408
|
+
log('warn', `[kb-sweep] consolidate write ${g.category}/${digestFile}: ${err.message}`);
|
|
409
|
+
survivors.push(...g.entries); // write failed — keep originals visible
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
let collapsed = 0;
|
|
413
|
+
for (const e of g.entries) {
|
|
414
|
+
const fp = path.join(KB_DIR, e.category, e.file);
|
|
415
|
+
if (!fs.existsSync(fp)) continue;
|
|
416
|
+
if (_archiveKbFile(fp, `structurally consolidated into ${g.category}/${digestFile}`)) { consolidated++; collapsed++; }
|
|
417
|
+
}
|
|
418
|
+
if (collapsed > 0) groupsCollapsed++;
|
|
419
|
+
}
|
|
420
|
+
return { survivors, consolidated, groupsCollapsed, digestsWritten };
|
|
421
|
+
}
|
|
422
|
+
|
|
171
423
|
/** Batched LLM sweep — finds within-batch dupes, reclassifies, removes stale. */
|
|
172
424
|
async function _llmBatchSweep(manifest, callLLM, trackEngineUsage, opts = {}) {
|
|
173
425
|
const plan = { duplicates: [], reclassify: [], remove: [] };
|
|
@@ -252,6 +504,9 @@ ${body}`;
|
|
|
252
504
|
|
|
253
505
|
const candidates = [];
|
|
254
506
|
for (const e of survivors) {
|
|
507
|
+
// Never rewrite a consolidation digest — the rewrite template would destroy
|
|
508
|
+
// its roll-up table.
|
|
509
|
+
if (_isDigestEntry(e)) continue;
|
|
255
510
|
const fp = path.join(KB_DIR, e.category, e.file);
|
|
256
511
|
const content = safeRead(fp);
|
|
257
512
|
if (content == null) continue;
|
|
@@ -540,6 +795,9 @@ async function _runKbSweepImpl(opts = {}) {
|
|
|
540
795
|
llmDuplicatesArchived: 0,
|
|
541
796
|
staleRemoved: 0,
|
|
542
797
|
ageExpired: 0,
|
|
798
|
+
notesConsolidated: 0,
|
|
799
|
+
consolidationGroups: 0,
|
|
800
|
+
digestsWritten: 0,
|
|
543
801
|
reclassified: 0,
|
|
544
802
|
rewritten: 0,
|
|
545
803
|
rewriteBytesBefore: 0,
|
|
@@ -579,27 +837,41 @@ async function _runKbSweepImpl(opts = {}) {
|
|
|
579
837
|
const { survivors: afterHash, archived: hashArchived } = _hashDedup(manifest, opts);
|
|
580
838
|
summary.hashDuplicatesArchived = hashArchived;
|
|
581
839
|
|
|
840
|
+
// 1.5. Structural consolidation (W-mr973knu) — PRIMARY balloon fix. Collapse
|
|
841
|
+
// large groups of near-identical boilerplate notes (agent-failure dumps, no-op
|
|
842
|
+
// / merge-conflict reports) into one recoverable digest per (agent, kind,
|
|
843
|
+
// week). Runs on recent entries too, so it shrinks the KB continuously rather
|
|
844
|
+
// than merely aging files out. Downstream passes operate on the survivors so
|
|
845
|
+
// the LLM/rewrite passes no longer waste calls on the collapsed notes.
|
|
846
|
+
const consolidateResult = _structuralConsolidate(afterHash, opts);
|
|
847
|
+
summary.notesConsolidated = consolidateResult.consolidated;
|
|
848
|
+
summary.consolidationGroups = consolidateResult.groupsCollapsed;
|
|
849
|
+
summary.digestsWritten = consolidateResult.digestsWritten;
|
|
850
|
+
const afterConsolidate = consolidateResult.survivors.filter(
|
|
851
|
+
e => e._digest || fs.existsSync(path.join(KB_DIR, e.category, e.file)),
|
|
852
|
+
);
|
|
853
|
+
|
|
582
854
|
// 2. LLM batch sweep — within-batch dupes + reclassify + remove stale
|
|
583
855
|
// Only runs against survivors, but we need indices that match the LIST sent to the LLM
|
|
584
|
-
const llmManifest =
|
|
856
|
+
const llmManifest = afterConsolidate;
|
|
585
857
|
const plan = await _llmBatchSweep(llmManifest, callLLM, trackEngineUsage, opts);
|
|
586
858
|
const llmActions = _applyLlmPlan(plan, llmManifest, opts);
|
|
587
859
|
summary.llmDuplicatesArchived = llmActions.merged;
|
|
588
860
|
summary.staleRemoved = llmActions.removed;
|
|
589
861
|
summary.reclassified = llmActions.reclassified;
|
|
590
862
|
|
|
591
|
-
// 2.5. Age-based TTL expiry (W-mr48yth5) —
|
|
592
|
-
// older than their per-category TTL to
|
|
593
|
-
//
|
|
594
|
-
//
|
|
595
|
-
//
|
|
596
|
-
const survivingAfterLlm =
|
|
863
|
+
// 2.5. Age-based TTL expiry (W-mr48yth5) — SECONDARY safety net. Archives
|
|
864
|
+
// transient-category entries older than their per-category TTL to
|
|
865
|
+
// knowledge/_swept/. Structural consolidation above is the primary reducer;
|
|
866
|
+
// this catches genuinely stale one-offs that never clustered into a digest.
|
|
867
|
+
// Durable categories (no TTL) are untouched.
|
|
868
|
+
const survivingAfterLlm = afterConsolidate.filter(e => fs.existsSync(path.join(KB_DIR, e.category, e.file)));
|
|
597
869
|
const ageResult = _expireByAge(survivingAfterLlm, opts);
|
|
598
870
|
summary.ageExpired = ageResult.expired;
|
|
599
871
|
|
|
600
872
|
// 3. Per-entry rewrite (compress + normalize)
|
|
601
|
-
// Filter to entries that survived hash + LLM + age passes (still on disk)
|
|
602
|
-
const stillOnDisk =
|
|
873
|
+
// Filter to entries that survived hash + consolidation + LLM + age passes (still on disk)
|
|
874
|
+
const stillOnDisk = afterConsolidate.filter(e => fs.existsSync(path.join(KB_DIR, e.category, e.file)));
|
|
603
875
|
const rewriteResult = await _rewritePass(stillOnDisk, callLLM, trackEngineUsage, opts);
|
|
604
876
|
summary.rewritten = rewriteResult.processed;
|
|
605
877
|
summary.rewriteBytesBefore = rewriteResult.bytesBefore;
|
|
@@ -620,7 +892,7 @@ async function _runKbSweepImpl(opts = {}) {
|
|
|
620
892
|
}
|
|
621
893
|
|
|
622
894
|
summary.durationMs = Date.now() - t0;
|
|
623
|
-
summary.summary = `${summary.hashDuplicatesArchived} hash-dup, ${summary.llmDuplicatesArchived} llm-dup, ${summary.staleRemoved} stale, ${summary.ageExpired} age-expired, ${summary.reclassified} reclassified, ${summary.rewritten} rewritten (${(summary.bytesBefore - summary.bytesAfter).toLocaleString()} bytes saved)`;
|
|
895
|
+
summary.summary = `${summary.hashDuplicatesArchived} hash-dup, ${summary.notesConsolidated} consolidated (${summary.digestsWritten} digests), ${summary.llmDuplicatesArchived} llm-dup, ${summary.staleRemoved} stale, ${summary.ageExpired} age-expired, ${summary.reclassified} reclassified, ${summary.rewritten} rewritten (${(summary.bytesBefore - summary.bytesAfter).toLocaleString()} bytes saved)`;
|
|
624
896
|
|
|
625
897
|
if (!opts.dryRun) {
|
|
626
898
|
try { shared.mutateKbSwept(() => ({ timestamp: ts(), summary: summary.summary, detail: summary })); } catch { /* ignore */ }
|
|
@@ -781,6 +1053,17 @@ module.exports = {
|
|
|
781
1053
|
_entryAgeMs,
|
|
782
1054
|
_resolveTtlByCategory,
|
|
783
1055
|
_archiveKbFile,
|
|
1056
|
+
_expireByAge,
|
|
1057
|
+
_entryAgeMs,
|
|
1058
|
+
_resolveTtlByCategory,
|
|
1059
|
+
_structuralConsolidate,
|
|
1060
|
+
_resolveConsolidationConfig,
|
|
1061
|
+
_consolidationKind,
|
|
1062
|
+
_periodBucket,
|
|
1063
|
+
_digestRow,
|
|
1064
|
+
_isDigestEntry,
|
|
1065
|
+
_extractHeadField,
|
|
1066
|
+
CONSOLIDATION_DIGEST_PREFIX,
|
|
784
1067
|
COMPRESS_THRESHOLD_BYTES,
|
|
785
1068
|
SWEPT_FLAG_KEY,
|
|
786
1069
|
};
|
package/engine/live-checkout.js
CHANGED
|
@@ -39,25 +39,30 @@
|
|
|
39
39
|
* NO fast-forward, NO reset). The operator owns the checkout
|
|
40
40
|
* state per the live-checkout contract; the engine never
|
|
41
41
|
* overwrites local commits.
|
|
42
|
-
* b. rev-parse fails → branch is new → fork it.
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
42
|
+
* b. rev-parse fails → branch is new → fork it. Two ordered guards run
|
|
43
|
+
* before the fork:
|
|
44
|
+
* i. WRONG-BASE GUARD (W-mr3lunnq000o9f41) — UNLESS `allowNonMainBase`
|
|
45
|
+
* is set (PR-targeted fixes / shared-branch continuations opt out):
|
|
46
|
+
* if HEAD's branch is NOT the project base (`mainRef`), forking off
|
|
47
|
+
* it would silently bake every commit already on that branch into
|
|
48
|
+
* the new branch and its PR (the ADO PR 5411214 scope-contamination
|
|
49
|
+
* incident). Recovery is LOCAL-ONLY (issue #226's no-forced-fetch
|
|
50
|
+
* guarantee): if `refs/heads/<mainRef>` exists locally, PLAIN-checkout
|
|
51
|
+
* it (no fetch/reset/--force) and fork off it; else (optionally after
|
|
52
|
+
* opt-in auto-base-repair materializes a local `<mainRef>` from the
|
|
53
|
+
* existing `refs/remotes/origin/<mainRef>` ref, W-mr3lunnq000o9f41)
|
|
54
|
+
* FAIL CLOSED with { ok:false, reason:'wrong-base' }.
|
|
55
|
+
* ii. STALE-BASE GUARD (W-mr98op8w000ma4ad) — runs once HEAD is confirmed
|
|
56
|
+
* on `mainRef` (either it started there, or guard i left it there):
|
|
57
|
+
* if the local `mainRef` tip has diverged from `origin/<mainRef>`
|
|
58
|
+
* with unpushed commits, bail with { ok:false, reason:'stale-main', ... }
|
|
59
|
+
* rather than fork a contaminated base (see RETURN SHAPES).
|
|
60
|
+
* When both guards pass, `git checkout -b <branchName>` is created from
|
|
61
|
+
* the CURRENT HEAD (NOT origin/<mainRef>). In live mode the operator's
|
|
62
|
+
* checked-out HEAD IS the baseline; branching off origin/<mainRef> forces
|
|
63
|
+
* on-demand blob fetching on blobless partial clones (Scalar/GVFS-managed
|
|
64
|
+
* ADO repos), which fails in headless mode because the GVFS cache server
|
|
65
|
+
* receives no auth and credential.helper is disabled. See issue #226.
|
|
61
66
|
* 5. Returns { ok:true, branch, created:boolean, originalRef, originalRefType }.
|
|
62
67
|
*
|
|
63
68
|
* RETURN SHAPES:
|
|
@@ -80,6 +85,19 @@
|
|
|
80
85
|
* (W-mr28h2j2000y0de1) — the target branch is already checked out in another
|
|
81
86
|
* worktree; `git checkout <branch>` refuses deterministically. Non-retryable
|
|
82
87
|
* at the caller — a human/engine must remove or reassign the other worktree.
|
|
88
|
+
* { ok:false, reason:'stale-main', mainRef, ahead, localMainSha, originMainSha, branch, originalRef, originalRefType }
|
|
89
|
+
* (W-mr98op8w000ma4ad — STALE-BASE GUARD) — a NEW branch was about to be
|
|
90
|
+
* forked off the current HEAD while HEAD sits on the project base ref
|
|
91
|
+
* (mainRef), but the LOCAL mainRef tip has diverged from origin/<mainRef>
|
|
92
|
+
* with `ahead` unpushed commit(s). Forking would silently inherit that
|
|
93
|
+
* committed contamination into the new branch and its PR. Detected without a
|
|
94
|
+
* fetch (issue #226) by comparing local `refs/heads/<mainRef>` against the
|
|
95
|
+
* EXISTING `refs/remotes/origin/<mainRef>` ref via
|
|
96
|
+
* `git rev-list --count origin/<mainRef>..<mainRef>`. The guard is SKIPPED
|
|
97
|
+
* (proceeds normally) when HEAD is not on mainRef, when the origin/<mainRef>
|
|
98
|
+
* remote-tracking ref is absent (local-only/no-remote repo), or when the
|
|
99
|
+
* rev-list count cannot be computed (transient). Non-retryable at the caller
|
|
100
|
+
* — a human must reconcile the local base branch; the engine never resets it.
|
|
83
101
|
*
|
|
84
102
|
* NOTE: `mainRef` is still accepted (and validated) for caller-contract
|
|
85
103
|
* stability, but it is NOT used to seed the new branch in live mode — HEAD
|
|
@@ -565,13 +583,32 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
565
583
|
return { ok: false, reason: 'dirty', dirtyFiles, branchInfo };
|
|
566
584
|
}
|
|
567
585
|
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
586
|
+
// W-mrav3u0c0008ce38 — live-checkout WI types that never push a branch
|
|
587
|
+
// (setup/test/verify live-validation dispatches on checkoutMode:'live'
|
|
588
|
+
// or hybrid liveValidation projects) leave `origin/<branchName>`
|
|
589
|
+
// unresolvable, so a bare `reset --hard origin/<branchName>` fails every
|
|
590
|
+
// time with "ambiguous argument … unknown revision" and the tree stays
|
|
591
|
+
// dirty forever. Probe `origin/<branchName>` first (after the fetch, so
|
|
592
|
+
// it reflects the latest remote state); only fall back to
|
|
593
|
+
// `origin/<mainRef>` when that branch genuinely doesn't exist upstream.
|
|
594
|
+
// PR-continuation flows (fix/implement WIs resuming an existing pushed
|
|
595
|
+
// branch) keep resetting to `origin/<branchName>` exactly as before.
|
|
571
596
|
let resetOk = true;
|
|
597
|
+
let resetTarget = `origin/${branchName}`;
|
|
572
598
|
try {
|
|
573
599
|
await git(['fetch', 'origin'], baseOpts);
|
|
574
|
-
|
|
600
|
+
try {
|
|
601
|
+
await git(['rev-parse', '--verify', resetTarget], baseOpts);
|
|
602
|
+
} catch {
|
|
603
|
+
resetTarget = `origin/${mainRef}`;
|
|
604
|
+
if (typeof log === 'function') {
|
|
605
|
+
log(`live-checkout auto-reset: origin/${branchName} does not exist (never pushed) — falling back to '${resetTarget}'`);
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
if (typeof log === 'function') {
|
|
609
|
+
log(`live-checkout auto-reset: discarding ${dirtyFiles.length} dirty path(s) on '${branchName}' via fetch + reset --hard ${resetTarget}`);
|
|
610
|
+
}
|
|
611
|
+
await git(['reset', '--hard', resetTarget], baseOpts);
|
|
575
612
|
} catch (e) {
|
|
576
613
|
resetOk = false;
|
|
577
614
|
if (typeof log === 'function') {
|
|
@@ -613,7 +650,7 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
613
650
|
'',
|
|
614
651
|
`⚠️ The live-checkout tree at \`${localPath}\` was dirty at dispatch time and`,
|
|
615
652
|
'`liveCheckoutAutoReset` is enabled, so it was force-reset to',
|
|
616
|
-
|
|
653
|
+
`\`${resetTarget}\`. **The following uncommitted changes were DISCARDED**`,
|
|
617
654
|
'(recover from `git reflog` / `git fsck --lost-found` if needed):',
|
|
618
655
|
'',
|
|
619
656
|
'```',
|
|
@@ -949,6 +986,102 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
949
986
|
}
|
|
950
987
|
}
|
|
951
988
|
|
|
989
|
+
|
|
990
|
+
// ── Step 4d: STALE-BASE GUARD (W-mr98op8w000ma4ad). ───────────────────
|
|
991
|
+
// We are about to fork a NEW branch off the CURRENT HEAD. In the common
|
|
992
|
+
// live-mode case HEAD sits on the project base ref (mainRef). Step 1 (dirty
|
|
993
|
+
// check) and Step 2 (mid-operation/detached preflight) both passed, but a
|
|
994
|
+
// CLEAN tree can still hide COMMITTED contamination: a prior/parallel
|
|
995
|
+
// dispatch (or a human) may have left extra commits on the local mainRef
|
|
996
|
+
// branch that were never pushed to origin/<mainRef>. `git checkout -b` would
|
|
997
|
+
// then silently inherit those commits into the new branch (and its PR) — the
|
|
998
|
+
// exact scope-contamination class this guard prevents (original incident ADO
|
|
999
|
+
// PR 5411214; recurrence on AB#12016662 / ADO PR 5419146, ~20 unrelated OCM
|
|
1000
|
+
// files baked into a 2-file a11y fix). This survives the dirty check (the
|
|
1001
|
+
// contamination is COMMITTED, not uncommitted) and — unlike a HEAD parked on
|
|
1002
|
+
// some OTHER branch — cannot be spotted by a headBranch !== mainRef check,
|
|
1003
|
+
// because here headBranch IS mainRef.
|
|
1004
|
+
//
|
|
1005
|
+
// Detect it by comparing the local mainRef tip against the EXISTING
|
|
1006
|
+
// origin/<mainRef> remote-tracking ref (NO fetch — issue #226's
|
|
1007
|
+
// no-forced-fetch guarantee): any commit reachable from local mainRef but not
|
|
1008
|
+
// from origin/<mainRef> (`git rev-list --count origin/<mainRef>..<mainRef>` >
|
|
1009
|
+
// 0) means the base is contaminated with unpushed commits. Local mainRef
|
|
1010
|
+
// being merely BEHIND origin (fewer commits) is fine — that just yields a
|
|
1011
|
+
// branch based on an older tip, not contamination — so we count only the
|
|
1012
|
+
// local-ahead side, not strict equality.
|
|
1013
|
+
//
|
|
1014
|
+
// Scope: only when HEAD is actually on mainRef (that IS the base we fork
|
|
1015
|
+
// off). If HEAD is on some other branch the operator/engine deliberately
|
|
1016
|
+
// parked it there (e.g. the orphan-branch self-heal above) — out of scope.
|
|
1017
|
+
//
|
|
1018
|
+
// AMBIGUITY / local-only repos: if the origin/<mainRef> remote-tracking ref
|
|
1019
|
+
// does not exist locally we CANNOT compare without fetching, which #226
|
|
1020
|
+
// forbids. Rather than fail closed on legitimate local-only / no-remote repos
|
|
1021
|
+
// (which have no origin ref by design and for which forking off HEAD is the
|
|
1022
|
+
// only and expected behavior), we SKIP the check and log. Failing closed
|
|
1023
|
+
// there would regress those setups; the contamination class this targets only
|
|
1024
|
+
// exists when an origin baseline is present to diverge from. A transient
|
|
1025
|
+
// rev-list failure (origin ref present but the count could not be computed)
|
|
1026
|
+
// likewise proceeds without the check rather than converting a transient git
|
|
1027
|
+
// hiccup into a permanent non-retryable refusal.
|
|
1028
|
+
if (curBranch && curBranch === mainRef) {
|
|
1029
|
+
let originMainSha = '';
|
|
1030
|
+
try {
|
|
1031
|
+
const originRaw = await git(['rev-parse', '--verify', '--quiet', `refs/remotes/origin/${mainRef}`], baseOpts);
|
|
1032
|
+
originMainSha = (typeof originRaw === 'string' ? originRaw : '').trim();
|
|
1033
|
+
} catch {
|
|
1034
|
+
originMainSha = ''; // remote-tracking ref absent → ambiguous, skip below
|
|
1035
|
+
}
|
|
1036
|
+
if (originMainSha) {
|
|
1037
|
+
let aheadCount = -1;
|
|
1038
|
+
try {
|
|
1039
|
+
const cntRaw = await git(['rev-list', '--count', `refs/remotes/origin/${mainRef}..${mainRef}`], baseOpts);
|
|
1040
|
+
aheadCount = parseInt((typeof cntRaw === 'string' ? cntRaw : '').trim(), 10);
|
|
1041
|
+
if (Number.isNaN(aheadCount)) aheadCount = -1;
|
|
1042
|
+
} catch {
|
|
1043
|
+
aheadCount = -1; // rev-list failed → transient/ambiguous; proceed without the check
|
|
1044
|
+
}
|
|
1045
|
+
if (aheadCount > 0) {
|
|
1046
|
+
let localMainSha = '';
|
|
1047
|
+
try {
|
|
1048
|
+
const localRaw = await git(['rev-parse', '--verify', '--quiet', `refs/heads/${mainRef}`], baseOpts);
|
|
1049
|
+
localMainSha = (typeof localRaw === 'string' ? localRaw : '').trim();
|
|
1050
|
+
} catch { /* best-effort — informational only */ }
|
|
1051
|
+
logFn(
|
|
1052
|
+
`[live-checkout] STALE-BASE GUARD: refusing to fork '${branchName}' off local '${mainRef}' — ` +
|
|
1053
|
+
`local '${mainRef}' is ${aheadCount} commit(s) ahead of 'origin/${mainRef}' (unpushed/contaminated base). ` +
|
|
1054
|
+
`A human must reconcile the local base branch before dispatch; the engine never resets it for you.`,
|
|
1055
|
+
'warn'
|
|
1056
|
+
);
|
|
1057
|
+
return {
|
|
1058
|
+
ok: false,
|
|
1059
|
+
reason: 'stale-main',
|
|
1060
|
+
mainRef,
|
|
1061
|
+
ahead: aheadCount,
|
|
1062
|
+
localMainSha,
|
|
1063
|
+
originMainSha,
|
|
1064
|
+
branch: branchName,
|
|
1065
|
+
originalRef,
|
|
1066
|
+
originalRefType,
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
if (aheadCount < 0) {
|
|
1070
|
+
logFn(
|
|
1071
|
+
`[live-checkout] STALE-BASE GUARD: could not compute divergence of local '${mainRef}' vs 'origin/${mainRef}' ` +
|
|
1072
|
+
`(rev-list failed); proceeding without the base-divergence check.`,
|
|
1073
|
+
'debug'
|
|
1074
|
+
);
|
|
1075
|
+
}
|
|
1076
|
+
} else {
|
|
1077
|
+
logFn(
|
|
1078
|
+
`[live-checkout] STALE-BASE GUARD: no local 'origin/${mainRef}' remote-tracking ref — ` +
|
|
1079
|
+
`skipping base-divergence check (local-only/no-remote repo; #226 no-fetch).`,
|
|
1080
|
+
'debug'
|
|
1081
|
+
);
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
|
|
952
1085
|
// Create from HEAD — now guaranteed to be the project base branch unless
|
|
953
1086
|
// allowNonMainBase bypassed the guard. NOT origin/<mainRef>: in live mode the
|
|
954
1087
|
// operator's HEAD is the baseline, and seeding from a remote ref forces
|
package/engine/metrics-store.js
CHANGED
|
Binary file
|