@dougharness/cli 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ablate.js +2 -2
- package/dist/ablate.js.map +1 -1
- package/dist/generate/agents.js +4 -0
- package/dist/generate/agents.js.map +1 -1
- package/dist/generate/claude-md.js +1 -1
- package/dist/generate/claude-md.js.map +1 -1
- package/node_modules/@dougharness/flow/.claude-plugin/plugin.json +1 -1
- package/node_modules/@dougharness/flow/agents/coder.md +2 -0
- package/node_modules/@dougharness/flow/agents/implementer.md +2 -0
- package/node_modules/@dougharness/flow/agents/planner.md +2 -1
- package/node_modules/@dougharness/flow/agents/reviewer.md +2 -0
- package/node_modules/@dougharness/flow/agents/tester.md +2 -0
- package/node_modules/@dougharness/flow/lib/board.mjs +9 -9
- package/node_modules/@dougharness/flow/lib/card-loop.mjs +54 -5
- package/node_modules/@dougharness/flow/lib/code-index.mjs +2 -2
- package/node_modules/@dougharness/flow/lib/coordinator.mjs +4 -2
- package/node_modules/@dougharness/flow/lib/decisions.mjs +3 -5
- package/node_modules/@dougharness/flow/lib/learn.mjs +105 -57
- package/node_modules/@dougharness/flow/lib/memory.mjs +1418 -57
- package/node_modules/@dougharness/flow/lib/plan.mjs +40 -0
- package/node_modules/@dougharness/flow/package.json +1 -1
- package/node_modules/@dougharness/flow/scripts/autopilot.mjs +178 -1
- package/node_modules/@dougharness/flow/scripts/board.mjs +43 -15
- package/node_modules/@dougharness/flow/scripts/learn.mjs +4 -2
- package/node_modules/@dougharness/flow/scripts/memory.mjs +870 -39
- package/node_modules/@dougharness/flow/scripts/plan.mjs +3 -3
- package/node_modules/@dougharness/flow/skills/doug-hand/SKILL.md +8 -2
- package/node_modules/@dougharness/flow/skills/doug-learn/SKILL.md +1 -1
- package/node_modules/@dougharness/flow/skills/doug-next/SKILL.md +4 -0
- package/node_modules/@dougharness/flow/skills/doug-start/SKILL.md +9 -8
- package/node_modules/@dougharness/flow/skills/doug-swarm/SKILL.md +1 -1
- package/node_modules/@dougharness/flow/skills/run-report/SKILL.md +6 -3
- package/node_modules/@dougharness/flow/workflows/card-queen.js +54 -5
- package/node_modules/@dougharness/flow/workflows/doug-implement.js +34 -19
- package/node_modules/@dougharness/gates/.claude-plugin/plugin.json +1 -1
- package/node_modules/@dougharness/gates/lib/baseline.mjs +9 -0
- package/node_modules/@dougharness/gates/lib/block-ledger.mjs +6 -3
- package/node_modules/@dougharness/gates/lib/gate-lock.mjs +175 -4
- package/node_modules/@dougharness/gates/lib/test-totals.mjs +1 -1
- package/node_modules/@dougharness/gates/lib/verified-tree.mjs +33 -0
- package/node_modules/@dougharness/gates/package.json +1 -1
- package/node_modules/@dougharness/gates/scripts/stop-gate.mjs +110 -125
- package/node_modules/@dougharness/gates/scripts/verify.mjs +47 -13
- package/package.json +3 -3
|
@@ -11,14 +11,15 @@
|
|
|
11
11
|
// can supersede a lesson (superseded_by) but never delete or rewrite its text — two triggers on the table enforce
|
|
12
12
|
// that in the database itself, not just in this module's API.
|
|
13
13
|
import { createRequire } from "node:module";
|
|
14
|
-
import { fileURLToPath } from "node:url";
|
|
15
|
-
import { mkdirSync, existsSync, readFileSync, statSync, readdirSync, writeFileSync } from "node:fs";
|
|
14
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
15
|
+
import { mkdirSync, existsSync, readFileSync, statSync, readdirSync, writeFileSync, appendFileSync } from "node:fs";
|
|
16
16
|
import { dirname, resolve, join } from "node:path";
|
|
17
17
|
import { createHash } from "node:crypto";
|
|
18
18
|
import { homedir } from "node:os";
|
|
19
19
|
import { request as httpRequest } from "node:http";
|
|
20
20
|
import { request as httpsRequest } from "node:https";
|
|
21
|
-
import { classifyBlocks, adversaryBlocks } from "./board.mjs";
|
|
21
|
+
import { classifyBlocks, adversaryBlocks, addCard } from "./board.mjs";
|
|
22
|
+
import { initialState, currentAction, next as cardLoopNext, briefPrompt } from "./card-loop.mjs";
|
|
22
23
|
import { transcriptUsage, projectSlug, CODEX_PRICES, CODEX_DEFAULT_MODEL } from "./cost.mjs";
|
|
23
24
|
import { MODELS_HEADING } from "./models.mjs";
|
|
24
25
|
|
|
@@ -61,7 +62,15 @@ const LESSON_KINDS = ["feedback", "project", "pitfall", "pattern"];
|
|
|
61
62
|
// verified is null on a hand row even though the gate passed: no independent verifier ran on that track, and
|
|
62
63
|
// the gate's own text lives in the `gate` column instead. A consumer that averages fix_passes or counts
|
|
63
64
|
// blocks across both tracks must filter on track. wall_clock, gate, and note only exist on a hand row; they
|
|
64
|
-
// are null on a flow row.
|
|
65
|
+
// are null on a flow row. Exception (card memory-outcome-arms): `reflect --hand --rounds <n>` (not a dry run)
|
|
66
|
+
// writes passes n and fix_passes n - 1 onto the landing's hand row; without --rounds both stay null, since the
|
|
67
|
+
// default of 1 round is not a measurement. A later `record --hand` of the same commit keeps those values (the
|
|
68
|
+
// upsert only replaces passes and fix_passes when the new row carries passes).
|
|
69
|
+
//
|
|
70
|
+
// memory_config (v10, card memory-outcome-arms): the canonical JSON (keys sorted) of the memoryConfig() the row
|
|
71
|
+
// ran under, copied from condition.json; null on an unstamped row. It travels with the condition columns. A flow
|
|
72
|
+
// row is stamped with memoryConfig({ track: "flow" }) (delivery "index+fetch" with fetchTokens, cards memory-flow-index-fetch, memory-atomic-lessons) when recorded
|
|
73
|
+
// through `memory.mjs record <report.json>`; a hand row carries the hand stamp (delivery "index+fetch").
|
|
65
74
|
//
|
|
66
75
|
// partial (v5, card worker-context-handoff): 1 when the task's own outcome was still partial after its one
|
|
67
76
|
// resume (graceful degradation on a context limit, never a verification failure), 0 when it was not, null on a
|
|
@@ -77,8 +86,10 @@ const LESSON_KINDS = ["feedback", "project", "pitfall", "pattern"];
|
|
|
77
86
|
// ran under (`<class>/default` until exploration exists); assigned_by is `'policy'` or `'explore'`;
|
|
78
87
|
// explore_probability is the explore arm's assignment probability, null otherwise. backfilled is 1 only for a
|
|
79
88
|
// row `memory.mjs condition backfill` stamped after the fact, 0 for every other row (including an unstamped
|
|
80
|
-
// hand row, which has never been touched by either path).
|
|
81
|
-
//
|
|
89
|
+
// hand row, which has never been touched by either path).
|
|
90
|
+
// A stopped hand card report yields a stop row (stopOutcomeRow below): track 'hand', run 'stop:<report hash>', stop_class and
|
|
91
|
+
// stop_reason from the report, commit_sha null. On the flow track these seven columns are null: a flow
|
|
92
|
+
// row is never stamped by `condition open`/`record --hand` (its memory_config alone is stamped, see above).
|
|
82
93
|
const OUTCOMES_COLUMNS_SQL = `
|
|
83
94
|
id INTEGER PRIMARY KEY,
|
|
84
95
|
recorded TEXT NOT NULL,
|
|
@@ -124,6 +135,7 @@ const OUTCOMES_COLUMNS_SQL = `
|
|
|
124
135
|
assigned_by TEXT,
|
|
125
136
|
explore_probability REAL,
|
|
126
137
|
backfilled INTEGER NOT NULL DEFAULT 0,
|
|
138
|
+
memory_config TEXT,
|
|
127
139
|
UNIQUE(run, task)
|
|
128
140
|
`;
|
|
129
141
|
|
|
@@ -150,7 +162,10 @@ const LESSONS_COLUMNS_SQL = `
|
|
|
150
162
|
embedding_model TEXT,
|
|
151
163
|
embedding_dims INTEGER,
|
|
152
164
|
embedding BLOB,
|
|
153
|
-
stale TEXT
|
|
165
|
+
stale TEXT,
|
|
166
|
+
seen INTEGER NOT NULL DEFAULT 0,
|
|
167
|
+
archived TEXT,
|
|
168
|
+
archived_reason TEXT
|
|
154
169
|
`;
|
|
155
170
|
|
|
156
171
|
const LESSONS_TRIGGERS_SQL = `
|
|
@@ -260,7 +275,7 @@ const V1_OUTCOME_COLUMNS = [
|
|
|
260
275
|
];
|
|
261
276
|
|
|
262
277
|
function migrateV1ToV2(db) {
|
|
263
|
-
db.exec("BEGIN");
|
|
278
|
+
db.exec("BEGIN IMMEDIATE");
|
|
264
279
|
try {
|
|
265
280
|
db.exec(`CREATE TABLE outcomes_v2 (${OUTCOMES_COLUMNS_SQL});`);
|
|
266
281
|
db.exec(
|
|
@@ -285,7 +300,7 @@ const V2_LESSON_COLUMNS = [
|
|
|
285
300
|
];
|
|
286
301
|
|
|
287
302
|
function migrateV2ToV3(db) {
|
|
288
|
-
db.exec("BEGIN");
|
|
303
|
+
db.exec("BEGIN IMMEDIATE");
|
|
289
304
|
try {
|
|
290
305
|
db.exec(`CREATE TABLE lessons_v3 (${LESSONS_COLUMNS_SQL});`);
|
|
291
306
|
db.exec(
|
|
@@ -305,7 +320,7 @@ function migrateV2ToV3(db) {
|
|
|
305
320
|
// v3 had no reflections table. Unlike v1->v2 and v2->v3, nothing existing is rebuilt: adding a table needs no
|
|
306
321
|
// rebuild-under-a-temp-name dance, just the CREATE plus the version bump, in the same transactional pattern.
|
|
307
322
|
function migrateV3ToV4(db) {
|
|
308
|
-
db.exec("BEGIN");
|
|
323
|
+
db.exec("BEGIN IMMEDIATE");
|
|
309
324
|
try {
|
|
310
325
|
db.exec(`CREATE TABLE reflections (${REFLECTIONS_COLUMNS_SQL});`);
|
|
311
326
|
db.exec("PRAGMA user_version = 4");
|
|
@@ -323,7 +338,7 @@ function migrateV3ToV4(db) {
|
|
|
323
338
|
// this guard makes the step a no-op there instead of a duplicate-column error, while still doing the ALTER for a
|
|
324
339
|
// genuine v4 store that only ever had the v4 columns.
|
|
325
340
|
function migrateV4ToV5(db) {
|
|
326
|
-
db.exec("BEGIN");
|
|
341
|
+
db.exec("BEGIN IMMEDIATE");
|
|
327
342
|
try {
|
|
328
343
|
const cols = db.prepare("PRAGMA table_info(outcomes)").all().map((c) => c.name);
|
|
329
344
|
if (!cols.includes("partial")) db.exec("ALTER TABLE outcomes ADD COLUMN partial INTEGER;");
|
|
@@ -339,7 +354,7 @@ function migrateV4ToV5(db) {
|
|
|
339
354
|
// the four CREATEs and the version bump. No add-only trigger is added for these tables — the code index is a
|
|
340
355
|
// cache, never add-only.
|
|
341
356
|
function migrateV5ToV6(db) {
|
|
342
|
-
db.exec("BEGIN");
|
|
357
|
+
db.exec("BEGIN IMMEDIATE");
|
|
343
358
|
try {
|
|
344
359
|
db.exec(CODE_INDEX_SCHEMA_SQL);
|
|
345
360
|
db.exec("PRAGMA user_version = 6");
|
|
@@ -366,7 +381,7 @@ const V7_CONDITION_COLUMNS = [
|
|
|
366
381
|
];
|
|
367
382
|
|
|
368
383
|
function migrateV6ToV7(db) {
|
|
369
|
-
db.exec("BEGIN");
|
|
384
|
+
db.exec("BEGIN IMMEDIATE");
|
|
370
385
|
try {
|
|
371
386
|
const cols = db.prepare("PRAGMA table_info(outcomes)").all().map((c) => c.name);
|
|
372
387
|
for (const [name, type] of V7_CONDITION_COLUMNS) {
|
|
@@ -388,7 +403,7 @@ function migrateV6ToV7(db) {
|
|
|
388
403
|
const V8_CODEX_COLUMNS = [["codex_usd", "REAL"]];
|
|
389
404
|
|
|
390
405
|
function migrateV7ToV8(db) {
|
|
391
|
-
db.exec("BEGIN");
|
|
406
|
+
db.exec("BEGIN IMMEDIATE");
|
|
392
407
|
try {
|
|
393
408
|
const cols = db.prepare("PRAGMA table_info(outcomes)").all().map((c) => c.name);
|
|
394
409
|
for (const [name, type] of V8_CODEX_COLUMNS) {
|
|
@@ -409,7 +424,7 @@ function migrateV7ToV8(db) {
|
|
|
409
424
|
const V9_STOP_COLUMNS = [["stop_class", "TEXT"]];
|
|
410
425
|
|
|
411
426
|
function migrateV8ToV9(db) {
|
|
412
|
-
db.exec("BEGIN");
|
|
427
|
+
db.exec("BEGIN IMMEDIATE");
|
|
413
428
|
try {
|
|
414
429
|
const cols = db.prepare("PRAGMA table_info(outcomes)").all().map((c) => c.name);
|
|
415
430
|
for (const [name, type] of V9_STOP_COLUMNS) {
|
|
@@ -423,15 +438,74 @@ function migrateV8ToV9(db) {
|
|
|
423
438
|
}
|
|
424
439
|
}
|
|
425
440
|
|
|
441
|
+
// v9 had no memory_config column on outcomes (card memory-outcome-arms): the canonical JSON of the memory
|
|
442
|
+
// configuration a row ran under, copied from condition.json's memoryConfig. Additive like v8->v9: one guarded
|
|
443
|
+
// ALTER, then the version bump; old rows stay null. OUTCOMES_COLUMNS_SQL carries memory_config for a v1 store's
|
|
444
|
+
// rebuild, so this step is a no-op there instead of a duplicate-column error.
|
|
445
|
+
const V10_MEMORY_COLUMNS = [["memory_config", "TEXT"]];
|
|
446
|
+
|
|
447
|
+
function migrateV9ToV10(db) {
|
|
448
|
+
db.exec("BEGIN IMMEDIATE");
|
|
449
|
+
try {
|
|
450
|
+
const cols = db.prepare("PRAGMA table_info(outcomes)").all().map((c) => c.name);
|
|
451
|
+
for (const [name, type] of V10_MEMORY_COLUMNS) {
|
|
452
|
+
if (!cols.includes(name)) db.exec(`ALTER TABLE outcomes ADD COLUMN ${name} ${type};`);
|
|
453
|
+
}
|
|
454
|
+
db.exec("PRAGMA user_version = 10");
|
|
455
|
+
db.exec("COMMIT");
|
|
456
|
+
} catch (err) {
|
|
457
|
+
db.exec("ROLLBACK");
|
|
458
|
+
throw err;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// v10 had no seen column on lessons (card memory-reflect-supersede): how many times a reflect pass judged a
|
|
463
|
+
// proposal a repeat of the lesson. One guarded ALTER (LESSONS_COLUMNS_SQL already carries seen for a rebuilt
|
|
464
|
+
// store), then the version bump. seen is not in the no-rewrite trigger's column list.
|
|
465
|
+
function migrateV10ToV11(db) {
|
|
466
|
+
db.exec("BEGIN IMMEDIATE");
|
|
467
|
+
try {
|
|
468
|
+
const cols = db.prepare("PRAGMA table_info(lessons)").all().map((c) => c.name);
|
|
469
|
+
if (!cols.includes("seen")) db.exec("ALTER TABLE lessons ADD COLUMN seen INTEGER NOT NULL DEFAULT 0;");
|
|
470
|
+
db.exec("PRAGMA user_version = 11");
|
|
471
|
+
db.exec("COMMIT");
|
|
472
|
+
} catch (err) {
|
|
473
|
+
db.exec("ROLLBACK");
|
|
474
|
+
throw err;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// v11 had no archived status on lessons (card memory-feedback-rank): archived is an ISO time (NULL = live) and
|
|
479
|
+
// archived_reason names the counters that met the archive rule. Two guarded ALTERs (LESSONS_COLUMNS_SQL already
|
|
480
|
+
// carries both for a rebuilt store), then the version bump. Neither is in the no-rewrite trigger's column list,
|
|
481
|
+
// and the no-delete trigger is unchanged: an archived row is kept.
|
|
482
|
+
function migrateV11ToV12(db) {
|
|
483
|
+
db.exec("BEGIN IMMEDIATE");
|
|
484
|
+
try {
|
|
485
|
+
const cols = db.prepare("PRAGMA table_info(lessons)").all().map((c) => c.name);
|
|
486
|
+
if (!cols.includes("archived")) db.exec("ALTER TABLE lessons ADD COLUMN archived TEXT;");
|
|
487
|
+
if (!cols.includes("archived_reason")) db.exec("ALTER TABLE lessons ADD COLUMN archived_reason TEXT;");
|
|
488
|
+
db.exec("PRAGMA user_version = 12");
|
|
489
|
+
db.exec("COMMIT");
|
|
490
|
+
} catch (err) {
|
|
491
|
+
db.exec("ROLLBACK");
|
|
492
|
+
throw err;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// Bounded wait for another process's write lock; well under the 5 s SessionStart hook timeout.
|
|
497
|
+
export const MEMORY_BUSY_TIMEOUT_MS = 2000;
|
|
498
|
+
|
|
426
499
|
export function openMemory(dir) {
|
|
427
500
|
const file = memoryPath(dir);
|
|
428
501
|
mkdirSync(dirname(file), { recursive: true });
|
|
429
|
-
const db = new DatabaseSync(file);
|
|
502
|
+
const db = new DatabaseSync(file, { timeout: MEMORY_BUSY_TIMEOUT_MS });
|
|
503
|
+
db.exec(`PRAGMA busy_timeout = ${MEMORY_BUSY_TIMEOUT_MS}`);
|
|
430
504
|
db.exec("PRAGMA foreign_keys = ON");
|
|
431
505
|
const version = db.prepare("PRAGMA user_version").get().user_version;
|
|
432
506
|
if (version === 0) {
|
|
433
507
|
db.exec(SCHEMA_SQL);
|
|
434
|
-
db.exec("PRAGMA user_version =
|
|
508
|
+
db.exec("PRAGMA user_version = 12");
|
|
435
509
|
} else if (version === 1) {
|
|
436
510
|
migrateV1ToV2(db);
|
|
437
511
|
migrateV2ToV3(db);
|
|
@@ -441,6 +515,9 @@ export function openMemory(dir) {
|
|
|
441
515
|
migrateV6ToV7(db);
|
|
442
516
|
migrateV7ToV8(db);
|
|
443
517
|
migrateV8ToV9(db);
|
|
518
|
+
migrateV9ToV10(db);
|
|
519
|
+
migrateV10ToV11(db);
|
|
520
|
+
migrateV11ToV12(db);
|
|
444
521
|
} else if (version === 2) {
|
|
445
522
|
migrateV2ToV3(db);
|
|
446
523
|
migrateV3ToV4(db);
|
|
@@ -449,6 +526,9 @@ export function openMemory(dir) {
|
|
|
449
526
|
migrateV6ToV7(db);
|
|
450
527
|
migrateV7ToV8(db);
|
|
451
528
|
migrateV8ToV9(db);
|
|
529
|
+
migrateV9ToV10(db);
|
|
530
|
+
migrateV10ToV11(db);
|
|
531
|
+
migrateV11ToV12(db);
|
|
452
532
|
} else if (version === 3) {
|
|
453
533
|
migrateV3ToV4(db);
|
|
454
534
|
migrateV4ToV5(db);
|
|
@@ -456,29 +536,56 @@ export function openMemory(dir) {
|
|
|
456
536
|
migrateV6ToV7(db);
|
|
457
537
|
migrateV7ToV8(db);
|
|
458
538
|
migrateV8ToV9(db);
|
|
539
|
+
migrateV9ToV10(db);
|
|
540
|
+
migrateV10ToV11(db);
|
|
541
|
+
migrateV11ToV12(db);
|
|
459
542
|
} else if (version === 4) {
|
|
460
543
|
migrateV4ToV5(db);
|
|
461
544
|
migrateV5ToV6(db);
|
|
462
545
|
migrateV6ToV7(db);
|
|
463
546
|
migrateV7ToV8(db);
|
|
464
547
|
migrateV8ToV9(db);
|
|
548
|
+
migrateV9ToV10(db);
|
|
549
|
+
migrateV10ToV11(db);
|
|
550
|
+
migrateV11ToV12(db);
|
|
465
551
|
} else if (version === 5) {
|
|
466
552
|
migrateV5ToV6(db);
|
|
467
553
|
migrateV6ToV7(db);
|
|
468
554
|
migrateV7ToV8(db);
|
|
469
555
|
migrateV8ToV9(db);
|
|
556
|
+
migrateV9ToV10(db);
|
|
557
|
+
migrateV10ToV11(db);
|
|
558
|
+
migrateV11ToV12(db);
|
|
470
559
|
} else if (version === 6) {
|
|
471
560
|
migrateV6ToV7(db);
|
|
472
561
|
migrateV7ToV8(db);
|
|
473
562
|
migrateV8ToV9(db);
|
|
563
|
+
migrateV9ToV10(db);
|
|
564
|
+
migrateV10ToV11(db);
|
|
565
|
+
migrateV11ToV12(db);
|
|
474
566
|
} else if (version === 7) {
|
|
475
567
|
migrateV7ToV8(db);
|
|
476
568
|
migrateV8ToV9(db);
|
|
569
|
+
migrateV9ToV10(db);
|
|
570
|
+
migrateV10ToV11(db);
|
|
571
|
+
migrateV11ToV12(db);
|
|
477
572
|
} else if (version === 8) {
|
|
478
573
|
migrateV8ToV9(db);
|
|
479
|
-
|
|
574
|
+
migrateV9ToV10(db);
|
|
575
|
+
migrateV10ToV11(db);
|
|
576
|
+
migrateV11ToV12(db);
|
|
577
|
+
} else if (version === 9) {
|
|
578
|
+
migrateV9ToV10(db);
|
|
579
|
+
migrateV10ToV11(db);
|
|
580
|
+
migrateV11ToV12(db);
|
|
581
|
+
} else if (version === 10) {
|
|
582
|
+
migrateV10ToV11(db);
|
|
583
|
+
migrateV11ToV12(db);
|
|
584
|
+
} else if (version === 11) {
|
|
585
|
+
migrateV11ToV12(db);
|
|
586
|
+
} else if (version !== 12) {
|
|
480
587
|
db.close();
|
|
481
|
-
throw new Error(`memory store ${file} is user_version ${version}; this code understands up to version
|
|
588
|
+
throw new Error(`memory store ${file} is user_version ${version}; this code understands up to version 12`);
|
|
482
589
|
}
|
|
483
590
|
return {
|
|
484
591
|
file,
|
|
@@ -512,12 +619,13 @@ export function probeSqliteFts5() {
|
|
|
512
619
|
// Read-only look at the store for `doug doctor`: creates nothing, migrates nothing, never throws.
|
|
513
620
|
export function inspectMemory(dir) {
|
|
514
621
|
const file = memoryPath(dir);
|
|
515
|
-
const out = { file, exists: false, userVersion: null, maxUserVersion:
|
|
622
|
+
const out = { file, exists: false, userVersion: null, maxUserVersion: 12, quickCheck: null, error: null };
|
|
516
623
|
if (!existsSync(file)) return out;
|
|
517
624
|
out.exists = true;
|
|
518
625
|
let db;
|
|
519
626
|
try {
|
|
520
|
-
db = new DatabaseSync(file, { readOnly: true });
|
|
627
|
+
db = new DatabaseSync(file, { readOnly: true, timeout: MEMORY_BUSY_TIMEOUT_MS });
|
|
628
|
+
db.exec(`PRAGMA busy_timeout = ${MEMORY_BUSY_TIMEOUT_MS}`);
|
|
521
629
|
out.userVersion = db.prepare("PRAGMA user_version").get().user_version;
|
|
522
630
|
out.quickCheck = String(db.prepare("PRAGMA quick_check").get().quick_check);
|
|
523
631
|
} catch (err) {
|
|
@@ -585,7 +693,7 @@ export function addLesson(m, opts) {
|
|
|
585
693
|
commit_sha: commit,
|
|
586
694
|
created,
|
|
587
695
|
};
|
|
588
|
-
m.exec("BEGIN");
|
|
696
|
+
m.exec("BEGIN IMMEDIATE");
|
|
589
697
|
try {
|
|
590
698
|
m.prepare(
|
|
591
699
|
`INSERT INTO lessons (id, text, kind, scope, citation, source_agent, source_model, task, card, commit_sha, created)
|
|
@@ -616,6 +724,20 @@ export function supersedeLesson(m, oldId, newId) {
|
|
|
616
724
|
return getLesson(m, oldId);
|
|
617
725
|
}
|
|
618
726
|
|
|
727
|
+
// The one live-lesson predicate: not superseded, not stale, not archived. Every reader of "live" lessons uses it.
|
|
728
|
+
export const LIVE_LESSON_SQL = "superseded_by IS NULL AND stale IS NULL AND archived IS NULL";
|
|
729
|
+
export const isLiveLesson = (row) => !row.superseded_by && !row.stale && !row.archived;
|
|
730
|
+
|
|
731
|
+
// Restores an archived lesson (card memory-archived-readers): clears archived and archived_reason, the way back
|
|
732
|
+
// from reflect --harmful's archive. Neither column is protected by lessons_no_rewrite; nothing is deleted.
|
|
733
|
+
export function unarchiveLesson(m, id) {
|
|
734
|
+
const row = getLesson(m, id);
|
|
735
|
+
if (!row) throw new Error(`lesson ${id} does not exist`);
|
|
736
|
+
if (!row.archived) throw new Error(`lesson ${id} is not archived`);
|
|
737
|
+
m.prepare("UPDATE lessons SET archived = NULL, archived_reason = NULL WHERE id = ?").run(id);
|
|
738
|
+
return getLesson(m, id);
|
|
739
|
+
}
|
|
740
|
+
|
|
619
741
|
// Clears a stale mark (set by recallLessons's citation re-check) so the lesson is a candidate again — the way
|
|
620
742
|
// back once a checkout is fixed up or the mark turns out to be wrong. `stale` is not one of the columns the
|
|
621
743
|
// lessons_no_rewrite trigger protects, so this is an ordinary UPDATE, not a special case.
|
|
@@ -633,6 +755,42 @@ export function listLessons(m, { includeSuperseded = false } = {}) {
|
|
|
633
755
|
return rows.map(lessonRowToObject);
|
|
634
756
|
}
|
|
635
757
|
|
|
758
|
+
// Ids of every lesson fetched with `lesson show --card`: the {"id": ...} lines of every
|
|
759
|
+
// <dir>/.doug/.state/reports/*/fetched.jsonl. Never throws; unreadable files and bad lines are skipped.
|
|
760
|
+
export function readFetchedLessonIds(dir) {
|
|
761
|
+
const ids = new Set();
|
|
762
|
+
let cards = [];
|
|
763
|
+
try {
|
|
764
|
+
cards = readdirSync(join(dir, ".doug", ".state", "reports"));
|
|
765
|
+
} catch {
|
|
766
|
+
return ids;
|
|
767
|
+
}
|
|
768
|
+
for (const card of cards) {
|
|
769
|
+
let text;
|
|
770
|
+
try {
|
|
771
|
+
text = readFileSync(join(dir, ".doug", ".state", "reports", card, "fetched.jsonl"), "utf8");
|
|
772
|
+
} catch {
|
|
773
|
+
continue;
|
|
774
|
+
}
|
|
775
|
+
for (const line of text.split("\n")) {
|
|
776
|
+
try {
|
|
777
|
+
const id = JSON.parse(line)?.id;
|
|
778
|
+
if (typeof id === "string") ids.add(id);
|
|
779
|
+
} catch {}
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
return ids;
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
// Live lessons no run has confirmed (ADR 0019): helpful = 0 and harmful = 0 and never fetched. Read-only.
|
|
786
|
+
export function listUnconfirmedLessons(m, { fetchedIds = new Set() } = {}) {
|
|
787
|
+
return m
|
|
788
|
+
.prepare(`SELECT id, kind, created, text FROM lessons WHERE ${LIVE_LESSON_SQL} AND helpful = 0 AND harmful = 0 ORDER BY created ASC, id ASC`)
|
|
789
|
+
.all()
|
|
790
|
+
.filter((r) => !fetchedIds.has(r.id))
|
|
791
|
+
.map((r) => ({ id: r.id, kind: r.kind, created: r.created, title: lessonTitle(r.text) }));
|
|
792
|
+
}
|
|
793
|
+
|
|
636
794
|
// Whitespace-split tokens, each quoted for FTS5 MATCH and OR'd together; shared by searchLessons and
|
|
637
795
|
// recallLessons's BM25 side, and by lib/code-index.mjs's searchIndex (same FTS5 dialect, one code_chunks_fts
|
|
638
796
|
// table instead of lessons_fts). Returns null for an empty/whitespace-only query (nothing to match).
|
|
@@ -698,7 +856,7 @@ export function lessonsNeedingEmbedding(m, { model, dims }) {
|
|
|
698
856
|
const rows = m
|
|
699
857
|
.prepare(
|
|
700
858
|
`SELECT * FROM lessons
|
|
701
|
-
WHERE
|
|
859
|
+
WHERE ${LIVE_LESSON_SQL}
|
|
702
860
|
AND (embedding IS NULL OR embedding_model IS NOT ? OR embedding_dims IS NOT ?)
|
|
703
861
|
ORDER BY created ASC, id ASC`
|
|
704
862
|
)
|
|
@@ -930,20 +1088,21 @@ function recallRows(m, provider, nowMs, staleDays, sig) {
|
|
|
930
1088
|
// No live row holds a vector under this model/dims: nothing to decode, and no blob is read.
|
|
931
1089
|
const any = m
|
|
932
1090
|
.prepare(
|
|
933
|
-
|
|
1091
|
+
`SELECT 1 AS x FROM lessons WHERE ${LIVE_LESSON_SQL} AND embedding IS NOT NULL AND embedding_model = ? AND embedding_dims = ? LIMIT 1`
|
|
934
1092
|
)
|
|
935
1093
|
.get(provider.model, provider.dims);
|
|
936
1094
|
if (!any) return null;
|
|
937
1095
|
|
|
938
1096
|
const stmt = m.prepare(
|
|
939
|
-
`SELECT id, created, confirmed, scope,
|
|
1097
|
+
`SELECT id, created, confirmed, scope, harmful, helpful, citation,
|
|
940
1098
|
CASE WHEN embedding_model = ? AND embedding_dims = ? THEN embedding END AS emb
|
|
941
1099
|
FROM lessons
|
|
942
|
-
WHERE
|
|
1100
|
+
WHERE ${LIVE_LESSON_SQL} AND ${SQL_OUTSIDE_WINDOW} = 0`
|
|
943
1101
|
);
|
|
944
1102
|
const ids = [];
|
|
945
1103
|
const anchorMs = [];
|
|
946
1104
|
const scopeRaw = [];
|
|
1105
|
+
const fbInfo = [];
|
|
947
1106
|
const vecs = [];
|
|
948
1107
|
let bytes = 0;
|
|
949
1108
|
const cutoffMs = (cutoffJd - 2440587.5) * 86400000;
|
|
@@ -952,6 +1111,7 @@ function recallRows(m, provider, nowMs, staleDays, sig) {
|
|
|
952
1111
|
ids.push(r.id);
|
|
953
1112
|
anchorMs.push(new Date(r.confirmed || r.created).getTime());
|
|
954
1113
|
scopeRaw.push(r.scope);
|
|
1114
|
+
fbInfo.push({ harmful: r.harmful, helpful: r.helpful, citation: r.citation });
|
|
955
1115
|
const vec = blobToFloats(r.emb);
|
|
956
1116
|
vecs.push(vec);
|
|
957
1117
|
if (vec) bytes += vec.byteLength;
|
|
@@ -964,6 +1124,7 @@ function recallRows(m, provider, nowMs, staleDays, sig) {
|
|
|
964
1124
|
ids,
|
|
965
1125
|
anchorMs: Float64Array.from(anchorMs),
|
|
966
1126
|
scopeRaw,
|
|
1127
|
+
fbInfo,
|
|
967
1128
|
vecs,
|
|
968
1129
|
live: new Uint8Array(ids.length).fill(1),
|
|
969
1130
|
bytes,
|
|
@@ -1007,7 +1168,7 @@ function followOwnWrites(m, entry, written, confirmedIso, sigBefore) {
|
|
|
1007
1168
|
// count outside the window by exactly ageDaysOf's rule, the live total, and the oldest hidden age in whole days.
|
|
1008
1169
|
export function windowHidden(m, { staleDays = 30, now = new Date() } = {}) {
|
|
1009
1170
|
const nowMs = (now instanceof Date ? now : new Date(now)).getTime();
|
|
1010
|
-
const rows = m.prepare(
|
|
1171
|
+
const rows = m.prepare(`SELECT created, confirmed FROM lessons WHERE ${LIVE_LESSON_SQL}`).all();
|
|
1011
1172
|
let count = 0;
|
|
1012
1173
|
let oldest = -Infinity;
|
|
1013
1174
|
for (const r of rows) {
|
|
@@ -1035,8 +1196,51 @@ export function windowNote(hidden, staleDays = 30) {
|
|
|
1035
1196
|
// returned unchecked (checked: false in the result) rather than either throwing (no checkoutDir given) or
|
|
1036
1197
|
// marking every row stale forever (a wrong one given). Never throws for a provider or checkout problem (a
|
|
1037
1198
|
// database problem may still throw, same as every other export here). See design #2 in the memory-recall card.
|
|
1199
|
+
// Feedback in recall scoring (card memory-feedback-rank, ADR 0019). harmful (the lead's explicit mark) and the
|
|
1200
|
+
// lesson's evidence class (by citation) multiply the score; helpful enters only as a capped, optionally decayed
|
|
1201
|
+
// term that ships off (helpfulCap 0), because it rises on every verified use and would feed itself. harmful 0,
|
|
1202
|
+
// weight 1 and helpfulCap 0 leave the score exactly as before. The multiplier is applied to the score's
|
|
1203
|
+
// magnitude (score + |score| * (factor - 1)) so a negative cosine is lowered by a factor under 1, not raised.
|
|
1204
|
+
export const RECALL_FEEDBACK = {
|
|
1205
|
+
harmfulPenalty: 0.01,
|
|
1206
|
+
helpfulCap: 0,
|
|
1207
|
+
helpfulStep: 0.05,
|
|
1208
|
+
helpfulDecay: 0,
|
|
1209
|
+
evidenceWeights: { report: 1, history: 1, file: 1, none: 1 },
|
|
1210
|
+
};
|
|
1211
|
+
|
|
1212
|
+
export function evidenceClassOf(lesson) {
|
|
1213
|
+
const c = lesson && typeof lesson.citation === "string" ? lesson.citation : "";
|
|
1214
|
+
if (!c) return "none";
|
|
1215
|
+
if (c.startsWith("report:")) return "report";
|
|
1216
|
+
if (c.startsWith("history:")) return "history";
|
|
1217
|
+
return "file";
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
function mergeFeedback(over) {
|
|
1221
|
+
const o = over || {};
|
|
1222
|
+
return { ...RECALL_FEEDBACK, ...o, evidenceWeights: { ...RECALL_FEEDBACK.evidenceWeights, ...(o.evidenceWeights || {}) } };
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
function feedbackScore(score, info, fb) {
|
|
1226
|
+
if (!info) return score;
|
|
1227
|
+
const harmful = Number(info.harmful) || 0;
|
|
1228
|
+
let factor = 1;
|
|
1229
|
+
if (harmful > 0) factor *= 1 / (1 + fb.harmfulPenalty * harmful);
|
|
1230
|
+
const w = fb.evidenceWeights[evidenceClassOf(info)];
|
|
1231
|
+
if (w !== undefined && w !== 1) factor *= w;
|
|
1232
|
+
if (fb.helpfulCap > 0) {
|
|
1233
|
+
const h = Math.max(0, Number(info.helpful) || 0);
|
|
1234
|
+
const d = fb.helpfulDecay;
|
|
1235
|
+
const eff = d > 0 && d < 1 ? (1 - Math.pow(d, h)) / (1 - d) : h;
|
|
1236
|
+
factor *= 1 + fb.helpfulStep * Math.min(eff, fb.helpfulCap);
|
|
1237
|
+
}
|
|
1238
|
+
return factor === 1 ? score : score + Math.abs(score) * (factor - 1);
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1038
1241
|
export async function recallLessons(m, query, opts = {}) {
|
|
1039
1242
|
const { files = [], k = 8, now = new Date(), provider = null, checkoutDir = null, staleDays = 30, detectedProvider = null } = opts;
|
|
1243
|
+
const fb = mergeFeedback(opts.feedback);
|
|
1040
1244
|
const nowDate = now instanceof Date ? now : new Date(now);
|
|
1041
1245
|
const nowIsoStr = nowDate.toISOString();
|
|
1042
1246
|
const nowMs = nowDate.getTime();
|
|
@@ -1050,10 +1254,10 @@ export async function recallLessons(m, query, opts = {}) {
|
|
|
1050
1254
|
if (match) {
|
|
1051
1255
|
const rows = m
|
|
1052
1256
|
.prepare(
|
|
1053
|
-
`SELECT l.id, l.created, l.confirmed, l.scope,
|
|
1257
|
+
`SELECT l.id, l.created, l.confirmed, l.scope, l.harmful, l.helpful, l.citation,
|
|
1054
1258
|
(l.embedding IS NOT NULL AND l.embedding_model = ? AND l.embedding_dims = ?) AS has_match
|
|
1055
1259
|
FROM lessons_fts f JOIN lessons l ON l.id = f.id
|
|
1056
|
-
WHERE lessons_fts MATCH ? AND
|
|
1260
|
+
WHERE lessons_fts MATCH ? AND ${LIVE_LESSON_SQL}
|
|
1057
1261
|
ORDER BY bm25(lessons_fts) LIMIT 50`
|
|
1058
1262
|
)
|
|
1059
1263
|
.all(provider ? provider.model : null, provider ? provider.dims : null, match);
|
|
@@ -1061,7 +1265,7 @@ export async function recallLessons(m, query, opts = {}) {
|
|
|
1061
1265
|
const age = ageDaysOf(r.confirmed || r.created, nowMs);
|
|
1062
1266
|
if (!(age <= staleDays)) continue;
|
|
1063
1267
|
bm25Ranked.push(r.id);
|
|
1064
|
-
bm25Info.set(r.id, { scope: r.scope, hasMatch: !!r.has_match, age });
|
|
1268
|
+
bm25Info.set(r.id, { scope: r.scope, hasMatch: !!r.has_match, age, fb: { harmful: r.harmful, helpful: r.helpful, citation: r.citation } });
|
|
1065
1269
|
}
|
|
1066
1270
|
}
|
|
1067
1271
|
|
|
@@ -1123,7 +1327,7 @@ export async function recallLessons(m, query, opts = {}) {
|
|
|
1123
1327
|
// model, or the caller detected a usable provider; hybrid reports how many live in-window rows lack a vector.
|
|
1124
1328
|
// The existence check is bounded and only runs when no other term already decided it.
|
|
1125
1329
|
const hasAnyVector = (db) =>
|
|
1126
|
-
!!db.prepare(
|
|
1330
|
+
!!db.prepare(`SELECT 1 AS x FROM lessons WHERE ${LIVE_LESSON_SQL} AND embedding IS NOT NULL LIMIT 1`).get();
|
|
1127
1331
|
const degraded = mode === "keyword-only" && (!!provider || hasAnyVector(m) || !!detectedProvider);
|
|
1128
1332
|
const coverage = {};
|
|
1129
1333
|
if (mode === "hybrid") {
|
|
@@ -1147,21 +1351,27 @@ export async function recallLessons(m, query, opts = {}) {
|
|
|
1147
1351
|
// object per stored vector.
|
|
1148
1352
|
const bm25Only = (id) => {
|
|
1149
1353
|
const info = bm25Info.get(id);
|
|
1150
|
-
return { id, score: weigh(info.age, info.scope, 1 / (60 + bm25Rank.get(id) + 1)), sides: ["bm25"] };
|
|
1354
|
+
return { id, score: feedbackScore(weigh(info.age, info.scope, 1 / (60 + bm25Rank.get(id) + 1)), info.fb, fb), sides: ["bm25"] };
|
|
1151
1355
|
};
|
|
1152
1356
|
let candidates;
|
|
1153
1357
|
if (mode === "hybrid") {
|
|
1154
1358
|
// (e) vectored lessons by cosine x recency x scope, then BM25 hits with no vector, in BM25 order.
|
|
1155
1359
|
const order = Int32Array.from(cosine);
|
|
1156
1360
|
const scores = new Float64Array(rows.n);
|
|
1157
|
-
for (const r of cosine) scores[r] = weigh(ages[r], rows.scopeRaw[r], cosineScores[r]);
|
|
1361
|
+
for (const r of cosine) scores[r] = feedbackScore(weigh(ages[r], rows.scopeRaw[r], cosineScores[r]), rows.fbInfo[r], fb);
|
|
1158
1362
|
order.sort((x, y) => scores[y] - scores[x] || idCmp(rows.ids[x], rows.ids[y]));
|
|
1159
1363
|
const dense = { length: order.length, at: (j) => {
|
|
1160
1364
|
const r = order[j];
|
|
1161
1365
|
const id = rows.ids[r];
|
|
1162
1366
|
return { id, score: scores[r], sides: bm25Rank.has(id) ? ["bm25", "vector"] : ["vector"] };
|
|
1163
1367
|
} };
|
|
1164
|
-
|
|
1368
|
+
// The tail keeps BM25 order (not the recency-weighted score), reordered only by the feedback factor: the sort
|
|
1369
|
+
// key is the rank score 1/(60+rank) through feedbackScore, so a harmful twin falls below its harmful-0 twin.
|
|
1370
|
+
const appended = bm25Ranked
|
|
1371
|
+
.filter((id) => !bm25Info.get(id).hasMatch)
|
|
1372
|
+
.map((id, pos) => ({ cand: bm25Only(id), pos, key: feedbackScore(1 / (60 + bm25Rank.get(id) + 1), bm25Info.get(id).fb, fb) }))
|
|
1373
|
+
.sort((a, b) => b.key - a.key || a.pos - b.pos)
|
|
1374
|
+
.map((x) => x.cand);
|
|
1165
1375
|
candidates = {
|
|
1166
1376
|
length: dense.length + appended.length,
|
|
1167
1377
|
at: (j) => (j < dense.length ? dense.at(j) : appended[j - dense.length]),
|
|
@@ -1259,6 +1469,72 @@ export function renderLessons(lessons) {
|
|
|
1259
1469
|
return text.length ? { text: text.join("\n"), ids } : { text: null, ids: [] };
|
|
1260
1470
|
}
|
|
1261
1471
|
|
|
1472
|
+
// The lesson index (card memory-progressive-access, ADR 0019 retrieval): one line per lesson in rank order carrying
|
|
1473
|
+
// the id, a one-line title, the kind and the token cost of the WHOLE text (chars / 4, the repo's rule), so the lead
|
|
1474
|
+
// can fetch the full text of the lessons a card needs with `memory.mjs lesson show <id>`. A line that would push the
|
|
1475
|
+
// running total past `cap` is skipped (not stopped at) and its id is reported in `omitted`: no silent filter.
|
|
1476
|
+
const INDEX_TITLE_CHARS = 100;
|
|
1477
|
+
export const lessonTokens = (text) => Math.ceil(String(text).length / 4);
|
|
1478
|
+
export function lessonTitle(text) {
|
|
1479
|
+
const first = String(text).split("\n").find((l) => l.trim() !== "") ?? "";
|
|
1480
|
+
const line = first.trim().replace(/\s+/g, " ");
|
|
1481
|
+
if (line.length <= INDEX_TITLE_CHARS) return line;
|
|
1482
|
+
const slice = line.slice(0, INDEX_TITLE_CHARS);
|
|
1483
|
+
const sp = slice.lastIndexOf(" ");
|
|
1484
|
+
return (sp > 0 ? slice.slice(0, sp) : slice).trimEnd() + "…";
|
|
1485
|
+
}
|
|
1486
|
+
export function renderLessonIndex(lessons, { cap = LESSONS_CHAR_CAP } = {}) {
|
|
1487
|
+
const text = [];
|
|
1488
|
+
const ids = [];
|
|
1489
|
+
const omitted = [];
|
|
1490
|
+
let length = 0;
|
|
1491
|
+
for (const lesson of Array.isArray(lessons) ? lessons : []) {
|
|
1492
|
+
const line = `- [${lesson.id}] ${lessonTitle(lesson.text)} (${lesson.kind}, ~${lessonTokens(lesson.text)} tokens)`;
|
|
1493
|
+
const nextLength = length + (text.length ? 1 : 0) + line.length;
|
|
1494
|
+
if (nextLength > cap) {
|
|
1495
|
+
omitted.push(lesson.id);
|
|
1496
|
+
continue;
|
|
1497
|
+
}
|
|
1498
|
+
text.push(line);
|
|
1499
|
+
ids.push(lesson.id);
|
|
1500
|
+
length = nextLength;
|
|
1501
|
+
}
|
|
1502
|
+
return { text: text.length ? text.join("\n") : null, ids, omitted };
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
// The flow track's delivery (card memory-atomic-lessons): the index, then the full text of the top lessons in
|
|
1506
|
+
// rank order up to FLOW_FETCH_TOKENS (chars / 4, rounded up, per lesson). A lesson whose full text would pass the
|
|
1507
|
+
// remaining budget is skipped (not stopped at), stays in the index, is named on a `not fetched` line and is never
|
|
1508
|
+
// cut. Fetched lessons use the `lesson show` form. `ids` are the FETCHED ids in rank order (what the run credits as
|
|
1509
|
+
// used), `indexIds` every indexed id, `omitted` the ids the index cap dropped, `unfetched` the indexed ids over budget.
|
|
1510
|
+
export const FLOW_FETCH_TOKENS = 4000;
|
|
1511
|
+
export function renderFlowLessons(lessons, { indexCap = LESSONS_CHAR_CAP, fetchTokens = FLOW_FETCH_TOKENS } = {}) {
|
|
1512
|
+
const index = renderLessonIndex(lessons, { cap: indexCap });
|
|
1513
|
+
if (index.text === null) return { text: null, ids: [], indexIds: [], omitted: index.omitted, unfetched: [] };
|
|
1514
|
+
const byId = new Map((Array.isArray(lessons) ? lessons : []).map((l) => [l.id, l]));
|
|
1515
|
+
const fetched = [];
|
|
1516
|
+
const unfetched = [];
|
|
1517
|
+
let used = 0;
|
|
1518
|
+
for (const id of index.ids) {
|
|
1519
|
+
const lesson = byId.get(id);
|
|
1520
|
+
const tokens = lessonTokens(lesson.text);
|
|
1521
|
+
if (used + tokens > fetchTokens) {
|
|
1522
|
+
unfetched.push(id);
|
|
1523
|
+
continue;
|
|
1524
|
+
}
|
|
1525
|
+
used += tokens;
|
|
1526
|
+
fetched.push(lesson);
|
|
1527
|
+
}
|
|
1528
|
+
const parts = [index.text];
|
|
1529
|
+
if (index.omitted.length) parts.push(`index: omitted ${index.omitted.length} lessons over the ${indexCap}-character index budget: ${index.omitted.join(", ")}`);
|
|
1530
|
+
if (unfetched.length) parts.push(`not fetched (over the ~${fetchTokens}-token fetch budget): ${unfetched.join(", ")}`);
|
|
1531
|
+
let text = parts.join("\n");
|
|
1532
|
+
if (fetched.length) {
|
|
1533
|
+
text += "\n\n" + fetched.map((l) => `lesson ${l.id} (${l.kind}, ~${lessonTokens(l.text)} tokens)\ncitation: ${l.citation || "none"}\n\n${l.text}`).join("\n\n");
|
|
1534
|
+
}
|
|
1535
|
+
return { text, ids: fetched.map((l) => l.id), indexIds: index.ids, omitted: index.omitted, unfetched };
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1262
1538
|
// The lessons a session starts with when no card query exists (card memory-every-session): live, non-stale
|
|
1263
1539
|
// lessons confirmed-or-created within staleDays, ordered by helpful minus harmful desc, then confirmed-or-created
|
|
1264
1540
|
// desc, then id ascending. No provider, no citation re-check, nothing written.
|
|
@@ -1268,7 +1544,7 @@ export function topLessons(m, { k = 5, staleDays = 30, now = new Date() } = {})
|
|
|
1268
1544
|
.prepare(
|
|
1269
1545
|
`SELECT id, text, kind, scope, citation, source_agent, source_model, task, card, commit_sha, created, confirmed,
|
|
1270
1546
|
helpful, harmful, superseded_by, stale
|
|
1271
|
-
FROM lessons WHERE
|
|
1547
|
+
FROM lessons WHERE ${LIVE_LESSON_SQL}`
|
|
1272
1548
|
)
|
|
1273
1549
|
.all()
|
|
1274
1550
|
.map(lessonRowToObject)
|
|
@@ -1424,7 +1700,7 @@ function findPlanTask(plan, id) {
|
|
|
1424
1700
|
}
|
|
1425
1701
|
|
|
1426
1702
|
export function outcomeRows(report, opts) {
|
|
1427
|
-
const { plan = null, classes = [], run = null, reportHash, reportPath = null, card = null, commit = null, cost = null, now = nowIso() } = opts || {};
|
|
1703
|
+
const { plan = null, classes = [], run = null, reportHash, reportPath = null, card = null, commit = null, cost = null, now = nowIso(), memoryConfig: flowMemoryConfig = null } = opts || {};
|
|
1428
1704
|
const allBlocks = classifyBlocks(report, classes).map((b) => ({
|
|
1429
1705
|
id: b.id,
|
|
1430
1706
|
pass: b.pass,
|
|
@@ -1494,8 +1770,9 @@ export function outcomeRows(report, opts) {
|
|
|
1494
1770
|
gate: null,
|
|
1495
1771
|
note: null,
|
|
1496
1772
|
partial: toIntOrNull(task.partial),
|
|
1497
|
-
// The seven condition columns (card outcomes-condition-columns) are
|
|
1498
|
-
//
|
|
1773
|
+
// The seven condition columns (card outcomes-condition-columns) are null on a flow row — see the comment
|
|
1774
|
+
// above OUTCOMES_COLUMNS_SQL; memory_config is the flow stamp when the caller passes one (card
|
|
1775
|
+
// memory-flow-index-fetch). backfilled is 0, never null: it is NOT NULL DEFAULT 0.
|
|
1499
1776
|
harness_commit: null,
|
|
1500
1777
|
config_hash: null,
|
|
1501
1778
|
class: null,
|
|
@@ -1503,6 +1780,7 @@ export function outcomeRows(report, opts) {
|
|
|
1503
1780
|
assigned_by: null,
|
|
1504
1781
|
explore_probability: null,
|
|
1505
1782
|
backfilled: 0,
|
|
1783
|
+
memory_config: flowMemoryConfig ? JSON.stringify(sortKeysDeep(flowMemoryConfig)) : null,
|
|
1506
1784
|
});
|
|
1507
1785
|
}
|
|
1508
1786
|
}
|
|
@@ -1600,34 +1878,327 @@ export function handOutcomeRow({ card, commit, wallClock = null, gate = null, no
|
|
|
1600
1878
|
assigned_by: condition ? condition.assignedBy ?? null : null,
|
|
1601
1879
|
explore_probability: condition ? condition.exploreProbability ?? null : null,
|
|
1602
1880
|
backfilled: 0,
|
|
1881
|
+
memory_config: condition && condition.memoryConfig ? JSON.stringify(sortKeysDeep(condition.memoryConfig)) : null,
|
|
1882
|
+
};
|
|
1883
|
+
}
|
|
1884
|
+
|
|
1885
|
+
function sortKeysDeep(v) {
|
|
1886
|
+
if (Array.isArray(v)) return v.map(sortKeysDeep);
|
|
1887
|
+
if (v && typeof v === "object") return Object.fromEntries(Object.keys(v).sort().map((k) => [k, sortKeysDeep(v[k])]));
|
|
1888
|
+
return v;
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
// The memory configuration an outcome row is stamped with (card memory-outcome-arms, ADR 0019 Measurement), so
|
|
1892
|
+
// outcomes can be compared per configuration. Pure: the caller reads the project config. Excludes baseUrl and
|
|
1893
|
+
// apiKeyEnv (where the provider lives, not what it computes). MEMORY_STAGE names the recall/ranking/delivery
|
|
1894
|
+
// generation: a card that changes recall or ranking for every track bumps it, which starts a new arm. A change to one
|
|
1895
|
+
// track's delivery alone (card memory-atomic-lessons: the flow track) changes that track's stamp fields instead.
|
|
1896
|
+
export const MEMORY_STAGE = "progressive-access";
|
|
1897
|
+
|
|
1898
|
+
// `track` names the delivery the row's briefs actually got (card memory-flow-index-fetch): a hand card's briefs
|
|
1899
|
+
// carry the index plus a lead fetch; a flow task's briefs (plan.mjs) carry the index plus the full text of the top
|
|
1900
|
+
// lessons up to FLOW_FETCH_TOKENS (card memory-atomic-lessons), told apart from the hand stamp by fetch/fetchTokens. Any other track throws
|
|
1901
|
+
// rather than claiming a delivery.
|
|
1902
|
+
export function memoryConfig({ embeddings = null, staleDays = 30, track = "hand" } = {}) {
|
|
1903
|
+
if (track !== "hand" && track !== "flow") throw new Error(`memoryConfig: unknown track ${JSON.stringify(track)}`);
|
|
1904
|
+
const emb = embeddings ? { provider: embeddings.provider ?? null, model: embeddings.model ?? null, dims: embeddings.dims ?? null } : null;
|
|
1905
|
+
if (track === "flow") {
|
|
1906
|
+
return { stage: MEMORY_STAGE, embeddings: emb, staleDays, k: 8, delivery: "index+fetch", indexChars: LESSONS_CHAR_CAP, fetch: "top-k", fetchTokens: FLOW_FETCH_TOKENS };
|
|
1907
|
+
}
|
|
1908
|
+
return {
|
|
1909
|
+
stage: MEMORY_STAGE,
|
|
1910
|
+
embeddings: emb,
|
|
1911
|
+
staleDays,
|
|
1912
|
+
k: 8,
|
|
1913
|
+
delivery: "index+fetch",
|
|
1914
|
+
indexChars: LESSONS_CHAR_CAP,
|
|
1915
|
+
};
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
// One stopped hand card report, one row (card memory-record-stops, ADR 0019 gap 3): run = 'stop:<first 12 hex of
|
|
1919
|
+
// the report hash>', so recording the same report again updates the same row, a different stop of the card adds
|
|
1920
|
+
// a row, and a later landing ('hand:<commit>') sits beside it. stop_class is the report's typed `reason` and
|
|
1921
|
+
// stop_reason its prose `detail` (the reason when detail is blank). `blocks` are the blocks.jsonl lines
|
|
1922
|
+
// attributed to the run, kept as {at, hook, agent, class, reason}. Every other column is null as on handOutcomeRow.
|
|
1923
|
+
export function stopOutcomeRow({ card, report, reportHash, reportPath = null, run = null, blocks = [], now = nowIso(), condition = null, memoryConfig: stamp = null } = {}) {
|
|
1924
|
+
if (!card) throw new Error("stopOutcomeRow requires card");
|
|
1925
|
+
if (!report || report.outcome !== "stopped") throw new Error("stopOutcomeRow requires a stopped report");
|
|
1926
|
+
if (!reportHash) throw new Error("stopOutcomeRow requires reportHash");
|
|
1927
|
+
const detail = typeof report.detail === "string" && report.detail.trim() !== "" ? report.detail : null;
|
|
1928
|
+
return {
|
|
1929
|
+
...handOutcomeRow({ card, commit: "x", now, condition }),
|
|
1930
|
+
...(!(condition && condition.memoryConfig) && stamp ? { memory_config: JSON.stringify(sortKeysDeep(stamp)) } : {}),
|
|
1931
|
+
run: `stop:${String(reportHash).slice(0, 12)}`,
|
|
1932
|
+
run_id: run ?? null,
|
|
1933
|
+
report_hash: reportHash,
|
|
1934
|
+
report_path: reportPath,
|
|
1935
|
+
commit_sha: null,
|
|
1936
|
+
stop_class: report.reason ?? null,
|
|
1937
|
+
stop_reason: detail ?? report.reason ?? null,
|
|
1938
|
+
blocks: blocks.map((b) => ({ at: b.at ?? null, hook: b.hook ?? null, agent: b.agent ?? null, class: b.class ?? null, reason: b.reason ?? null })),
|
|
1603
1939
|
};
|
|
1604
1940
|
}
|
|
1605
1941
|
|
|
1942
|
+
// The agent id a blocks.jsonl line belongs to: a string `agentId` field, else the id in a subagent-stop reason.
|
|
1943
|
+
export function blockAgentId(line) {
|
|
1944
|
+
if (!line || typeof line !== "object") return null;
|
|
1945
|
+
if (typeof line.agentId === "string" && line.agentId !== "") return line.agentId;
|
|
1946
|
+
const m = typeof line.reason === "string" ? /Subagent \S+ \(([^)\s]+)\)/.exec(line.reason) : null;
|
|
1947
|
+
return m ? m[1] : null;
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
function parseJsonLines(text) {
|
|
1951
|
+
const out = [];
|
|
1952
|
+
for (const raw of String(text ?? "").split("\n")) {
|
|
1953
|
+
if (raw.trim() === "") continue;
|
|
1954
|
+
try {
|
|
1955
|
+
const v = JSON.parse(raw);
|
|
1956
|
+
if (v !== null && typeof v === "object" && !Array.isArray(v)) out.push(v);
|
|
1957
|
+
} catch {
|
|
1958
|
+
// malformed line: skipped
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
return out;
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
// Every agentId on any parseable line of the given queen journal texts.
|
|
1965
|
+
export function runAgentIds(journalTexts) {
|
|
1966
|
+
const ids = new Set();
|
|
1967
|
+
for (const t of journalTexts ?? []) for (const v of parseJsonLines(t)) if (typeof v.agentId === "string" && v.agentId !== "") ids.add(v.agentId);
|
|
1968
|
+
return ids;
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
// The parsed blocks.jsonl lines whose agent id is one of the run's, in file order. A main-session line has none.
|
|
1972
|
+
export function attributeBlocks(ledgerText, agentIds) {
|
|
1973
|
+
return parseJsonLines(ledgerText).filter((v) => {
|
|
1974
|
+
const id = blockAgentId(v);
|
|
1975
|
+
return id !== null && agentIds.has(id);
|
|
1976
|
+
});
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
// Run history (card memory-run-history, ADR 0019 L0): an append-only file per card run under
|
|
1980
|
+
// .doug/.state/history/<card>/<run>.jsonl. Capture depth (ADR 0019 line 30, Open there, decided here for the artifact
|
|
1981
|
+
// slice): copy the text of Doug's own run artifacts (card report, attributed gate blocks and trace denials, each
|
|
1982
|
+
// seat's hand-back from the queen journal, each seat's brief re-rendered by replaying lib/card-loop.mjs); never copy
|
|
1983
|
+
// Claude Code transcripts and never reference into the Claude Code store. Each entry is secret-scanned on its WHOLE
|
|
1984
|
+
// text first (a hit replaces the entire entry), then capped at HISTORY_ENTRY_MAX_BYTES keeping the head; each run
|
|
1985
|
+
// file is capped at HISTORY_RUN_MAX_BYTES with a final 'cap' line counting the dropped entries (no silent filter).
|
|
1986
|
+
// No age bound: files are never deleted or rewritten.
|
|
1987
|
+
export const HISTORY_DIR_RELPATH = ".doug/.state/history";
|
|
1988
|
+
export const HISTORY_ENTRY_MAX_BYTES = 65536;
|
|
1989
|
+
export const HISTORY_RUN_MAX_BYTES = 1048576;
|
|
1990
|
+
|
|
1991
|
+
export function historyEntry({ kind, seat = null, round = null, source, value }, { scan, maxBytes = HISTORY_ENTRY_MAX_BYTES } = {}) {
|
|
1992
|
+
const text0 = typeof value === "string" ? value : String(JSON.stringify(value === undefined ? null : value, null, 2));
|
|
1993
|
+
const bytes = Buffer.byteLength(text0, "utf8");
|
|
1994
|
+
const base = { kind, seat, round, source, bytes };
|
|
1995
|
+
let redacted = null;
|
|
1996
|
+
if (typeof scan !== "function") redacted = "no-scanner";
|
|
1997
|
+
else {
|
|
1998
|
+
try {
|
|
1999
|
+
const hit = scan(text0);
|
|
2000
|
+
if (hit) redacted = typeof hit.rule === "string" && hit.rule !== "" ? hit.rule : "secret";
|
|
2001
|
+
} catch {
|
|
2002
|
+
redacted = "scan-error";
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
if (redacted !== null) return { ...base, truncated: false, redacted, text: `[redacted: ${redacted}]` };
|
|
2006
|
+
if (bytes <= maxBytes) return { ...base, truncated: false, redacted: null, text: text0 };
|
|
2007
|
+
const buf = Buffer.from(text0, "utf8");
|
|
2008
|
+
let cut = maxBytes;
|
|
2009
|
+
while (cut > 0 && (buf[cut] & 0xc0) === 0x80) cut--;
|
|
2010
|
+
return { ...base, truncated: true, redacted: null, text: buf.subarray(0, cut).toString("utf8") };
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
// Seat runs of a queen Workflow journal, in started-line order: {seat, round, agentId, result}.
|
|
2014
|
+
export function journalSeatRuns(journalText) {
|
|
2015
|
+
try {
|
|
2016
|
+
const started = [];
|
|
2017
|
+
const results = new Map();
|
|
2018
|
+
for (const v of parseJsonLines(journalText)) {
|
|
2019
|
+
if (v.type === "started" && typeof v.key === "string") started.push(v);
|
|
2020
|
+
else if (v.type === "result" && typeof v.key === "string") results.set(v.key, v.result);
|
|
2021
|
+
}
|
|
2022
|
+
return started.map((v) => {
|
|
2023
|
+
const parts = String(v.label ?? "").split(":");
|
|
2024
|
+
const last = parts[parts.length - 1];
|
|
2025
|
+
const round = /^\d+$/.test(last) ? Number(last) : null;
|
|
2026
|
+
return { seat: parts[0], round, agentId: v.agentId ?? null, result: results.has(v.key) ? results.get(v.key) ?? null : null };
|
|
2027
|
+
});
|
|
2028
|
+
} catch {
|
|
2029
|
+
return [];
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
2032
|
+
|
|
2033
|
+
// Each seat run's brief, re-rendered by replaying the card-loop contract over the runs' hand-backs; null from the
|
|
2034
|
+
// first seat that does not match the contract's next seat (or on any error).
|
|
2035
|
+
export function replayBriefs(queenArgs, seatRuns) {
|
|
2036
|
+
const runs = Array.isArray(seatRuns) ? seatRuns : [];
|
|
2037
|
+
const out = runs.map(() => null);
|
|
2038
|
+
try {
|
|
2039
|
+
if (queenArgs === null || typeof queenArgs !== "object" || queenArgs.card === null || typeof queenArgs.card !== "object") return out;
|
|
2040
|
+
let s = initialState(queenArgs.card, queenArgs.options);
|
|
2041
|
+
for (let i = 0; i < runs.length; i++) {
|
|
2042
|
+
const a = currentAction(s);
|
|
2043
|
+
if (a.kind !== "seat" || a.seat !== runs[i].seat) break;
|
|
2044
|
+
out[i] = briefPrompt(a.brief, { repoRoot: queenArgs.repoRoot, worktreePath: queenArgs.worktreePath });
|
|
2045
|
+
s = cardLoopNext(s, runs[i].result).state;
|
|
2046
|
+
}
|
|
2047
|
+
} catch {
|
|
2048
|
+
// out keeps null from the failing seat on
|
|
2049
|
+
}
|
|
2050
|
+
return out;
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
// The parsed trace lines that are PermissionDenied events of one of the run's agents, in file order.
|
|
2054
|
+
export function attributeDenials(traceText, agentIds) {
|
|
2055
|
+
try {
|
|
2056
|
+
return parseJsonLines(traceText).filter((v) => v.event === "PermissionDenied" && typeof v.agent === "string" && agentIds.has(v.agent));
|
|
2057
|
+
} catch {
|
|
2058
|
+
return [];
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
const historySafe = (x) => String(x).replace(/[^A-Za-z0-9_.-]/g, "_").replace(/\.{2,}/g, "_").slice(0, 120) || "_";
|
|
2063
|
+
|
|
2064
|
+
// Writes <dir>/.doug/.state/history/<card>/<run>.jsonl once; an existing file is never touched.
|
|
2065
|
+
export function writeRunHistory(dir, { card, run, entries, scan, maxEntryBytes = HISTORY_ENTRY_MAX_BYTES, maxRunBytes = HISTORY_RUN_MAX_BYTES }) {
|
|
2066
|
+
const path = join(dir, HISTORY_DIR_RELPATH, historySafe(card), historySafe(run) + ".jsonl");
|
|
2067
|
+
const lines = [];
|
|
2068
|
+
let used = 0;
|
|
2069
|
+
let truncated = 0;
|
|
2070
|
+
let dropped = 0;
|
|
2071
|
+
for (const e of entries) {
|
|
2072
|
+
const h = historyEntry(e, { scan, maxBytes: maxEntryBytes });
|
|
2073
|
+
const line = JSON.stringify({ kind: h.kind, card, run, seat: h.seat, round: h.round, source: h.source, bytes: h.bytes, truncated: h.truncated, redacted: h.redacted, text: h.text });
|
|
2074
|
+
const size = Buffer.byteLength(line) + 1;
|
|
2075
|
+
if (dropped === 0 && used + size <= maxRunBytes) {
|
|
2076
|
+
lines.push(line);
|
|
2077
|
+
used += size;
|
|
2078
|
+
if (h.truncated) truncated++;
|
|
2079
|
+
} else dropped++;
|
|
2080
|
+
}
|
|
2081
|
+
if (dropped > 0) lines.push(JSON.stringify({ kind: "cap", card, run, dropped, maxRunBytes }));
|
|
2082
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
2083
|
+
try {
|
|
2084
|
+
writeFileSync(path, lines.map((l) => l + "\n").join(""), { flag: "wx" });
|
|
2085
|
+
} catch (e) {
|
|
2086
|
+
if (e && e.code === "EEXIST") return { path, written: false, lines: 0, truncated: 0, dropped: 0 };
|
|
2087
|
+
throw e;
|
|
2088
|
+
}
|
|
2089
|
+
return { path, written: true, lines: lines.length - (dropped > 0 ? 1 : 0), truncated, dropped };
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
// The run-history secret scanner both writers use (autopilot report, reflect --hand), so redaction cannot drift:
|
|
2093
|
+
// .doug/config.json `secrets`; enabled === false scans nothing; otherwise the vendored secret-rules scanner. An
|
|
2094
|
+
// import failure returns undefined, which reads every entry as redacted 'no-scanner'.
|
|
2095
|
+
export async function historyScanner(dir) {
|
|
2096
|
+
let secrets = {};
|
|
2097
|
+
try {
|
|
2098
|
+
secrets = JSON.parse(readFileSync(join(dir, ".doug/config.json"), "utf8")).secrets ?? {};
|
|
2099
|
+
} catch {
|
|
2100
|
+
// no config: defaults
|
|
2101
|
+
}
|
|
2102
|
+
if (secrets && secrets.enabled === false) return () => null;
|
|
2103
|
+
try {
|
|
2104
|
+
const sr = await import(pathToFileURL(join(dir, ".doug/hooks/lib/secret-rules.mjs")).href);
|
|
2105
|
+
return (t) => sr.scanText(t, secrets ?? {});
|
|
2106
|
+
} catch {
|
|
2107
|
+
return undefined;
|
|
2108
|
+
}
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
// History fetch (card memory-progressive-access, ADR 0019 L0 citation form history:<card>/<file>:<line>).
|
|
2112
|
+
// <ref> = [history:]<card>[/<run>[.jsonl]][:<line>]; ids are limited to safe name characters so a ref can never leave
|
|
2113
|
+
// the history directory.
|
|
2114
|
+
export function parseHistoryRef(ref) {
|
|
2115
|
+
const bad = () => new Error(`bad history id: ${JSON.stringify(ref)}`);
|
|
2116
|
+
if (typeof ref !== "string") throw bad();
|
|
2117
|
+
let rest = ref.startsWith("history:") ? ref.slice("history:".length) : ref;
|
|
2118
|
+
let line = null;
|
|
2119
|
+
const colon = rest.lastIndexOf(":");
|
|
2120
|
+
if (colon !== -1) {
|
|
2121
|
+
const n = rest.slice(colon + 1);
|
|
2122
|
+
if (!/^[0-9]+$/.test(n) || Number(n) < 1) throw bad();
|
|
2123
|
+
line = Number(n);
|
|
2124
|
+
rest = rest.slice(0, colon);
|
|
2125
|
+
}
|
|
2126
|
+
const parts = rest.split("/");
|
|
2127
|
+
if (parts.length > 2) throw bad();
|
|
2128
|
+
const [card, runRaw] = parts;
|
|
2129
|
+
const run = runRaw === undefined ? null : runRaw.replace(/\.jsonl$/, "");
|
|
2130
|
+
for (const part of [card, run]) {
|
|
2131
|
+
if (part === null) continue;
|
|
2132
|
+
if (!/^[A-Za-z0-9_.-]+$/.test(part) || part.includes("..")) throw bad();
|
|
2133
|
+
}
|
|
2134
|
+
if (line !== null && run === null) throw bad();
|
|
2135
|
+
return { card, run, line };
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
export function readRunHistory(dir, { card, run = null, line = null }) {
|
|
2139
|
+
const root = join(dir, HISTORY_DIR_RELPATH);
|
|
2140
|
+
const cardDir = join(root, card);
|
|
2141
|
+
const parseLines = (path) =>
|
|
2142
|
+
readFileSync(path, "utf8")
|
|
2143
|
+
.split("\n")
|
|
2144
|
+
.filter((l) => l !== "")
|
|
2145
|
+
.map((l) => JSON.parse(l));
|
|
2146
|
+
if (run === null) {
|
|
2147
|
+
if (!existsSync(cardDir)) throw new Error(`no history for card ${card}`);
|
|
2148
|
+
const runs = readdirSync(cardDir)
|
|
2149
|
+
.filter((f) => f.endsWith(".jsonl"))
|
|
2150
|
+
.map((f) => f.slice(0, -".jsonl".length))
|
|
2151
|
+
.sort()
|
|
2152
|
+
.map((r) => {
|
|
2153
|
+
const entries = parseLines(join(cardDir, r + ".jsonl")).filter((e) => e.kind !== "cap");
|
|
2154
|
+
return { run: r, entries: entries.length, tokens: entries.reduce((n, e) => n + lessonTokens(e.text ?? ""), 0) };
|
|
2155
|
+
});
|
|
2156
|
+
return { card, runs };
|
|
2157
|
+
}
|
|
2158
|
+
const file = join(cardDir, run + ".jsonl");
|
|
2159
|
+
if (!existsSync(file)) throw new Error(`no history for ${card}/${run}`);
|
|
2160
|
+
const lines = parseLines(file);
|
|
2161
|
+
if (line !== null) {
|
|
2162
|
+
if (line > lines.length) throw new Error(`no line ${line} in ${card}/${run} (${lines.length} lines)`);
|
|
2163
|
+
const e = lines[line - 1];
|
|
2164
|
+
return { ...e, line, tokens: lessonTokens(e.text ?? "") };
|
|
2165
|
+
}
|
|
2166
|
+
const entries = lines.map((e, i) => {
|
|
2167
|
+
if (e.kind === "cap") return { line: i + 1, kind: "cap", dropped: e.dropped, maxRunBytes: e.maxRunBytes };
|
|
2168
|
+
return { line: i + 1, kind: e.kind, seat: e.seat, round: e.round, source: e.source, bytes: e.bytes, truncated: e.truncated, redacted: e.redacted, tokens: lessonTokens(e.text ?? "") };
|
|
2169
|
+
});
|
|
2170
|
+
return { card, run, tokens: entries.reduce((n, e) => n + (e.tokens ?? 0), 0), entries };
|
|
2171
|
+
}
|
|
2172
|
+
|
|
1606
2173
|
const OUTCOME_COLUMNS = [
|
|
1607
2174
|
"recorded", "run", "run_id", "report_hash", "report_path", "card", "task", "spec_hash", "files", "size", "shape",
|
|
1608
2175
|
"models", "implemented", "verified", "reviewed", "review_issues", "review_issue_count", "adversary_verdict",
|
|
1609
2176
|
"adversary_blocked", "adversary_source", "blocks", "passes", "fix_passes", "stop_reason", "stop_class", "in_scope",
|
|
1610
2177
|
"budget_spent", "tokens", "usd", "codex_usd", "commit_sha", "track", "wall_clock", "gate", "note", "partial",
|
|
1611
|
-
"harness_commit", "config_hash", "class", "arm", "assigned_by", "explore_probability", "backfilled",
|
|
2178
|
+
"harness_commit", "config_hash", "class", "arm", "assigned_by", "explore_probability", "backfilled", "memory_config",
|
|
1612
2179
|
];
|
|
1613
2180
|
|
|
1614
2181
|
const JSON_COLUMNS = new Set(["files", "models", "review_issues", "blocks", "budget_spent", "tokens"]);
|
|
1615
2182
|
|
|
1616
|
-
// The
|
|
2183
|
+
// The eight condition columns (seven from card outcomes-condition-columns, plus memory_config) upsert atomically, all eight together, keyed on
|
|
1617
2184
|
// whether the *new* row carries a harness_commit — never column-by-column (a plain per-column COALESCE let a
|
|
1618
2185
|
// re-record with no condition zero out an existing stamp's `backfilled` flag while keeping its old
|
|
1619
2186
|
// harness_commit, mixing half of one stamp with half of another, since `backfilled` defaults to 0, not null,
|
|
1620
2187
|
// when a caller omits it). When `excluded.harness_commit` is null (this row carries no condition), every one of
|
|
1621
|
-
// the
|
|
2188
|
+
// the eight columns keeps the existing row's value untouched; otherwise every one of the eight takes the new
|
|
1622
2189
|
// row's value, including `backfilled` (a fresh `condition open`/`record --hand` stamp legitimately replaces an
|
|
1623
2190
|
// older backfilled placeholder with backfilled 0). This is what keeps a `condition backfill` stamp — or any
|
|
1624
2191
|
// other already-recorded stamp — intact across a later re-record for the same run+task that carries none.
|
|
1625
|
-
const CONDITION_COLUMNS = new Set(["harness_commit", "config_hash", "class", "arm", "assigned_by", "explore_probability", "backfilled"]);
|
|
2192
|
+
const CONDITION_COLUMNS = new Set(["harness_commit", "config_hash", "class", "arm", "assigned_by", "explore_probability", "backfilled", "memory_config"]);
|
|
2193
|
+
|
|
2194
|
+
// passes and fix_passes upsert as a pair keyed on the new row's passes (card memory-record-keeps-rounds): a
|
|
2195
|
+
// re-record whose row carries no passes (a hand row) keeps the measured pair `reflect --hand --rounds` wrote.
|
|
2196
|
+
const ROUND_COLUMNS = new Set(["passes", "fix_passes"]);
|
|
1626
2197
|
|
|
1627
2198
|
export function recordOutcomes(m, rows) {
|
|
1628
2199
|
let inserted = 0;
|
|
1629
2200
|
let updated = 0;
|
|
1630
|
-
m.exec("BEGIN");
|
|
2201
|
+
m.exec("BEGIN IMMEDIATE");
|
|
1631
2202
|
try {
|
|
1632
2203
|
for (const row of rows) {
|
|
1633
2204
|
const existing = m.prepare("SELECT id FROM outcomes WHERE run = ? AND task = ?").get(row.run, row.task);
|
|
@@ -1642,7 +2213,9 @@ export function recordOutcomes(m, rows) {
|
|
|
1642
2213
|
.map((c) =>
|
|
1643
2214
|
CONDITION_COLUMNS.has(c)
|
|
1644
2215
|
? `${c} = CASE WHEN excluded.harness_commit IS NULL THEN outcomes.${c} ELSE excluded.${c} END`
|
|
1645
|
-
:
|
|
2216
|
+
: ROUND_COLUMNS.has(c)
|
|
2217
|
+
? `${c} = CASE WHEN excluded.passes IS NULL THEN outcomes.${c} ELSE excluded.${c} END`
|
|
2218
|
+
: `${c} = excluded.${c}`
|
|
1646
2219
|
)
|
|
1647
2220
|
.join(", ");
|
|
1648
2221
|
m.prepare(
|
|
@@ -1680,6 +2253,58 @@ export function listOutcomes(m, { run = null, task = null } = {}) {
|
|
|
1680
2253
|
});
|
|
1681
2254
|
}
|
|
1682
2255
|
|
|
2256
|
+
// Compare outcome arms by memory configuration (card memory-outcome-arms, ADR 0019 Measurement). Reads outcomes and
|
|
2257
|
+
// reflections only. One arm per distinct memory_config, ordered by the arm's earliest recorded row; rows with no
|
|
2258
|
+
// memory_config are counted in `unstamped` and belong to no arm. landings are rows with a commit, stops rows with
|
|
2259
|
+
// no commit and either a stop_class or a `stop:` run (a stop row with no typed cause). An arm is compared (`ready`) once it has `min` landings; until then its four metrics
|
|
2260
|
+
// are null: landRate = landings / (landings + stops); fixRounds = mean fix_passes over landings that have one
|
|
2261
|
+
// (n of them; mean null when n is 0); harmfulPerLanding = harmful marks of the reflections joined to a landing
|
|
2262
|
+
// by card and commit_sha, per landing; costPerLanding = mean usd over priced landings (null when none).
|
|
2263
|
+
export function compareMemoryArms(m, { min = 12 } = {}) {
|
|
2264
|
+
const rows = m.prepare("SELECT * FROM outcomes ORDER BY recorded ASC, id ASC").all();
|
|
2265
|
+
const byConfig = new Map();
|
|
2266
|
+
let unstamped = 0;
|
|
2267
|
+
for (const row of rows) {
|
|
2268
|
+
if (row.memory_config === null || row.memory_config === undefined) {
|
|
2269
|
+
unstamped++;
|
|
2270
|
+
continue;
|
|
2271
|
+
}
|
|
2272
|
+
if (!byConfig.has(row.memory_config)) byConfig.set(row.memory_config, []);
|
|
2273
|
+
byConfig.get(row.memory_config).push(row);
|
|
2274
|
+
}
|
|
2275
|
+
const reflections = m.prepare("SELECT card, commit_sha, counters FROM reflections WHERE card IS NOT NULL AND commit_sha IS NOT NULL").all();
|
|
2276
|
+
const harmfulOf = (r) => {
|
|
2277
|
+
try {
|
|
2278
|
+
const h = JSON.parse(r.counters).harmful;
|
|
2279
|
+
return Array.isArray(h) ? h.length : 0;
|
|
2280
|
+
} catch {
|
|
2281
|
+
return 0;
|
|
2282
|
+
}
|
|
2283
|
+
};
|
|
2284
|
+
const arms = [];
|
|
2285
|
+
for (const [config, armRows] of byConfig) {
|
|
2286
|
+
const landed = armRows.filter((r) => r.commit_sha !== null && r.commit_sha !== undefined);
|
|
2287
|
+
const stops = armRows.filter((r) => (r.commit_sha === null || r.commit_sha === undefined) && ((r.stop_class !== null && r.stop_class !== undefined) || (typeof r.run === "string" && r.run.startsWith("stop:")))).length;
|
|
2288
|
+
const landings = landed.length;
|
|
2289
|
+
const ready = landings >= min;
|
|
2290
|
+
const arm = { config, landings, stops, ready, landRate: null, fixRounds: null, harmfulPerLanding: null, costPerLanding: null, priced: null };
|
|
2291
|
+
if (ready) {
|
|
2292
|
+
arm.landRate = landings / (landings + stops);
|
|
2293
|
+
const fixes = landed.map((r) => r.fix_passes).filter((v) => typeof v === "number");
|
|
2294
|
+
arm.fixRounds = { mean: fixes.length ? fixes.reduce((a, b) => a + b, 0) / fixes.length : null, n: fixes.length };
|
|
2295
|
+
const keys = new Set(landed.map((r) => `${r.task}\u0000${r.commit_sha}`));
|
|
2296
|
+
let harmful = 0;
|
|
2297
|
+
for (const r of reflections) if (keys.has(`${r.card}\u0000${r.commit_sha}`)) harmful += harmfulOf(r);
|
|
2298
|
+
arm.harmfulPerLanding = harmful / landings;
|
|
2299
|
+
const usds = landed.map((r) => r.usd).filter((v) => typeof v === "number");
|
|
2300
|
+
arm.priced = usds.length;
|
|
2301
|
+
arm.costPerLanding = usds.length ? usds.reduce((a, b) => a + b, 0) / usds.length : null;
|
|
2302
|
+
}
|
|
2303
|
+
arms.push(arm);
|
|
2304
|
+
}
|
|
2305
|
+
return { min, unstamped, arms, comparable: arms.filter((a) => a.ready).length >= 2 };
|
|
2306
|
+
}
|
|
2307
|
+
|
|
1683
2308
|
// --- reflect (card memory-reflect) ------------------------------------------------------------------------------
|
|
1684
2309
|
// A cheap-row LLM pass, run after `memory.mjs record`, that proposes at most 3 lessons from one report and bumps
|
|
1685
2310
|
// helpful/harmful counters on lessons the report's tasks used. Everything here is pure and unit-testable except
|
|
@@ -1704,6 +2329,11 @@ export const REFLECT_SCHEMA = {
|
|
|
1704
2329
|
task: { type: "string" },
|
|
1705
2330
|
scope: { type: "array", items: { type: "string" } },
|
|
1706
2331
|
notDerivableBecause: { type: "string" },
|
|
2332
|
+
// Optional (card memory-reflect-supersede, ADR 0019 card 8): the id of one existing lesson listed in the
|
|
2333
|
+
// prompt that this proposal repeats (nothing new is written, that lesson's seen count rises) or
|
|
2334
|
+
// supersedes (it contradicts or corrects it). Never both; code checks the id is live and listed.
|
|
2335
|
+
repeats: { type: "string" },
|
|
2336
|
+
supersedes: { type: "string" },
|
|
1707
2337
|
// Optional (card reflect-offers-claude-md): the CLAUDE.md bullet this lesson would be, only when it is a
|
|
1708
2338
|
// rule a future session should follow.
|
|
1709
2339
|
claudeMd: {
|
|
@@ -1736,6 +2366,13 @@ function reflectionRowToObject(row) {
|
|
|
1736
2366
|
};
|
|
1737
2367
|
}
|
|
1738
2368
|
|
|
2369
|
+
// The archive rule (card memory-feedback-rank, ADR 0019): harmful at least min and at least helpful. Archived
|
|
2370
|
+
// rows are kept (never deleted) and hidden from recall; ARCHIVE_HARMFUL_MIN is the bench-checked threshold.
|
|
2371
|
+
export const ARCHIVE_HARMFUL_MIN = 2;
|
|
2372
|
+
export function meetsArchiveRule({ harmful, helpful }, min = ARCHIVE_HARMFUL_MIN) {
|
|
2373
|
+
return harmful >= min && harmful >= helpful;
|
|
2374
|
+
}
|
|
2375
|
+
|
|
1739
2376
|
// Deterministic, no LLM. harmful is an explicit mark (card reflect-harmful-irrelevant-lessons): only an id in
|
|
1740
2377
|
// `harmfulIds` (the lead's judgement that the lesson misled the work) gets harmful +1, once however often it is
|
|
1741
2378
|
// named or used; a real block alone bumps nothing. For each task whose memoryUsed is non-empty, a real block earns
|
|
@@ -1825,6 +2462,12 @@ export function buildReflectPrompt(report, { blocks = [], plan = null, existingL
|
|
|
1825
2462
|
);
|
|
1826
2463
|
lines.push("- notDerivableBecause must say why the lesson is not something a reader could already get by reading the code; never restate a fact already sitting verbatim in a file.");
|
|
1827
2464
|
lines.push("- never propose a lesson whose text is already covered by one of the existing lessons listed below.");
|
|
2465
|
+
lines.push(
|
|
2466
|
+
"- supersedes is optional: set it to the id of one existing lesson listed below when your new lesson contradicts or corrects that lesson. The old lesson is then retired in favour of yours."
|
|
2467
|
+
);
|
|
2468
|
+
lines.push(
|
|
2469
|
+
"- repeats is optional: set it to the id of one existing lesson listed below when your lesson says the same thing as that lesson. Nothing new is written and that lesson's seen count rises. Never set both repeats and supersedes."
|
|
2470
|
+
);
|
|
1828
2471
|
lines.push("- propose at most 3 lessons; an empty list is a perfectly good answer when nothing here is worth remembering.");
|
|
1829
2472
|
lines.push("- a task marked partial=true below ran out of context, not verification; propose no lesson that blames its code or its tests for that.");
|
|
1830
2473
|
const headings = dir ? claudeMdHeadings(dir) : [];
|
|
@@ -1854,7 +2497,7 @@ export function buildReflectPrompt(report, { blocks = [], plan = null, existingL
|
|
|
1854
2497
|
}
|
|
1855
2498
|
}
|
|
1856
2499
|
lines.push("");
|
|
1857
|
-
lines.push("Existing lessons (never propose a near-duplicate of one of these):");
|
|
2500
|
+
lines.push("Existing lessons (never propose a near-duplicate of one of these; name one in repeats or supersedes when it applies):");
|
|
1858
2501
|
const shownLessons = mostRecentLessons(existingLessons, REFLECT_PROMPT_LESSON_CAP);
|
|
1859
2502
|
if (existingLessons.length > REFLECT_PROMPT_LESSON_CAP) {
|
|
1860
2503
|
lines.push(`(showing the ${REFLECT_PROMPT_LESSON_CAP} most recently created/confirmed of ${existingLessons.length} total; older lessons are not shown)`);
|
|
@@ -1967,17 +2610,20 @@ export function jaccard(a, b) {
|
|
|
1967
2610
|
// - Jaccard similarity over lower-cased word tokens (length >= 3) at or above REFLECT_DUPLICATE_JACCARD against
|
|
1968
2611
|
// every non-superseded existing lesson or a row appended earlier in this same pass (`existingLessons` and
|
|
1969
2612
|
// `appendedRows`), so two near-identical proposals in one pass only ever add the first.
|
|
1970
|
-
// Returns { duplicate: <live lesson row> } (
|
|
1971
|
-
|
|
2613
|
+
// Returns { duplicate: <live lesson row> } (an identical hash: a repeat), { similar: <lesson row> } (word overlap,
|
|
2614
|
+
// never merged), { rejected: <superseded lesson row> }, or null (not a dup). skipSimilarity: a model judgement
|
|
2615
|
+
// (supersedes) was given, so word overlap is not consulted.
|
|
2616
|
+
function findReflectDuplicate(m, proposal, { existingLessons, appendedRows, skipSimilarity = false }) {
|
|
1972
2617
|
const proposalId = computeLessonId({ kind: proposal.kind, citation: proposal.citation || null, text: proposal.text });
|
|
1973
2618
|
const collision = getLesson(m, proposalId);
|
|
1974
2619
|
if (collision) return collision.superseded_by ? { rejected: collision } : { duplicate: collision };
|
|
1975
2620
|
for (const row of appendedRows) {
|
|
1976
2621
|
if (row.id === proposalId) return { duplicate: row };
|
|
1977
2622
|
}
|
|
2623
|
+
if (skipSimilarity) return null;
|
|
1978
2624
|
const tokens = reflectTokens(proposal.text);
|
|
1979
2625
|
for (const lesson of [...existingLessons, ...appendedRows]) {
|
|
1980
|
-
if (jaccard(tokens, reflectTokens(lesson.text)) >= REFLECT_DUPLICATE_JACCARD) return {
|
|
2626
|
+
if (jaccard(tokens, reflectTokens(lesson.text)) >= REFLECT_DUPLICATE_JACCARD) return { similar: lesson };
|
|
1981
2627
|
}
|
|
1982
2628
|
return null;
|
|
1983
2629
|
}
|
|
@@ -1985,7 +2631,7 @@ function findReflectDuplicate(m, proposal, { existingLessons, appendedRows }) {
|
|
|
1985
2631
|
// Filters C.1-C.4, in order. C.5 (at most 3 appended) is enforced by the caller, but only against a proposal
|
|
1986
2632
|
// this function has already said ok:true and not a duplicate — a would-be duplicate past the cap still bumps.
|
|
1987
2633
|
// Returns { ok:true }, { ok:true, duplicate:<lesson row> } (bump instead of append), or { ok:false, reason }.
|
|
1988
|
-
function evaluateReflectProposal(m, report, blocks, proposal, { dir, existingLessons, appendedRows }) {
|
|
2634
|
+
function evaluateReflectProposal(m, report, blocks, proposal, { dir, existingLessons, appendedRows, judgement = null }) {
|
|
1989
2635
|
if (!proposal || typeof proposal !== "object") return { ok: false, reason: "proposal is not an object" };
|
|
1990
2636
|
const { kind, text, citation, task, notDerivableBecause } = proposal;
|
|
1991
2637
|
if (kind !== "pitfall" && kind !== "pattern") return { ok: false, reason: `kind must be "pitfall" or "pattern", got ${JSON.stringify(kind)}` };
|
|
@@ -2002,16 +2648,24 @@ function evaluateReflectProposal(m, report, blocks, proposal, { dir, existingLes
|
|
|
2002
2648
|
if (typeof notDerivableBecause !== "string" || !notDerivableBecause.trim()) return { ok: false, reason: "notDerivableBecause is empty" };
|
|
2003
2649
|
const derivableCheck = checkReflectDerivable(proposal, dir);
|
|
2004
2650
|
if (!derivableCheck.ok) return derivableCheck;
|
|
2005
|
-
|
|
2651
|
+
// A valid repeats judgement: nothing new is written, the named lesson's seen count rises.
|
|
2652
|
+
if (judgement && judgement.kind === "repeats") return { ok: true, duplicate: judgement.target };
|
|
2653
|
+
const dup = findReflectDuplicate(m, proposal, { existingLessons, appendedRows, skipSimilarity: Boolean(judgement) });
|
|
2006
2654
|
if (dup && dup.rejected) return { ok: false, reason: `addLesson id collision with a superseded lesson ${dup.rejected.id}` };
|
|
2655
|
+
// An identical hash is a code fact, not a similarity score: a repeat of the colliding lesson.
|
|
2007
2656
|
if (dup && dup.duplicate) return { ok: true, duplicate: dup.duplicate };
|
|
2657
|
+
// The similarity score never merges (ADR 0019 card 8): with no judgement a near-duplicate is dropped.
|
|
2658
|
+
if (dup && dup.similar) {
|
|
2659
|
+
return { ok: false, reason: `near-duplicate of ${dup.similar.id} (word overlap >= ${REFLECT_DUPLICATE_JACCARD}); no repeats or supersedes judgement` };
|
|
2660
|
+
}
|
|
2661
|
+
if (judgement && judgement.kind === "supersedes") return { ok: true, supersedes: judgement.target };
|
|
2008
2662
|
return { ok: true };
|
|
2009
2663
|
}
|
|
2010
2664
|
|
|
2011
2665
|
// Cost (D): prefer the real per-message transcript at <claudeDir>/projects/<projectSlug(dir)>/<sessionId>.jsonl,
|
|
2012
2666
|
// same as cost.mjs prices every other agent; fall back to the envelope's own total_cost_usd (a client-side
|
|
2013
2667
|
// estimate) when the transcript is missing or its model is unpriced; null with no source when neither is usable.
|
|
2014
|
-
function computeReflectCost({ dir, claudeDir, sessionId, totalCostUsd }) {
|
|
2668
|
+
export function computeReflectCost({ dir, claudeDir, sessionId, totalCostUsd }) {
|
|
2015
2669
|
const resolvedClaudeDir = claudeDir || join(homedir(), ".claude");
|
|
2016
2670
|
if (sessionId) {
|
|
2017
2671
|
const file = join(resolvedClaudeDir, "projects", projectSlug(dir), `${sessionId}.jsonl`);
|
|
@@ -2133,7 +2787,7 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2133
2787
|
now = nowIso(),
|
|
2134
2788
|
} = opts;
|
|
2135
2789
|
if (!reportHash) throw new Error("runReflect requires a reportHash");
|
|
2136
|
-
if (typeof propose !== "function") throw new Error("runReflect requires a propose function");
|
|
2790
|
+
if (typeof propose !== "function" && !dryRun) throw new Error("runReflect requires a propose function");
|
|
2137
2791
|
|
|
2138
2792
|
const existing = m.prepare("SELECT * FROM reflections WHERE report_hash = ?").get(reportHash);
|
|
2139
2793
|
if (existing) return { alreadyReflected: true, row: reflectionRowToObject(existing) };
|
|
@@ -2160,12 +2814,23 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2160
2814
|
}
|
|
2161
2815
|
|
|
2162
2816
|
let counters;
|
|
2817
|
+
const archived = [];
|
|
2163
2818
|
if (dryRun) {
|
|
2164
2819
|
counters = computeReflectCounters(m, blocks, report, { apply: false, harmfulIds });
|
|
2165
2820
|
} else {
|
|
2166
|
-
m.exec("BEGIN");
|
|
2821
|
+
m.exec("BEGIN IMMEDIATE");
|
|
2167
2822
|
try {
|
|
2168
2823
|
counters = computeReflectCounters(m, blocks, report, { apply: true, harmfulIds });
|
|
2824
|
+
for (const id of counters.harmful) {
|
|
2825
|
+
const row = getLesson(m, id);
|
|
2826
|
+
if (!row || row.archived || !meetsArchiveRule(row)) continue;
|
|
2827
|
+
m.prepare("UPDATE lessons SET archived = ?, archived_reason = ? WHERE id = ?").run(
|
|
2828
|
+
now,
|
|
2829
|
+
`harmful ${row.harmful}, helpful ${row.helpful}`,
|
|
2830
|
+
id
|
|
2831
|
+
);
|
|
2832
|
+
archived.push(id);
|
|
2833
|
+
}
|
|
2169
2834
|
m.prepare(
|
|
2170
2835
|
`INSERT INTO reflections (report_hash, recorded, run, card, commit_sha, model, counters, proposed, appended, bumped, rejected, usd, usd_source, session_id)
|
|
2171
2836
|
VALUES (?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, NULL, NULL, NULL)`
|
|
@@ -2180,6 +2845,30 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2180
2845
|
const existingLessons = listLessons(m);
|
|
2181
2846
|
const prompt = buildReflectPrompt(report, { blocks, plan, existingLessons, dir });
|
|
2182
2847
|
|
|
2848
|
+
// A dry run with no propose makes no model call: it reports the counters it would credit and the prompt it would send, and spends nothing.
|
|
2849
|
+
if (dryRun && typeof propose !== "function") {
|
|
2850
|
+
return {
|
|
2851
|
+
ok: true,
|
|
2852
|
+
dryRun: true,
|
|
2853
|
+
reportHash,
|
|
2854
|
+
prompt,
|
|
2855
|
+
counters,
|
|
2856
|
+
archived: [],
|
|
2857
|
+
proposed: null,
|
|
2858
|
+
appended: [],
|
|
2859
|
+
bumped: [],
|
|
2860
|
+
rejected: [],
|
|
2861
|
+
claudeMd: [],
|
|
2862
|
+
claudeMdDropped: [],
|
|
2863
|
+
notes: [],
|
|
2864
|
+
superseded: [],
|
|
2865
|
+
usd: 0,
|
|
2866
|
+
usdSource: "none",
|
|
2867
|
+
model,
|
|
2868
|
+
sessionId: null,
|
|
2869
|
+
};
|
|
2870
|
+
}
|
|
2871
|
+
|
|
2183
2872
|
let proposeResult = null;
|
|
2184
2873
|
let proposeError = null;
|
|
2185
2874
|
try {
|
|
@@ -2205,6 +2894,7 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2205
2894
|
dryRun,
|
|
2206
2895
|
reportHash,
|
|
2207
2896
|
counters,
|
|
2897
|
+
archived,
|
|
2208
2898
|
proposed: null,
|
|
2209
2899
|
appended: [],
|
|
2210
2900
|
bumped: [],
|
|
@@ -2213,6 +2903,8 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2213
2903
|
usdSource: null,
|
|
2214
2904
|
claudeMd: [],
|
|
2215
2905
|
claudeMdDropped: [],
|
|
2906
|
+
notes: [],
|
|
2907
|
+
superseded: [],
|
|
2216
2908
|
model,
|
|
2217
2909
|
sessionId: null,
|
|
2218
2910
|
reason: proposeError.message,
|
|
@@ -2227,13 +2919,25 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2227
2919
|
const suggestions = new Map(); // lesson id -> { lesson text, claudeMd suggestion } for appended lessons only
|
|
2228
2920
|
const bumped = [];
|
|
2229
2921
|
const rejected = [];
|
|
2922
|
+
const notes = [];
|
|
2923
|
+
const superseded = [];
|
|
2924
|
+
const shownIds = new Set(mostRecentLessons(existingLessons, REFLECT_PROMPT_LESSON_CAP).map((l) => l.id));
|
|
2925
|
+
const supersededThisPass = new Set();
|
|
2230
2926
|
for (const proposal of proposals) {
|
|
2231
|
-
const
|
|
2927
|
+
const judged = judgeReflectProposal(m, proposal, { shownIds, supersededThisPass, notes });
|
|
2928
|
+
if (judged.reject) {
|
|
2929
|
+
rejected.push({ text: proposal && typeof proposal.text === "string" ? proposal.text : null, reason: judged.reject });
|
|
2930
|
+
continue;
|
|
2931
|
+
}
|
|
2932
|
+
const verdict = evaluateReflectProposal(m, report, blocks, proposal, { dir, existingLessons, appendedRows, judgement: judged.judgement });
|
|
2232
2933
|
if (verdict.duplicate) {
|
|
2233
|
-
// The 3-item cap (C.5) limits appends, never evaluation: a
|
|
2234
|
-
// the lesson it
|
|
2235
|
-
if (!dryRun) m.prepare("UPDATE lessons SET
|
|
2934
|
+
// The 3-item cap (C.5) limits appends, never evaluation: a repeat found past the cap still bumps seen on
|
|
2935
|
+
// the lesson it names, it is just never counted against the 3. A repeat bumps seen, never helpful.
|
|
2936
|
+
if (!dryRun) m.prepare("UPDATE lessons SET seen = seen + 1 WHERE id = ?").run(verdict.duplicate.id);
|
|
2236
2937
|
bumped.push(verdict.duplicate.id);
|
|
2938
|
+
if (judged.judgement && judged.judgement.kind === "supersedes") {
|
|
2939
|
+
notes.push(`supersedes ${judged.judgement.target.id} ignored: the proposal is an exact repeat of ${verdict.duplicate.id}`);
|
|
2940
|
+
}
|
|
2237
2941
|
continue;
|
|
2238
2942
|
}
|
|
2239
2943
|
if (!verdict.ok) {
|
|
@@ -2248,6 +2952,10 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2248
2952
|
const dryId = computeLessonId({ kind: proposal.kind, citation: proposal.citation || null, text: proposal.text });
|
|
2249
2953
|
appendedRows.push({ id: dryId, text: proposal.text, kind: proposal.kind });
|
|
2250
2954
|
suggestions.set(dryId, { text: proposal.text, claudeMd: proposal.claudeMd });
|
|
2955
|
+
if (verdict.supersedes) {
|
|
2956
|
+
superseded.push({ old: verdict.supersedes.id, new: dryId });
|
|
2957
|
+
supersededThisPass.add(verdict.supersedes.id);
|
|
2958
|
+
}
|
|
2251
2959
|
continue;
|
|
2252
2960
|
}
|
|
2253
2961
|
const reportTask = findReportTask(report, proposal.task);
|
|
@@ -2263,6 +2971,15 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2263
2971
|
});
|
|
2264
2972
|
appendedRows.push(row);
|
|
2265
2973
|
suggestions.set(row.id, { text: proposal.text, claudeMd: proposal.claudeMd });
|
|
2974
|
+
if (verdict.supersedes) {
|
|
2975
|
+
try {
|
|
2976
|
+
supersedeLesson(m, verdict.supersedes.id, row.id);
|
|
2977
|
+
superseded.push({ old: verdict.supersedes.id, new: row.id });
|
|
2978
|
+
supersededThisPass.add(verdict.supersedes.id);
|
|
2979
|
+
} catch (err) {
|
|
2980
|
+
notes.push(`supersedes ${verdict.supersedes.id} not applied: ${err && err.message ? err.message : String(err)}`);
|
|
2981
|
+
}
|
|
2982
|
+
}
|
|
2266
2983
|
}
|
|
2267
2984
|
|
|
2268
2985
|
let claudeMd = [];
|
|
@@ -2298,15 +3015,659 @@ export async function runReflect(m, report, opts = {}) {
|
|
|
2298
3015
|
dryRun,
|
|
2299
3016
|
reportHash,
|
|
2300
3017
|
counters,
|
|
3018
|
+
archived,
|
|
2301
3019
|
proposed: proposals.length,
|
|
2302
3020
|
appended: appendedRows.map((r) => r.id),
|
|
2303
3021
|
bumped,
|
|
2304
3022
|
rejected,
|
|
2305
3023
|
claudeMd,
|
|
2306
3024
|
claudeMdDropped,
|
|
3025
|
+
notes,
|
|
3026
|
+
superseded,
|
|
2307
3027
|
usd: cost.usd,
|
|
2308
3028
|
usdSource: cost.usdSource,
|
|
2309
3029
|
model,
|
|
2310
3030
|
sessionId,
|
|
2311
3031
|
};
|
|
2312
3032
|
}
|
|
3033
|
+
|
|
3034
|
+
// The judgement fields of one reflect proposal (card memory-reflect-supersede). A target is valid only when it is
|
|
3035
|
+
// one of the lessons the prompt listed, is live, and was not superseded earlier in this pass. An invalid id is
|
|
3036
|
+
// ignored with a note and the proposal goes on as if the field were absent. Both fields set is a rejection.
|
|
3037
|
+
function judgeReflectProposal(m, proposal, { shownIds, supersededThisPass, notes }) {
|
|
3038
|
+
const field = (name) => (proposal && typeof proposal[name] === "string" && proposal[name].trim() ? proposal[name].trim() : null);
|
|
3039
|
+
const repeatsId = field("repeats");
|
|
3040
|
+
const supersedesId = field("supersedes");
|
|
3041
|
+
if (repeatsId && supersedesId) return { reject: "repeats and supersedes are both set; name one lesson in one field" };
|
|
3042
|
+
const check = (name, id) => {
|
|
3043
|
+
const row = getLesson(m, id);
|
|
3044
|
+
let why = null;
|
|
3045
|
+
if (!row) why = "no such lesson";
|
|
3046
|
+
else if (row.superseded_by || supersededThisPass.has(id)) why = "already superseded";
|
|
3047
|
+
else if (!shownIds.has(id)) why = "not listed in the prompt";
|
|
3048
|
+
if (why) {
|
|
3049
|
+
notes.push(`${name} ${id} ignored: ${why}`);
|
|
3050
|
+
return null;
|
|
3051
|
+
}
|
|
3052
|
+
return row;
|
|
3053
|
+
};
|
|
3054
|
+
if (repeatsId) {
|
|
3055
|
+
const target = check("repeats", repeatsId);
|
|
3056
|
+
return { judgement: target ? { kind: "repeats", target } : null };
|
|
3057
|
+
}
|
|
3058
|
+
if (supersedesId) {
|
|
3059
|
+
const target = check("supersedes", supersedesId);
|
|
3060
|
+
return { judgement: target ? { kind: "supersedes", target } : null };
|
|
3061
|
+
}
|
|
3062
|
+
return { judgement: null };
|
|
3063
|
+
}
|
|
3064
|
+
|
|
3065
|
+
// --- failure lessons (card memory-failure-lessons, ADR 0019 lines 39-41 and 60) -------------------------------------
|
|
3066
|
+
// Reflection over the failure slice of the run history: stopped reports, blocks classed `real`, and denials. A
|
|
3067
|
+
// pitfall cites the exact history line (history:<card>/<run>:<line>); a repeat of a pitfall is the model's `repeats`
|
|
3068
|
+
// judgement and lands in an append-only occurrence ledger (never in helpful, never from word similarity), which
|
|
3069
|
+
// learn.mjs counts by distinct cards and sessions.
|
|
3070
|
+
|
|
3071
|
+
export const PITFALL_LEDGER_RELPATH = ".doug/.state/memory/pitfalls.jsonl";
|
|
3072
|
+
|
|
3073
|
+
const FAILURE_ITEM_TEXT_MAX = 4000;
|
|
3074
|
+
const FAILURE_PROMPT_MAX = 16000;
|
|
3075
|
+
const FAILURE_TAIL_MAX = 1000;
|
|
3076
|
+
const FAILURE_APPEND_MAX = 3;
|
|
3077
|
+
|
|
3078
|
+
function parseJsonObject(text) {
|
|
3079
|
+
try {
|
|
3080
|
+
const v = JSON.parse(text);
|
|
3081
|
+
return v !== null && typeof v === "object" && !Array.isArray(v) ? v : null;
|
|
3082
|
+
} catch {
|
|
3083
|
+
return null;
|
|
3084
|
+
}
|
|
3085
|
+
}
|
|
3086
|
+
|
|
3087
|
+
function failureItemText(kind, v, raw) {
|
|
3088
|
+
const parts = [];
|
|
3089
|
+
if (kind === "stop") {
|
|
3090
|
+
for (const k of ["reason", "stoppedAt", "detail", "error"]) if (typeof v[k] === "string" && v[k]) parts.push(`${k}: ${v[k]}`);
|
|
3091
|
+
if (!parts.length) parts.push(`report: ${raw}`);
|
|
3092
|
+
} else if (kind === "block") {
|
|
3093
|
+
if (typeof v.reason === "string") parts.push(`reason: ${v.reason}`);
|
|
3094
|
+
for (const f of Array.isArray(v.failures) ? v.failures : []) {
|
|
3095
|
+
if (!f || typeof f !== "object") continue;
|
|
3096
|
+
const tail = typeof f.tail === "string" ? f.tail.slice(-FAILURE_TAIL_MAX) : "";
|
|
3097
|
+
parts.push(`command: ${f.command ?? ""} (exit ${f.exit ?? "?"}${f.timedOut ? ", timed out" : ""})${tail ? `\n${tail}` : ""}`);
|
|
3098
|
+
}
|
|
3099
|
+
} else {
|
|
3100
|
+
parts.push(`tool: ${v.tool ?? ""}`);
|
|
3101
|
+
if (v.detail !== undefined) parts.push(`detail: ${typeof v.detail === "string" ? v.detail : JSON.stringify(v.detail)}`);
|
|
3102
|
+
}
|
|
3103
|
+
return parts.join("\n");
|
|
3104
|
+
}
|
|
3105
|
+
|
|
3106
|
+
// The failure slice of one card's run history; never throws. See ADR 0019 lines 40 and 50: a block that is not
|
|
3107
|
+
// `real` is counted in `hidden` by class, and an entry that cannot be parsed in `unreadable`.
|
|
3108
|
+
export function failureSlice(dir, card, { session = null } = {}) {
|
|
3109
|
+
const out = { items: [], hidden: {}, unreadable: 0 };
|
|
3110
|
+
try {
|
|
3111
|
+
const cardName = historySafe(card);
|
|
3112
|
+
const cardDir = join(dir, HISTORY_DIR_RELPATH, cardName);
|
|
3113
|
+
if (!existsSync(cardDir)) return out;
|
|
3114
|
+
const files = readdirSync(cardDir).filter((f) => f.endsWith(".jsonl")).sort();
|
|
3115
|
+
for (const file of files) {
|
|
3116
|
+
const run = file.slice(0, -".jsonl".length);
|
|
3117
|
+
let text;
|
|
3118
|
+
try {
|
|
3119
|
+
text = readFileSync(join(cardDir, file), "utf8");
|
|
3120
|
+
} catch {
|
|
3121
|
+
continue;
|
|
3122
|
+
}
|
|
3123
|
+
const entries = [];
|
|
3124
|
+
for (const raw of text.split("\n")) {
|
|
3125
|
+
if (raw === "") continue;
|
|
3126
|
+
const e = parseJsonObject(raw);
|
|
3127
|
+
entries.push(e && typeof e.kind === "string" ? { kind: e.kind, text: typeof e.text === "string" ? e.text : "" } : null);
|
|
3128
|
+
}
|
|
3129
|
+
const parsed = entries.map((e) => (e && (e.kind === "report" || e.kind === "block" || e.kind === "denial") ? parseJsonObject(e.text) : null));
|
|
3130
|
+
const sessionOf = (v) => (v && typeof v.session === "string" && v.session !== "" ? v.session : null);
|
|
3131
|
+
const runSession = parsed.map(sessionOf).find((s) => s !== null) ?? null;
|
|
3132
|
+
entries.forEach((e, i) => {
|
|
3133
|
+
if (!e) return;
|
|
3134
|
+
const v = parsed[i];
|
|
3135
|
+
let kind = null;
|
|
3136
|
+
if (e.kind === "report") {
|
|
3137
|
+
if (!v) return void out.unreadable++;
|
|
3138
|
+
if (v.outcome === "stopped" || (v.ok === false && !v.paused)) kind = "stop";
|
|
3139
|
+
} else if (e.kind === "block") {
|
|
3140
|
+
if (!v) return void out.unreadable++;
|
|
3141
|
+
if (v.class === "real") kind = "block";
|
|
3142
|
+
else {
|
|
3143
|
+
const cls = typeof v.class === "string" && v.class ? v.class : "unclassified";
|
|
3144
|
+
out.hidden[cls] = (out.hidden[cls] ?? 0) + 1;
|
|
3145
|
+
}
|
|
3146
|
+
} else if (e.kind === "denial") kind = "denial";
|
|
3147
|
+
if (!kind) return;
|
|
3148
|
+
const body = v ?? {};
|
|
3149
|
+
out.items.push({
|
|
3150
|
+
ref: `history:${cardName}/${run}:${i + 1}`,
|
|
3151
|
+
kind,
|
|
3152
|
+
card: cardName,
|
|
3153
|
+
run,
|
|
3154
|
+
line: i + 1,
|
|
3155
|
+
class: kind === "block" ? v.class : null,
|
|
3156
|
+
files: kind === "block" && Array.isArray(v.files) ? v.files.filter((f) => typeof f === "string" && f !== "") : [],
|
|
3157
|
+
session: sessionOf(v) ?? runSession ?? (typeof session === "string" && session !== "" ? session : null),
|
|
3158
|
+
text: failureItemText(kind, body, e.text),
|
|
3159
|
+
});
|
|
3160
|
+
});
|
|
3161
|
+
}
|
|
3162
|
+
} catch {
|
|
3163
|
+
// fail open: whatever was read so far
|
|
3164
|
+
}
|
|
3165
|
+
return out;
|
|
3166
|
+
}
|
|
3167
|
+
|
|
3168
|
+
// Occurrence ledger: one JSON line per {lesson, citation} (de-duplicated), append-only.
|
|
3169
|
+
export function readPitfallOccurrences(dir) {
|
|
3170
|
+
let text;
|
|
3171
|
+
try {
|
|
3172
|
+
text = readFileSync(join(dir, PITFALL_LEDGER_RELPATH), "utf8");
|
|
3173
|
+
} catch {
|
|
3174
|
+
return [];
|
|
3175
|
+
}
|
|
3176
|
+
const seen = new Set();
|
|
3177
|
+
const out = [];
|
|
3178
|
+
for (const v of parseJsonLines(text)) {
|
|
3179
|
+
if (typeof v.lesson !== "string" || typeof v.citation !== "string") continue;
|
|
3180
|
+
const key = `${v.lesson}\u0000${v.citation}`;
|
|
3181
|
+
if (seen.has(key)) continue;
|
|
3182
|
+
seen.add(key);
|
|
3183
|
+
out.push(v);
|
|
3184
|
+
}
|
|
3185
|
+
return out;
|
|
3186
|
+
}
|
|
3187
|
+
|
|
3188
|
+
export function recordPitfallOccurrence(dir, occ) {
|
|
3189
|
+
const { lesson, citation, card = null, run = null, session = null, at = nowIso(), via, class: cls = null, files = [] } = occ ?? {};
|
|
3190
|
+
if (typeof lesson !== "string" || typeof citation !== "string") throw new Error("recordPitfallOccurrence needs a lesson and a citation");
|
|
3191
|
+
if (readPitfallOccurrences(dir).some((o) => o.lesson === lesson && o.citation === citation)) return { written: false };
|
|
3192
|
+
const path = join(dir, PITFALL_LEDGER_RELPATH);
|
|
3193
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
3194
|
+
appendFileSync(path, JSON.stringify({ lesson, citation, card, run, session, at, ...(via ? { via } : {}), class: typeof cls === "string" ? cls : null, files: Array.isArray(files) ? files : [] }) + "\n", "utf8");
|
|
3195
|
+
return { written: true };
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
export const FAILURE_REFLECT_SCHEMA = {
|
|
3199
|
+
type: "object",
|
|
3200
|
+
required: ["lessons"],
|
|
3201
|
+
properties: {
|
|
3202
|
+
lessons: {
|
|
3203
|
+
type: "array",
|
|
3204
|
+
items: {
|
|
3205
|
+
type: "object",
|
|
3206
|
+
required: ["kind", "text", "citation", "notDerivableBecause"],
|
|
3207
|
+
properties: {
|
|
3208
|
+
kind: { enum: ["pitfall"] },
|
|
3209
|
+
text: { type: "string" },
|
|
3210
|
+
citation: { type: "string" },
|
|
3211
|
+
notDerivableBecause: { type: "string" },
|
|
3212
|
+
scope: { type: "array", items: { type: "string" } },
|
|
3213
|
+
repeats: { type: "string" },
|
|
3214
|
+
},
|
|
3215
|
+
},
|
|
3216
|
+
},
|
|
3217
|
+
},
|
|
3218
|
+
};
|
|
3219
|
+
|
|
3220
|
+
export function buildFailureReflectPrompt(slice, { existingLessons = [] } = {}) {
|
|
3221
|
+
const lines = [
|
|
3222
|
+
"You are the reflection pass of a coding harness, reading the FAILURES of one card: stopped runs, gate blocks classed real, and permission denials.",
|
|
3223
|
+
"Propose at most 3 lessons of kind \"pitfall\": a short, specific rule that would have prevented the failure. Each lesson's `citation` must be exactly the ref of one item below (a history:<card>/<run>:<line> string). Do not cite anything else.",
|
|
3224
|
+
"`notDerivableBecause` says why the rule cannot be read from the repository's code or docs. Skip failures that are plain test failures with no transferable lesson.",
|
|
3225
|
+
"When an item is a repeat of an existing pitfall listed below, set `repeats` to that pitfall's id instead of inventing a new rule; do not reword an existing pitfall as a new one.",
|
|
3226
|
+
"",
|
|
3227
|
+
];
|
|
3228
|
+
const items = slice.items ?? [];
|
|
3229
|
+
lines.push(`Failure items (${items.length}):`);
|
|
3230
|
+
let used = 0;
|
|
3231
|
+
let shown = 0;
|
|
3232
|
+
for (const item of items) {
|
|
3233
|
+
let t = String(item.text ?? "");
|
|
3234
|
+
if (t.length > FAILURE_ITEM_TEXT_MAX) t = t.slice(0, FAILURE_ITEM_TEXT_MAX) + " [cut]";
|
|
3235
|
+
const block = `\n[${item.ref}] kind: ${item.kind}\n${t}`;
|
|
3236
|
+
if (used + block.length > FAILURE_PROMPT_MAX) break;
|
|
3237
|
+
lines.push(block);
|
|
3238
|
+
used += block.length;
|
|
3239
|
+
shown++;
|
|
3240
|
+
}
|
|
3241
|
+
if (shown < items.length) lines.push(`\n(${items.length - shown} more item(s) left out by the ${FAILURE_PROMPT_MAX}-character cap)`);
|
|
3242
|
+
const hidden = Object.entries(slice.hidden ?? {});
|
|
3243
|
+
if (hidden.length) lines.push(`\nBlocks not shown (not real): ${hidden.map(([k, n]) => `${k} ${n}`).join(", ")}.`);
|
|
3244
|
+
const pitfalls = existingLessons.filter((l) => l.kind === "pitfall" && !l.superseded_by);
|
|
3245
|
+
lines.push("", `Existing pitfalls (${pitfalls.length}), which \`repeats\` may name:`);
|
|
3246
|
+
for (const l of pitfalls) lines.push(`${l.id}: ${l.text}`);
|
|
3247
|
+
return lines.join("\n");
|
|
3248
|
+
}
|
|
3249
|
+
|
|
3250
|
+
export async function runFailureReflect(m, opts = {}) {
|
|
3251
|
+
const { dir = process.cwd(), card, session = null, propose, dryRun = false, model = null, claudeDir, now = nowIso() } = opts;
|
|
3252
|
+
if (typeof card !== "string" || !card) throw new Error("runFailureReflect requires a card");
|
|
3253
|
+
if (typeof propose !== "function" && !dryRun) throw new Error("runFailureReflect requires a propose function");
|
|
3254
|
+
const slice = failureSlice(dir, card, { session });
|
|
3255
|
+
const base = { items: slice.items.length, hidden: slice.hidden, unreadable: slice.unreadable };
|
|
3256
|
+
if (!slice.items.length) return { ok: true, dryRun, ...base, appended: [], repeated: [], rejected: [] };
|
|
3257
|
+
|
|
3258
|
+
const key = "failures:" + createHash("sha256").update(slice.items.map((i) => i.ref).sort().join("\n")).digest("hex");
|
|
3259
|
+
const existing = m.prepare("SELECT * FROM reflections WHERE report_hash = ?").get(key);
|
|
3260
|
+
if (existing) return { alreadyReflected: true, ...base, row: reflectionRowToObject(existing) };
|
|
3261
|
+
|
|
3262
|
+
// Code-side repeats (ADR 0019 line 60): a real block with the same class on a file a live pitfall's recorded
|
|
3263
|
+
// occurrence also names is another occurrence of that pitfall. Empty files never match.
|
|
3264
|
+
const fileRepeated = [];
|
|
3265
|
+
const sharesFile = (a, b) => Array.isArray(a) && Array.isArray(b) && a.some((f) => b.includes(f));
|
|
3266
|
+
for (const item of slice.items) {
|
|
3267
|
+
if (item.kind !== "block" || !item.files.length) continue;
|
|
3268
|
+
for (const o of readPitfallOccurrences(dir)) {
|
|
3269
|
+
if (o.class !== item.class || !sharesFile(o.files, item.files)) continue;
|
|
3270
|
+
const target = getLesson(m, o.lesson);
|
|
3271
|
+
if (!target || target.kind !== "pitfall" || target.superseded_by) continue;
|
|
3272
|
+
// List the pitfall only when this match is new: a block already recorded (a card's second stop re-reads it) adds nothing.
|
|
3273
|
+
const isNew = dryRun
|
|
3274
|
+
? !readPitfallOccurrences(dir).some((x) => x.lesson === target.id && x.citation === item.ref)
|
|
3275
|
+
: recordPitfallOccurrence(dir, { lesson: target.id, citation: item.ref, card: item.card, run: item.run, session: item.session, at: now, via: "file", class: item.class, files: item.files }).written;
|
|
3276
|
+
if (isNew && !fileRepeated.includes(target.id)) fileRepeated.push(target.id);
|
|
3277
|
+
}
|
|
3278
|
+
}
|
|
3279
|
+
|
|
3280
|
+
const prompt = buildFailureReflectPrompt(slice, { existingLessons: listLessons(m) });
|
|
3281
|
+
// A dry run with no propose makes no model call: it returns the prompt it would send and spends nothing.
|
|
3282
|
+
if (dryRun && typeof propose !== "function") {
|
|
3283
|
+
return { ok: true, dryRun: true, ...base, prompt, proposed: null, appended: [], repeated: [], rejected: [], usd: 0, usdSource: "none", model };
|
|
3284
|
+
}
|
|
3285
|
+
let result;
|
|
3286
|
+
try {
|
|
3287
|
+
result = await propose(prompt);
|
|
3288
|
+
if (!result || !Array.isArray(result.lessons)) throw new Error("the failure pass returned no lessons array");
|
|
3289
|
+
} catch (err) {
|
|
3290
|
+
return { ok: false, dryRun, ...base, appended: [], repeated: [], rejected: [{ text: null, reason: err.message }], reason: err.message };
|
|
3291
|
+
}
|
|
3292
|
+
|
|
3293
|
+
const byRef = new Map(slice.items.map((i) => [i.ref, i]));
|
|
3294
|
+
const appended = [];
|
|
3295
|
+
const appendedIds = new Set();
|
|
3296
|
+
const repeated = [...fileRepeated];
|
|
3297
|
+
const rejected = [];
|
|
3298
|
+
const occur = (lesson, item) => {
|
|
3299
|
+
if (!dryRun) recordPitfallOccurrence(dir, { lesson, citation: item.ref, card: item.card, run: item.run, session: item.session, at: now, class: item.class, files: item.files });
|
|
3300
|
+
if (!repeated.includes(lesson)) repeated.push(lesson);
|
|
3301
|
+
};
|
|
3302
|
+
for (const p of result.lessons) {
|
|
3303
|
+
const text = p && typeof p.text === "string" ? p.text : null;
|
|
3304
|
+
const reject = (reason) => rejected.push({ text, reason });
|
|
3305
|
+
if (!p || typeof p !== "object") {
|
|
3306
|
+
reject("proposal is not an object");
|
|
3307
|
+
continue;
|
|
3308
|
+
}
|
|
3309
|
+
if (p.kind !== "pitfall") {
|
|
3310
|
+
reject(`kind must be "pitfall", got ${JSON.stringify(p.kind)}`);
|
|
3311
|
+
continue;
|
|
3312
|
+
}
|
|
3313
|
+
if (!text || !text.trim()) {
|
|
3314
|
+
reject("text is empty");
|
|
3315
|
+
continue;
|
|
3316
|
+
}
|
|
3317
|
+
if (typeof p.notDerivableBecause !== "string" || !p.notDerivableBecause.trim()) {
|
|
3318
|
+
reject("notDerivableBecause is empty");
|
|
3319
|
+
continue;
|
|
3320
|
+
}
|
|
3321
|
+
const item = typeof p.citation === "string" ? byRef.get(p.citation) : undefined;
|
|
3322
|
+
if (!item) {
|
|
3323
|
+
reject(`citation ${JSON.stringify(p.citation)} is not a failure item of this card's history`);
|
|
3324
|
+
continue;
|
|
3325
|
+
}
|
|
3326
|
+
const derivable = checkReflectDerivable(p, dir);
|
|
3327
|
+
if (!derivable.ok) {
|
|
3328
|
+
reject(derivable.reason);
|
|
3329
|
+
continue;
|
|
3330
|
+
}
|
|
3331
|
+
if (p.repeats !== undefined && p.repeats !== null && p.repeats !== "") {
|
|
3332
|
+
const target = typeof p.repeats === "string" ? getLesson(m, p.repeats) : null;
|
|
3333
|
+
if (!target || target.kind !== "pitfall" || target.superseded_by) {
|
|
3334
|
+
reject(`repeats ${JSON.stringify(p.repeats)} is not a live pitfall lesson`);
|
|
3335
|
+
continue;
|
|
3336
|
+
}
|
|
3337
|
+
occur(target.id, item);
|
|
3338
|
+
continue;
|
|
3339
|
+
}
|
|
3340
|
+
const id = computeLessonId({ kind: "pitfall", citation: item.ref, text });
|
|
3341
|
+
const collision = getLesson(m, id);
|
|
3342
|
+
if (collision || appendedIds.has(id)) {
|
|
3343
|
+
if (collision && collision.superseded_by) reject(`addLesson id collision with a superseded lesson ${id}`);
|
|
3344
|
+
else occur(id, item);
|
|
3345
|
+
continue;
|
|
3346
|
+
}
|
|
3347
|
+
if (appended.length >= FAILURE_APPEND_MAX) {
|
|
3348
|
+
reject(`at most ${FAILURE_APPEND_MAX} lessons appended per reflection`);
|
|
3349
|
+
continue;
|
|
3350
|
+
}
|
|
3351
|
+
if (dryRun) {
|
|
3352
|
+
appended.push(id);
|
|
3353
|
+
appendedIds.add(id);
|
|
3354
|
+
continue;
|
|
3355
|
+
}
|
|
3356
|
+
const row = addLesson(m, {
|
|
3357
|
+
text,
|
|
3358
|
+
kind: "pitfall",
|
|
3359
|
+
scope: Array.isArray(p.scope) ? p.scope : [],
|
|
3360
|
+
citation: item.ref,
|
|
3361
|
+
source: { agent: "reflect", model },
|
|
3362
|
+
card: item.card,
|
|
3363
|
+
});
|
|
3364
|
+
appended.push(row.id);
|
|
3365
|
+
appendedIds.add(row.id);
|
|
3366
|
+
recordPitfallOccurrence(dir, { lesson: row.id, citation: item.ref, card: item.card, run: item.run, session: item.session, at: now, class: item.class, files: item.files });
|
|
3367
|
+
}
|
|
3368
|
+
|
|
3369
|
+
const sessionId = result.sessionId || null;
|
|
3370
|
+
const cost = computeReflectCost({ dir, claudeDir, sessionId, totalCostUsd: typeof result.totalCostUsd === "number" ? result.totalCostUsd : null });
|
|
3371
|
+
if (!dryRun) {
|
|
3372
|
+
m.prepare(
|
|
3373
|
+
`INSERT INTO reflections (report_hash, recorded, run, card, commit_sha, model, counters, proposed, appended, bumped, rejected, usd, usd_source, session_id)
|
|
3374
|
+
VALUES (?, ?, NULL, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3375
|
+
).run(key, now, card, model, JSON.stringify({ helpful: [], harmful: [], unknown: [] }), result.lessons.length, JSON.stringify(appended), JSON.stringify(repeated), JSON.stringify(rejected), cost.usd, cost.usdSource, sessionId);
|
|
3376
|
+
}
|
|
3377
|
+
return { ok: true, dryRun, ...base, proposed: result.lessons.length, appended, repeated, rejected, usd: cost.usd, usdSource: cost.usdSource, model, sessionId, key };
|
|
3378
|
+
}
|
|
3379
|
+
|
|
3380
|
+
// The on-demand cited project model (card memory-project-model, ADR 0019 L2): `memory.mjs model regen` asks one
|
|
3381
|
+
// cheap-row model call for statements about this project learned from runs, each citing live lessons or run
|
|
3382
|
+
// history, and keeps only statements whose every citation resolves in code. The whole file stays under the cap.
|
|
3383
|
+
export const PROJECT_MODEL_CHAR_CAP = 2000;
|
|
3384
|
+
export const PROJECT_MODEL_RELPATH = ".doug/project-model.md";
|
|
3385
|
+
const PROJECT_MODEL_SECTIONS = [
|
|
3386
|
+
["failure-patterns", "Failure patterns"],
|
|
3387
|
+
["conventions", "Conventions"],
|
|
3388
|
+
["open-risks", "Open risks"],
|
|
3389
|
+
];
|
|
3390
|
+
const PROJECT_MODEL_LESSON_TEXT_MAX = 500;
|
|
3391
|
+
|
|
3392
|
+
export const PROJECT_MODEL_SCHEMA = {
|
|
3393
|
+
type: "object",
|
|
3394
|
+
required: ["statements"],
|
|
3395
|
+
properties: {
|
|
3396
|
+
statements: {
|
|
3397
|
+
type: "array",
|
|
3398
|
+
items: {
|
|
3399
|
+
type: "object",
|
|
3400
|
+
required: ["section", "text", "citations"],
|
|
3401
|
+
properties: {
|
|
3402
|
+
section: { enum: PROJECT_MODEL_SECTIONS.map(([key]) => key) },
|
|
3403
|
+
text: { type: "string" },
|
|
3404
|
+
citations: { type: "array", items: { type: "string" }, minItems: 1 },
|
|
3405
|
+
},
|
|
3406
|
+
},
|
|
3407
|
+
},
|
|
3408
|
+
},
|
|
3409
|
+
};
|
|
3410
|
+
|
|
3411
|
+
// Every run-history ref `<card>/<run>` under .doug/.state/history, in directory order.
|
|
3412
|
+
export function listRunHistoryRefs(dir) {
|
|
3413
|
+
const root = join(dir, HISTORY_DIR_RELPATH);
|
|
3414
|
+
const refs = [];
|
|
3415
|
+
try {
|
|
3416
|
+
for (const card of readdirSync(root).sort()) {
|
|
3417
|
+
let files = [];
|
|
3418
|
+
try {
|
|
3419
|
+
files = readdirSync(join(root, card)).filter((f) => f.endsWith(".jsonl")).sort();
|
|
3420
|
+
} catch {
|
|
3421
|
+
continue;
|
|
3422
|
+
}
|
|
3423
|
+
for (const f of files) refs.push(`${card}/${f.slice(0, -".jsonl".length)}`);
|
|
3424
|
+
}
|
|
3425
|
+
} catch {
|
|
3426
|
+
return [];
|
|
3427
|
+
}
|
|
3428
|
+
return refs;
|
|
3429
|
+
}
|
|
3430
|
+
|
|
3431
|
+
export function buildProjectModelPrompt({ lessons = [], runs = [] } = {}) {
|
|
3432
|
+
const lines = [
|
|
3433
|
+
"You write the project model of a coding harness: what is true about THIS project now, learned from its runs only. Three sections: failure-patterns (what keeps going wrong), conventions (conventions learned from runs), open-risks (what is still unresolved or fragile).",
|
|
3434
|
+
"Write short factual statements. Each statement has a `section` (failure-patterns, conventions or open-risks), a `text`, and `citations`: one or more lesson ids (the 16 hex characters in brackets below) or run-history refs (`<card>/<run>` or `<card>/<run>:<line>`, exactly as listed below). You must cite; a statement whose every citation is not exactly one of the ids or refs below is dropped by code.",
|
|
3435
|
+
"Do not restate CLAUDE.md, the ADRs or board cards: link to them by path or card id instead. A statement that should become a rule is not written here; rules go through the proposal path.",
|
|
3436
|
+
`Keep the whole document under ${PROJECT_MODEL_CHAR_CAP} characters, header included: a handful of dense statements, each worth its line. Prefer statements that cite several lessons.`,
|
|
3437
|
+
"",
|
|
3438
|
+
`Live lessons (${lessons.length}):`,
|
|
3439
|
+
];
|
|
3440
|
+
for (const l of lessons) {
|
|
3441
|
+
const text = String(l.text ?? "").replace(/\s+/g, " ").trim();
|
|
3442
|
+
const shown = text.length > PROJECT_MODEL_LESSON_TEXT_MAX ? text.slice(0, PROJECT_MODEL_LESSON_TEXT_MAX) + "..." : text;
|
|
3443
|
+
lines.push(`- [${l.id}] (${l.kind}) ${shown}`);
|
|
3444
|
+
}
|
|
3445
|
+
lines.push("", `Run history refs (${runs.length}):`);
|
|
3446
|
+
for (const r of runs) lines.push(`- ${r}`);
|
|
3447
|
+
return lines.join("\n") + "\n";
|
|
3448
|
+
}
|
|
3449
|
+
|
|
3450
|
+
// A citation resolves when it is the id of a live (not superseded) lesson, or a history ref with a run whose file
|
|
3451
|
+
// (and line, when named) exists. Anything else, including a thrown parse or read, is unresolved. A statement with
|
|
3452
|
+
// any unresolved citation is dropped whole.
|
|
3453
|
+
function citationResolver(m, dir) {
|
|
3454
|
+
return (c) => {
|
|
3455
|
+
if (typeof c !== "string") return false;
|
|
3456
|
+
if (/^[0-9a-f]{16}$/.test(c)) {
|
|
3457
|
+
const row = getLesson(m, c);
|
|
3458
|
+
return !!row && !row.superseded_by;
|
|
3459
|
+
}
|
|
3460
|
+
try {
|
|
3461
|
+
const ref = parseHistoryRef(c);
|
|
3462
|
+
if (ref.run === null) return false;
|
|
3463
|
+
readRunHistory(dir, ref);
|
|
3464
|
+
return true;
|
|
3465
|
+
} catch {
|
|
3466
|
+
return false;
|
|
3467
|
+
}
|
|
3468
|
+
};
|
|
3469
|
+
}
|
|
3470
|
+
|
|
3471
|
+
export function checkProjectModelCitations(m, dir, statements) {
|
|
3472
|
+
const resolves = citationResolver(m, dir);
|
|
3473
|
+
const kept = [];
|
|
3474
|
+
const dropped = [];
|
|
3475
|
+
for (const s of statements) {
|
|
3476
|
+
const cites = Array.isArray(s && s.citations) ? s.citations : [];
|
|
3477
|
+
if (cites.length > 0 && cites.every(resolves)) kept.push(s);
|
|
3478
|
+
else dropped.push({ text: s && s.text, reason: "unresolved", citations: cites });
|
|
3479
|
+
}
|
|
3480
|
+
return { kept, dropped };
|
|
3481
|
+
}
|
|
3482
|
+
|
|
3483
|
+
// The on-demand idea pass (card memory-idea-proposals, ADR 0019 L3): `memory.mjs ideas` asks one cheap-row model
|
|
3484
|
+
// call over live lessons, run history and pitfall occurrences for idea cards and priority reviews. Ideas are
|
|
3485
|
+
// filed to Backlog only, cited, capped per pass; reviews go to a jsonl file for the user. Never Ready.
|
|
3486
|
+
export const IDEA_CAP = 3;
|
|
3487
|
+
export const PRIORITY_REVIEWS_RELPATH = ".doug/.state/memory/priority-reviews.jsonl";
|
|
3488
|
+
const IDEAS_LESSON_TEXT_MAX = 500;
|
|
3489
|
+
|
|
3490
|
+
export const IDEAS_SCHEMA = {
|
|
3491
|
+
type: "object",
|
|
3492
|
+
required: ["ideas", "reviews"],
|
|
3493
|
+
properties: {
|
|
3494
|
+
ideas: {
|
|
3495
|
+
type: "array",
|
|
3496
|
+
items: {
|
|
3497
|
+
type: "object",
|
|
3498
|
+
required: ["id", "title", "goal", "component", "size", "citations"],
|
|
3499
|
+
properties: {
|
|
3500
|
+
id: { type: "string" },
|
|
3501
|
+
title: { type: "string" },
|
|
3502
|
+
goal: { type: "string" },
|
|
3503
|
+
component: { type: "string" },
|
|
3504
|
+
size: { type: "string" },
|
|
3505
|
+
citations: { type: "array", items: { type: "string" }, minItems: 1 },
|
|
3506
|
+
},
|
|
3507
|
+
},
|
|
3508
|
+
},
|
|
3509
|
+
reviews: {
|
|
3510
|
+
type: "array",
|
|
3511
|
+
items: {
|
|
3512
|
+
type: "object",
|
|
3513
|
+
required: ["text", "cards", "citations"],
|
|
3514
|
+
properties: {
|
|
3515
|
+
text: { type: "string" },
|
|
3516
|
+
cards: { type: "array", items: { type: "string" }, minItems: 1 },
|
|
3517
|
+
citations: { type: "array", items: { type: "string" }, minItems: 1 },
|
|
3518
|
+
},
|
|
3519
|
+
},
|
|
3520
|
+
},
|
|
3521
|
+
},
|
|
3522
|
+
};
|
|
3523
|
+
|
|
3524
|
+
export function buildIdeasPrompt({ lessons = [], runs = [], pitfalls = [], cards = [] } = {}) {
|
|
3525
|
+
const lines = [
|
|
3526
|
+
"You look over the lessons and run history of a coding harness and propose follow-up work for its user: idea cards and priority reviews.",
|
|
3527
|
+
"Look for recurring failures, repeated manual steps and unfinished follow-ups. An idea is filed only when it is realistic drift (a form a model or a person would plausibly write) or when one card would close the whole class; otherwise propose nothing.",
|
|
3528
|
+
"Each idea has an `id` (kebab-case, not any card id listed below), `title`, `goal` (name the defect or gap plainly), `component`, `size` and `citations`. Do not duplicate a card listed below.",
|
|
3529
|
+
"Each review (`text`, `cards`, `citations`) groups Backlog cards for the user to look at, for example Backlog cards that share a recurring pitfall. `cards` may name Backlog cards only. Never propose a change to Ready.",
|
|
3530
|
+
"You must cite: `citations` are lesson ids (the 16 hex characters in brackets below) or run-history refs (`<card>/<run>` or `<card>/<run>:<line>`), exactly as listed below. An item with any citation not exactly one of them, or with none, is dropped by code.",
|
|
3531
|
+
"",
|
|
3532
|
+
`Live lessons (${lessons.length}):`,
|
|
3533
|
+
];
|
|
3534
|
+
for (const l of lessons) {
|
|
3535
|
+
const text = String(l.text ?? "").replace(/\s+/g, " ").trim();
|
|
3536
|
+
const shown = text.length > IDEAS_LESSON_TEXT_MAX ? text.slice(0, IDEAS_LESSON_TEXT_MAX) + "..." : text;
|
|
3537
|
+
lines.push(`- [${l.id}] (${l.kind}) ${shown}`);
|
|
3538
|
+
}
|
|
3539
|
+
lines.push("", `Run history refs (${runs.length}):`);
|
|
3540
|
+
for (const r of runs) lines.push(`- ${r}`);
|
|
3541
|
+
lines.push("", `Pitfall occurrences (${pitfalls.length}):`);
|
|
3542
|
+
for (const p of pitfalls) lines.push(`- lesson ${p.lesson} (card ${p.card ?? "?"}, run ${p.run ?? "?"}): ${p.citation ?? ""}`);
|
|
3543
|
+
lines.push("", `Existing Backlog and Ready cards (${cards.length}):`);
|
|
3544
|
+
for (const c of cards) lines.push(`- ${c.id} (${c.column}): ${c.title}`);
|
|
3545
|
+
return lines.join("\n") + "\n";
|
|
3546
|
+
}
|
|
3547
|
+
|
|
3548
|
+
export function checkIdeaCitations(m, dir, items) {
|
|
3549
|
+
const resolves = citationResolver(m, dir);
|
|
3550
|
+
const kept = [];
|
|
3551
|
+
const dropped = [];
|
|
3552
|
+
for (const it of items) {
|
|
3553
|
+
const cites = Array.isArray(it && it.citations) ? it.citations : [];
|
|
3554
|
+
if (cites.length > 0 && cites.every(resolves)) kept.push(it);
|
|
3555
|
+
else dropped.push({ id: it && it.id, text: it && it.text, reason: "unresolved", citations: cites });
|
|
3556
|
+
}
|
|
3557
|
+
return { kept, dropped };
|
|
3558
|
+
}
|
|
3559
|
+
|
|
3560
|
+
export const IDEA_ID_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
3561
|
+
|
|
3562
|
+
// Pure: files up to `cap` ideas as new Backlog cards appended after every existing card.
|
|
3563
|
+
export function fileIdeaCards(board, ideas, { cap = IDEA_CAP, date } = {}) {
|
|
3564
|
+
let next = board;
|
|
3565
|
+
const filed = [];
|
|
3566
|
+
const skipped = [];
|
|
3567
|
+
for (const it of ideas) {
|
|
3568
|
+
if (typeof it.id !== "string" || !IDEA_ID_RE.test(it.id)) {
|
|
3569
|
+
skipped.push({ id: it.id, reason: "id is not kebab-case" });
|
|
3570
|
+
continue;
|
|
3571
|
+
}
|
|
3572
|
+
if (filed.length >= cap) {
|
|
3573
|
+
skipped.push({ id: it.id, reason: "over cap" });
|
|
3574
|
+
continue;
|
|
3575
|
+
}
|
|
3576
|
+
const cites = Array.isArray(it.citations) ? it.citations.join(", ") : "";
|
|
3577
|
+
try {
|
|
3578
|
+
const added = addCard(
|
|
3579
|
+
next,
|
|
3580
|
+
{
|
|
3581
|
+
id: it.id,
|
|
3582
|
+
title: it.title,
|
|
3583
|
+
component: it.component,
|
|
3584
|
+
size: it.size,
|
|
3585
|
+
track: "hand",
|
|
3586
|
+
deps: [],
|
|
3587
|
+
column: "backlog",
|
|
3588
|
+
goal: `Memory-proposed (memory.mjs ideas, ${date}): ${it.goal} Evidence: ${cites}.`,
|
|
3589
|
+
},
|
|
3590
|
+
{ date }
|
|
3591
|
+
);
|
|
3592
|
+
const cards = added.cards.slice();
|
|
3593
|
+
cards[cards.length - 1] = { ...cards[cards.length - 1], source: `memory-proposed: ${cites}` };
|
|
3594
|
+
next = { ...added, cards };
|
|
3595
|
+
filed.push(it.id);
|
|
3596
|
+
} catch (err) {
|
|
3597
|
+
skipped.push({ id: it.id, reason: String(err && err.message ? err.message : err).split("\n").slice(0, 2).join(" ") });
|
|
3598
|
+
}
|
|
3599
|
+
}
|
|
3600
|
+
return { board: next, filed, skipped };
|
|
3601
|
+
}
|
|
3602
|
+
|
|
3603
|
+
export function readPriorityReviews(dir) {
|
|
3604
|
+
const file = join(dir, PRIORITY_REVIEWS_RELPATH);
|
|
3605
|
+
if (!existsSync(file)) return [];
|
|
3606
|
+
const out = [];
|
|
3607
|
+
try {
|
|
3608
|
+
for (const line of readFileSync(file, "utf8").split("\n")) {
|
|
3609
|
+
if (!line.trim()) continue;
|
|
3610
|
+
try {
|
|
3611
|
+
const r = JSON.parse(line);
|
|
3612
|
+
if (r && typeof r === "object") out.push(r);
|
|
3613
|
+
} catch {
|
|
3614
|
+
// malformed line: skip
|
|
3615
|
+
}
|
|
3616
|
+
}
|
|
3617
|
+
} catch {
|
|
3618
|
+
return [];
|
|
3619
|
+
}
|
|
3620
|
+
return out;
|
|
3621
|
+
}
|
|
3622
|
+
|
|
3623
|
+
function reviewKey(r) {
|
|
3624
|
+
const cards = Array.isArray(r && r.cards) ? r.cards : [];
|
|
3625
|
+
return JSON.stringify([[...new Set(cards)].sort(), String((r && r.text) ?? "").replace(/\s+/g, " ").trim()]);
|
|
3626
|
+
}
|
|
3627
|
+
|
|
3628
|
+
export function checkPriorityReviews(board, reviews, { cap = IDEA_CAP, recorded = [] } = {}) {
|
|
3629
|
+
const backlog = new Set(board.cards.filter((c) => c.column === "backlog").map((c) => c.id));
|
|
3630
|
+
const seen = new Set(recorded.map(reviewKey));
|
|
3631
|
+
const kept = [];
|
|
3632
|
+
const dropped = [];
|
|
3633
|
+
for (const r of reviews) {
|
|
3634
|
+
const key = reviewKey(r);
|
|
3635
|
+
if (seen.has(key)) {
|
|
3636
|
+
dropped.push({ text: r && r.text, reason: "already recorded" });
|
|
3637
|
+
continue;
|
|
3638
|
+
}
|
|
3639
|
+
const cards = Array.isArray(r && r.cards) ? r.cards : [];
|
|
3640
|
+
if (cards.length > 0 && cards.every((id) => backlog.has(id)) && kept.length < cap) {
|
|
3641
|
+
kept.push(r);
|
|
3642
|
+
seen.add(key);
|
|
3643
|
+
} else dropped.push({ text: r && r.text, reason: kept.length >= cap && cards.every((id) => backlog.has(id)) ? "over cap" : "not all backlog cards" });
|
|
3644
|
+
}
|
|
3645
|
+
return { kept, dropped };
|
|
3646
|
+
}
|
|
3647
|
+
|
|
3648
|
+
export function renderProjectModel(statements, { date = new Date().toISOString().slice(0, 10) } = {}) {
|
|
3649
|
+
const header =
|
|
3650
|
+
`Generated by \`memory.mjs model regen\` on ${date}; do not edit by hand. Each statement cites a lesson (\`memory.mjs lesson show <id>\`) or a run record (\`memory.mjs history show <ref>\`).`;
|
|
3651
|
+
const build = (list) => {
|
|
3652
|
+
const out = ["# Project model", "", header, ""];
|
|
3653
|
+
for (const [key, title] of PROJECT_MODEL_SECTIONS) {
|
|
3654
|
+
const inSection = list.filter((s) => s.section === key);
|
|
3655
|
+
if (!inSection.length) continue;
|
|
3656
|
+
out.push(`## ${title}`);
|
|
3657
|
+
for (const s of inSection) out.push(`- ${s.text} [${s.citations.join(", ")}]`);
|
|
3658
|
+
}
|
|
3659
|
+
return out.join("\n") + "\n";
|
|
3660
|
+
};
|
|
3661
|
+
const known = statements.filter((s) => PROJECT_MODEL_SECTIONS.some(([key]) => key === s.section));
|
|
3662
|
+
const priority = known.map((s, i) => ({ s, i })).sort((a, b) => b.s.citations.length - a.s.citations.length || a.i - b.i);
|
|
3663
|
+
const chosen = new Set();
|
|
3664
|
+
for (const { s } of priority) {
|
|
3665
|
+
const trial = known.filter((x) => chosen.has(x) || x === s);
|
|
3666
|
+
if (build(trial).length > PROJECT_MODEL_CHAR_CAP) break;
|
|
3667
|
+
chosen.add(s);
|
|
3668
|
+
}
|
|
3669
|
+
const kept = known.filter((s) => chosen.has(s));
|
|
3670
|
+
const overCap = statements.filter((s) => !chosen.has(s));
|
|
3671
|
+
const text = build(kept);
|
|
3672
|
+
return { text, chars: text.length, kept, overCap };
|
|
3673
|
+
}
|