dsh-plugin-soulbridge 0.0.119 → 0.0.120
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +315 -79
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -1116,6 +1116,12 @@ export function createPreStepHandler(deps) {
|
|
|
1116
1116
|
const decision = await next();
|
|
1117
1117
|
try {
|
|
1118
1118
|
if (decision?.kind !== 'enter') return decision;
|
|
1119
|
+
// Hoisted before the chain starts: its stage closures run synchronously inside
|
|
1120
|
+
// runStageChain, so reading these afterwards would hit the temporal dead zone.
|
|
1121
|
+
// `sessionId` is what the departure checklist compares buffered turns against
|
|
1122
|
+
// (0028); `archivedSessions` is the same host set closeIdleThreads uses below.
|
|
1123
|
+
const sessionId = payload?.agent?.id ?? payload?.agent?.session?.id ?? 'default';
|
|
1124
|
+
const archivedSessions = deps.archivedSessions?.() ?? [];
|
|
1119
1125
|
// We are inside a live turn now, which is the only place the host model is
|
|
1120
1126
|
// callable. Stages run in priority order and a stage that skipped its work
|
|
1121
1127
|
// must not claim the call — that is how the classifier starved for an hour.
|
|
@@ -1141,9 +1147,12 @@ export function createPreStepHandler(deps) {
|
|
|
1141
1147
|
// Live conversation before everything; understanding before the backfill.
|
|
1142
1148
|
// Fixed priority (extract always first) plus a long backfill is a starvation
|
|
1143
1149
|
// bug, not a preference: 29 closed threads waited while the backfill drained.
|
|
1144
|
-
|
|
1150
|
+
// The drain carries the departure context: it flushes a segment only when
|
|
1151
|
+
// these say something left the window (0028 决定 2), and waits otherwise.
|
|
1152
|
+
const drainSegment = () => drainExtractionQueue({ sessionId, archivedSessions });
|
|
1153
|
+
if (extractRunsFirst(extractQueue, consolidationPending)) stages.push(drainSegment);
|
|
1145
1154
|
stages.push(runConsolidationStep);
|
|
1146
|
-
stages.push(
|
|
1155
|
+
stages.push(drainSegment);
|
|
1147
1156
|
if (profilePending && stageTick % 2 === 0) stages.push(runProfileStep);
|
|
1148
1157
|
stages.push(runClassifyStep);
|
|
1149
1158
|
if (profilePending && stageTick % 2 === 1) stages.push(runProfileStep);
|
|
@@ -1175,7 +1184,6 @@ export function createPreStepHandler(deps) {
|
|
|
1175
1184
|
// Topics the conversation has moved past, closed once per turn. Counting
|
|
1176
1185
|
// actual turns — the host's turn numbers jump by 10–30 per exchange, so a
|
|
1177
1186
|
// numeric difference closed a topic one message after it opened.
|
|
1178
|
-
const sessionId = payload?.agent?.id ?? payload?.agent?.session?.id ?? 'default';
|
|
1179
1187
|
try {
|
|
1180
1188
|
const closeConfig = readConfig();
|
|
1181
1189
|
const closed = ensureSoul().closeIdleThreads(sessionId, {
|
|
@@ -1183,7 +1191,7 @@ export function createPreStepHandler(deps) {
|
|
|
1183
1191
|
idleMs: (closeConfig.threadIdleMinutes ?? 30) * 60_000,
|
|
1184
1192
|
// The host's own archive list: a session the user put away is finished
|
|
1185
1193
|
// by declaration, so its topics close immediately.
|
|
1186
|
-
archivedSessionIds:
|
|
1194
|
+
archivedSessionIds: archivedSessions,
|
|
1187
1195
|
at: Date.now(),
|
|
1188
1196
|
});
|
|
1189
1197
|
if (closed.length > 0) {
|
|
@@ -1352,6 +1360,17 @@ function flushTurnPair(sessionId) {
|
|
|
1352
1360
|
const EXTRACT_QUEUE_FILE = join(HOME, 'pending-extract.json');
|
|
1353
1361
|
const EXTRACT_MAX_ATTEMPTS = 3;
|
|
1354
1362
|
|
|
1363
|
+
/* 0028 决定 2: the departure checklist, declared once (rule 6).
|
|
1364
|
+
* The two size bounds and the age bound are ENGINEERING fallbacks — they flush a
|
|
1365
|
+
* buffer that would otherwise dangle, never a claim that a topic ended. All three
|
|
1366
|
+
* are test inputs to planSegment, not scattered literals. */
|
|
1367
|
+
export const SEGMENT_MAX_TURNS = 40;
|
|
1368
|
+
export const SEGMENT_MAX_CHARS = 60_000;
|
|
1369
|
+
export const SEGMENT_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
1370
|
+
const SEGMENT_STATE_FILE = join(HOME, 'segment-state.json');
|
|
1371
|
+
/** Seals are a marker, not work: the newest N prove "extracted at most once". */
|
|
1372
|
+
const SEGMENT_SEAL_CAP = 100;
|
|
1373
|
+
|
|
1355
1374
|
/**
|
|
1356
1375
|
* Read a queue file.
|
|
1357
1376
|
*
|
|
@@ -1422,14 +1441,146 @@ function bumpExtractAttempt(turnId) {
|
|
|
1422
1441
|
}
|
|
1423
1442
|
|
|
1424
1443
|
/**
|
|
1425
|
-
*
|
|
1426
|
-
*
|
|
1444
|
+
* Remove a whole segment's items by identity, computed from a fresh read.
|
|
1445
|
+
*
|
|
1446
|
+
* A segment is one extraction unit (0028): its turns leave the queue together or
|
|
1447
|
+
* stay together, so the id list is the handle — a thread id no longer is, because
|
|
1448
|
+
* attribution is an output of the extraction, not an input to it.
|
|
1427
1449
|
*/
|
|
1428
|
-
function
|
|
1429
|
-
|
|
1430
|
-
writeExtractQueue(
|
|
1431
|
-
|
|
1432
|
-
|
|
1450
|
+
function removeExtractItems(turnIds) {
|
|
1451
|
+
const ids = new Set(turnIds);
|
|
1452
|
+
writeExtractQueue(readExtractQueue().filter((item) => !ids.has(item.turnId)));
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
/* ------------------------------------------------------------------ *
|
|
1456
|
+
* Segments (0028): what left the context, and what was already flushed
|
|
1457
|
+
* ------------------------------------------------------------------ */
|
|
1458
|
+
|
|
1459
|
+
/**
|
|
1460
|
+
* Read the segment state: the departure marker and the seal records.
|
|
1461
|
+
*
|
|
1462
|
+
* An unreadable file is not an empty one (same rule as the queue files): it is
|
|
1463
|
+
* set aside with its bytes intact, because losing every seal record would let
|
|
1464
|
+
* already-extracted turns be enqueued again with nothing to say so.
|
|
1465
|
+
*/
|
|
1466
|
+
function readSegmentState() {
|
|
1467
|
+
try {
|
|
1468
|
+
const raw = JSON.parse(readFileSync(SEGMENT_STATE_FILE, 'utf8'));
|
|
1469
|
+
return {
|
|
1470
|
+
marker: raw?.marker ?? null,
|
|
1471
|
+
sealed: Array.isArray(raw?.sealed) ? raw.sealed : [],
|
|
1472
|
+
};
|
|
1473
|
+
} catch (error) {
|
|
1474
|
+
if (existsSync(SEGMENT_STATE_FILE)) {
|
|
1475
|
+
log(`segment state unreadable, setting it aside: ${error instanceof Error ? error.message : String(error)}`);
|
|
1476
|
+
try {
|
|
1477
|
+
renameSync(SEGMENT_STATE_FILE, `${SEGMENT_STATE_FILE}.corrupt-${Date.now()}`);
|
|
1478
|
+
} catch {
|
|
1479
|
+
/* the log above is the record */
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
return { marker: null, sealed: [] };
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
function writeSegmentState(state) {
|
|
1487
|
+
try {
|
|
1488
|
+
const sealed = Array.isArray(state.sealed) ? state.sealed : [];
|
|
1489
|
+
const kept = sealed.slice(-SEGMENT_SEAL_CAP);
|
|
1490
|
+
const dropped = sealed.length - kept.length;
|
|
1491
|
+
// A cap drop is visible like every other cap (rule 15/18), even though seals
|
|
1492
|
+
// are markers rather than pending work.
|
|
1493
|
+
if (dropped > 0) log(`segment seals over cap: wrote ${kept.length}, dropped ${dropped}`);
|
|
1494
|
+
writeFileSync(
|
|
1495
|
+
SEGMENT_STATE_FILE,
|
|
1496
|
+
JSON.stringify({ marker: state.marker ?? null, sealed: kept }, null, 2),
|
|
1497
|
+
'utf8',
|
|
1498
|
+
);
|
|
1499
|
+
} catch (error) {
|
|
1500
|
+
log(`segment state write failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
/**
|
|
1505
|
+
* The host said "this left the window" (0028 决定 2): set the marker and stop.
|
|
1506
|
+
*
|
|
1507
|
+
* Compression, session switch and archive only ever leave marks — no model call
|
|
1508
|
+
* happens on an event, because the host model is callable inside a step only.
|
|
1509
|
+
* The next drain reads the marker and flushes what it covers.
|
|
1510
|
+
*/
|
|
1511
|
+
function setSegmentMarker(marker) {
|
|
1512
|
+
const state = readSegmentState();
|
|
1513
|
+
writeSegmentState({ ...state, marker });
|
|
1514
|
+
log(`segment marker: trigger=${marker.trigger} session=${marker.session ?? '-'} at=${marker.at}`);
|
|
1515
|
+
trace({ event: 'segment-marker', ...marker });
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
/**
|
|
1519
|
+
* Record that a segment was flushed: the marker is spent, the seal list grows.
|
|
1520
|
+
*
|
|
1521
|
+
* The seal is the queryable half of declaration 1 ("a sealed segment is not
|
|
1522
|
+
* extracted twice"): removing the items makes a re-run unlikely, the record makes
|
|
1523
|
+
* the claim checkable — and lets enqueueExtraction refuse a re-queue.
|
|
1524
|
+
*/
|
|
1525
|
+
function sealSegment({ trigger, turnIds, turns, chars, outcome, session }) {
|
|
1526
|
+
const state = readSegmentState();
|
|
1527
|
+
writeSegmentState({
|
|
1528
|
+
marker: null,
|
|
1529
|
+
sealed: [
|
|
1530
|
+
...state.sealed,
|
|
1531
|
+
{ at: Date.now(), trigger, session: session ?? null, turns, chars, outcome, turnIds },
|
|
1532
|
+
],
|
|
1533
|
+
});
|
|
1534
|
+
trace({ event: 'segment-seal', trigger, turns, chars, outcome, session: session ?? null });
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
/**
|
|
1538
|
+
* The departure checklist (0028 决定 2): which buffered turns to flush, and why.
|
|
1539
|
+
*
|
|
1540
|
+
* Pure on purpose — entries, marker, archive set and clock are arguments, so the
|
|
1541
|
+
* trigger table in the brief is a test table (rule 26). Two kinds of answer:
|
|
1542
|
+
*
|
|
1543
|
+
* - a semantic departure: the first CONTIGUOUS run that has left the context
|
|
1544
|
+
* (its session is not the current one, or it predates the compaction marker).
|
|
1545
|
+
* Contiguity keeps the run a stretch of conversation; taking the first run
|
|
1546
|
+
* means a backfill batch appended behind live turns is not stranded behind
|
|
1547
|
+
* them — live turns wait for their own departure, history flushes on its own.
|
|
1548
|
+
* - an engineering fallback over the WHOLE buffer: over the size cap or older
|
|
1549
|
+
* than a day. These flush what would otherwise dangle; they never assert a
|
|
1550
|
+
* topic boundary (quiet-time: a time signal is only ever a fallback).
|
|
1551
|
+
*
|
|
1552
|
+
* Returns `{ entries, trigger }`; `trigger: null` means nothing has left yet.
|
|
1553
|
+
*/
|
|
1554
|
+
export function planSegment(entries, env = {}) {
|
|
1555
|
+
const {
|
|
1556
|
+
currentSessionId,
|
|
1557
|
+
markerAt = null,
|
|
1558
|
+
archived = false,
|
|
1559
|
+
now = Date.now(),
|
|
1560
|
+
maxTurns = SEGMENT_MAX_TURNS,
|
|
1561
|
+
maxChars = SEGMENT_MAX_CHARS,
|
|
1562
|
+
maxAgeMs = SEGMENT_MAX_AGE_MS,
|
|
1563
|
+
} = env;
|
|
1564
|
+
if (!Array.isArray(entries) || entries.length === 0) return { entries: [], trigger: null };
|
|
1565
|
+
// Archiving is the user declaring the session finished: everything buffered goes.
|
|
1566
|
+
if (archived) return { entries, trigger: 'archived' };
|
|
1567
|
+
const departed = (entry) => {
|
|
1568
|
+
if (currentSessionId && entry.sessionId && entry.sessionId !== currentSessionId) return 'session';
|
|
1569
|
+
if (markerAt !== null && entry.startedAt <= markerAt) return 'compaction';
|
|
1570
|
+
return null;
|
|
1571
|
+
};
|
|
1572
|
+
const start = entries.findIndex((entry) => departed(entry) !== null);
|
|
1573
|
+
if (start >= 0) {
|
|
1574
|
+
let end = start;
|
|
1575
|
+
while (end < entries.length && departed(entries[end]) !== null) end += 1;
|
|
1576
|
+
const run = entries.slice(start, end);
|
|
1577
|
+
return { entries: run, trigger: departed(run[0]) };
|
|
1578
|
+
}
|
|
1579
|
+
const chars = entries.reduce((sum, entry) => sum + (entry.chars ?? 0), 0);
|
|
1580
|
+
if (entries.length >= maxTurns || chars >= maxChars) return { entries, trigger: 'capacity' };
|
|
1581
|
+
const oldest = Math.min(...entries.map((entry) => entry.startedAt ?? now));
|
|
1582
|
+
if (now - oldest > maxAgeMs) return { entries, trigger: 'age' };
|
|
1583
|
+
return { entries: [], trigger: null };
|
|
1433
1584
|
}
|
|
1434
1585
|
|
|
1435
1586
|
/**
|
|
@@ -1443,12 +1594,19 @@ function removeExtractItemsOfThread(threadId, turnIds) {
|
|
|
1443
1594
|
export function enqueueExtraction(turnId, turnNo, origin = 'live') {
|
|
1444
1595
|
const queue = readExtractQueue();
|
|
1445
1596
|
if (queue.some((item) => item.turnId === turnId)) return false;
|
|
1597
|
+
// A sealed segment is not re-queued (0028 declaration 1): the seal record is
|
|
1598
|
+
// what makes "extracted at most once" checkable instead of merely intended.
|
|
1599
|
+
// Replays are the documented exception and carry origin=backfill.
|
|
1600
|
+
if (origin !== 'backfill' && readSegmentState().sealed.some((seg) => seg.turnIds.includes(turnId))) {
|
|
1601
|
+
log(`extract buffered: turn=${turnNo} already sealed, refused (a replay needs origin=backfill)`);
|
|
1602
|
+
return false;
|
|
1603
|
+
}
|
|
1446
1604
|
// `origin` separates live turns from a historical backfill. Without it the attribution
|
|
1447
1605
|
// experiment counts re-extracted old turns as fresh evidence, and a verdict about the
|
|
1448
1606
|
// new default-continue rule would be read off history that never saw that rule.
|
|
1449
1607
|
queue.push({ turnId, turnNo, attempts: 0, origin });
|
|
1450
1608
|
writeExtractQueue(queue);
|
|
1451
|
-
log(`extract
|
|
1609
|
+
log(`extract buffered turn=${turnNo} origin=${origin} pending=${queue.length} file=${EXTRACT_QUEUE_FILE}`);
|
|
1452
1610
|
return true;
|
|
1453
1611
|
}
|
|
1454
1612
|
|
|
@@ -1465,11 +1623,25 @@ function hasSubstance(text) {
|
|
|
1465
1623
|
return !/^(好的?|嗯+|哦+|可以|行|ok|okay|收到|明白|继续|好)[。!!.\s]*$/i.test(trimmed);
|
|
1466
1624
|
}
|
|
1467
1625
|
|
|
1468
|
-
|
|
1626
|
+
/**
|
|
1627
|
+
* Extract one unit: a segment (0028), or a single message when there is no segment.
|
|
1628
|
+
*
|
|
1629
|
+
* `unit` carries the prompt-numbered turns; the attribution the model returns is
|
|
1630
|
+
* a product of this call, never an input to it (决定 3). `turnId`/`turnNo` are the
|
|
1631
|
+
* segment HEAD — the continuity point before the unit — and the fallback source
|
|
1632
|
+
* for facts the model did not number.
|
|
1633
|
+
*/
|
|
1634
|
+
async function runExtraction(text, turnId, turnNo, origin = 'live', unit = null) {
|
|
1469
1635
|
const config = readConfig();
|
|
1470
1636
|
if (config.extract === 'off') return 'skipped-off';
|
|
1471
|
-
|
|
1472
|
-
|
|
1637
|
+
// The unit decides, not its last message: a segment whose final turn is "好的"
|
|
1638
|
+
// still holds every turn above it, and taking the unit or none of it is rule 17.
|
|
1639
|
+
const unitTexts = unit?.turns?.length ? unit.turns.map((entry) => entry.userText) : [text];
|
|
1640
|
+
if (typeof text !== 'string' || !unitTexts.some((entry) => typeof entry === 'string' && hasSubstance(entry))) {
|
|
1641
|
+
log(
|
|
1642
|
+
`extract skipped: no substance (chars=${unitTexts.reduce((n, entry) => n + (typeof entry === 'string' ? entry.length : 0), 0)})` +
|
|
1643
|
+
` unit=${unitTexts.length} turn=${turnNo}`,
|
|
1644
|
+
);
|
|
1473
1645
|
return 'skipped-empty';
|
|
1474
1646
|
}
|
|
1475
1647
|
const started = Date.now();
|
|
@@ -1487,9 +1659,10 @@ async function runExtraction(text, turnId, turnNo, origin = 'live', thread = nul
|
|
|
1487
1659
|
const previousThreadId = soul.threadBefore(turnRow.sessionId, turnNo);
|
|
1488
1660
|
if (previousThreadId) context.previousThreadId = previousThreadId;
|
|
1489
1661
|
}
|
|
1490
|
-
//
|
|
1491
|
-
// numbered the way the prompt numbers them
|
|
1492
|
-
|
|
1662
|
+
// 0028: the unit is the segment a departure flushed. The turns array is
|
|
1663
|
+
// numbered the way the prompt numbers them (1..n inside the unit), and each
|
|
1664
|
+
// fact may name its own turn — turnIds maps that number back to the real turn.
|
|
1665
|
+
const turns = unit?.turns ?? null;
|
|
1493
1666
|
const turnIds = turns ? Object.fromEntries(turns.map((entry) => [entry.turnNo, entry.id])) : undefined;
|
|
1494
1667
|
const draft = await soul.extract(text, {
|
|
1495
1668
|
llm: useLlm ? injectedLlm : undefined,
|
|
@@ -1524,7 +1697,7 @@ async function runExtraction(text, turnId, turnNo, origin = 'live', thread = nul
|
|
|
1524
1697
|
` gaps=${result.gaps.length} skipped=${result.skipped.length} draftSkipped=${(draft.skipped ?? []).length}` +
|
|
1525
1698
|
` slots->text=${result.droppedSlots.length} holder->world=${result.holderFallbacks.length}` +
|
|
1526
1699
|
` inferred=${inferred}` +
|
|
1527
|
-
` unit=${turns ? `${turns.length}turns` : '1turn'}
|
|
1700
|
+
` unit=${turns ? `${turns.length}turns` : '1turn'}` +
|
|
1528
1701
|
` ms=${ms} turn=${turnNo} origin=${origin}` +
|
|
1529
1702
|
` thread=${attribution.threadId}${attribution.created ? '(new)' : ''}` +
|
|
1530
1703
|
` join=${attribution.joinReason} threadLine=${draft.threadLine ?? 'rules'}` +
|
|
@@ -1573,23 +1746,17 @@ async function runExtraction(text, turnId, turnNo, origin = 'live', thread = nul
|
|
|
1573
1746
|
}
|
|
1574
1747
|
|
|
1575
1748
|
let draining = false;
|
|
1749
|
+
/**
|
|
1750
|
+
* The buffered size last reported as a wait (0028 declaration 5 observability).
|
|
1751
|
+
* Reset on every flush so the next wait always logs; between flushes it keeps the
|
|
1752
|
+
* trace to one line per buffer change instead of one per step.
|
|
1753
|
+
*/
|
|
1754
|
+
let lastSegmentWait = -1;
|
|
1576
1755
|
/** True while a stage chain is running, so two chains never overlap. */
|
|
1577
1756
|
let stageChainBusy = false;
|
|
1578
1757
|
/** Alternates the profile merge with the rest of the chain so it cannot be starved. */
|
|
1579
1758
|
let stageTick = 0;
|
|
1580
1759
|
|
|
1581
|
-
/**
|
|
1582
|
-
* Which queued turn to extract next.
|
|
1583
|
-
*
|
|
1584
|
-
* Live turns first, then history: FIFO alone would put 20 backfill items ahead of the
|
|
1585
|
-
* conversation happening right now, so a live turn waits dozens of steps to be read.
|
|
1586
|
-
*/
|
|
1587
|
-
export function pickExtractionItem(queue) {
|
|
1588
|
-
// Live turns first, then history. FIFO alone would put 20 backfill items ahead of the
|
|
1589
|
-
// conversation happening right now, so a live turn waits dozens of steps to be read.
|
|
1590
|
-
return queue.find((item) => (item.origin ?? 'live') !== 'backfill') ?? queue[0] ?? null;
|
|
1591
|
-
}
|
|
1592
|
-
|
|
1593
1760
|
/**
|
|
1594
1761
|
* Should extraction take this step before consolidation?
|
|
1595
1762
|
*
|
|
@@ -1604,18 +1771,21 @@ export function extractRunsFirst(extractQueue, consolidationPending) {
|
|
|
1604
1771
|
}
|
|
1605
1772
|
|
|
1606
1773
|
/**
|
|
1607
|
-
*
|
|
1774
|
+
* Flush ONE segment per step — the whole unit, and only at a departure (0028).
|
|
1608
1775
|
*
|
|
1609
|
-
* The host model is only callable inside the current step
|
|
1610
|
-
*
|
|
1611
|
-
*
|
|
1612
|
-
* queue
|
|
1776
|
+
* The host model is only callable inside the current step, so at most one model
|
|
1777
|
+
* call happens here. What makes it happen is not "the queue is non-empty" but the
|
|
1778
|
+
* departure checklist (planSegment): compaction marker, session switch/archive,
|
|
1779
|
+
* capacity, age. With none of them the queue only waits — in-session extraction
|
|
1780
|
+
* is 0 by construction, because those turns are still in the host's context and
|
|
1781
|
+
* extracting them again is the duplicate work 0028 exists to stop.
|
|
1613
1782
|
*
|
|
1614
|
-
* A model failure keeps the
|
|
1615
|
-
*
|
|
1616
|
-
*
|
|
1783
|
+
* A model failure keeps the WHOLE segment: one call covered it, so its items are
|
|
1784
|
+
* bumped together and share one attempt count. Three failures seal and drop the
|
|
1785
|
+
* segment with its size (rule 15) — sealed, so a later backfill cannot re-queue
|
|
1786
|
+
* what already failed three times.
|
|
1617
1787
|
*/
|
|
1618
|
-
export async function drainExtractionQueue() {
|
|
1788
|
+
export async function drainExtractionQueue(options = {}) {
|
|
1619
1789
|
if (draining) return false;
|
|
1620
1790
|
const config = readConfig();
|
|
1621
1791
|
if (config.extract === 'off') return false;
|
|
@@ -1623,63 +1793,110 @@ export async function drainExtractionQueue() {
|
|
|
1623
1793
|
if (queue.length === 0) return false;
|
|
1624
1794
|
draining = true;
|
|
1625
1795
|
try {
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1796
|
+
// Rows are the source of truth (rule 16): which session a turn belongs to and
|
|
1797
|
+
// when it happened come from `turns`, not from a copy in the queue file. A
|
|
1798
|
+
// failed read keeps the queue — a store error is not an empty result (rule 11).
|
|
1799
|
+
let entries;
|
|
1629
1800
|
try {
|
|
1630
1801
|
const soul = ensureSoul();
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
if (turns.length > 0) {
|
|
1639
|
-
thread = { id: threadId, turns };
|
|
1640
|
-
text = turns[turns.length - 1].userText;
|
|
1802
|
+
entries = [];
|
|
1803
|
+
for (const item of queue) {
|
|
1804
|
+
const row = soul.turn(item.turnId);
|
|
1805
|
+
if (!row) {
|
|
1806
|
+
log(`extract queue: turn ${item.turnId} missing, dropped`);
|
|
1807
|
+
removeExtractItem(item.turnId);
|
|
1808
|
+
continue;
|
|
1641
1809
|
}
|
|
1810
|
+
entries.push({
|
|
1811
|
+
item,
|
|
1812
|
+
row,
|
|
1813
|
+
sessionId: row.sessionId,
|
|
1814
|
+
startedAt: row.at,
|
|
1815
|
+
chars: row.userText.length + row.assistantText.length,
|
|
1816
|
+
});
|
|
1642
1817
|
}
|
|
1643
|
-
} catch {
|
|
1644
|
-
|
|
1818
|
+
} catch (error) {
|
|
1819
|
+
log(`segment resolve failed (queue kept): ${error instanceof Error ? error.message : String(error)}`);
|
|
1820
|
+
return false;
|
|
1645
1821
|
}
|
|
1646
|
-
if (
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1822
|
+
if (entries.length === 0) return false;
|
|
1823
|
+
const state = readSegmentState();
|
|
1824
|
+
const plan = planSegment(entries, {
|
|
1825
|
+
currentSessionId: options.sessionId,
|
|
1826
|
+
markerAt: state.marker?.at ?? null,
|
|
1827
|
+
archived: Boolean(options.sessionId && (options.archivedSessions ?? []).includes(options.sessionId)),
|
|
1828
|
+
now: Date.now(),
|
|
1829
|
+
});
|
|
1830
|
+
if (!plan.trigger) {
|
|
1831
|
+
// A marker whose covered turns are all gone can never fire again (time only
|
|
1832
|
+
// moves forward); leaving it would look like a pending departure forever.
|
|
1833
|
+
if (state.marker) {
|
|
1834
|
+
writeSegmentState({ ...state, marker: null });
|
|
1835
|
+
log(`segment marker: nothing left to flush (at=${state.marker.at}), cleared`);
|
|
1836
|
+
}
|
|
1837
|
+
// One wait line per buffer change, not per step: the queue's own backlog log
|
|
1838
|
+
// covers writes, this one makes declaration 5 (no departure = no extraction)
|
|
1839
|
+
// observable in production.
|
|
1840
|
+
if (entries.length !== lastSegmentWait) {
|
|
1841
|
+
lastSegmentWait = entries.length;
|
|
1842
|
+
trace({ event: 'segment-wait', buffered: entries.length, session: options.sessionId ?? null });
|
|
1843
|
+
}
|
|
1844
|
+
return false;
|
|
1650
1845
|
}
|
|
1651
|
-
|
|
1652
|
-
const
|
|
1846
|
+
const taken = plan.entries;
|
|
1847
|
+
const head = taken[0];
|
|
1848
|
+
const last = taken[taken.length - 1];
|
|
1849
|
+
const chars = taken.reduce((sum, entry) => sum + entry.chars, 0);
|
|
1850
|
+
// Prompt numbering is 1..n inside the unit; turnIds maps it back to the real
|
|
1851
|
+
// turn, so a fact's 轮次 lands on source_turn_id.
|
|
1852
|
+
const turns = taken.map((entry, index) => ({
|
|
1853
|
+
id: entry.item.turnId,
|
|
1854
|
+
turnNo: index + 1,
|
|
1855
|
+
userText: entry.row.userText,
|
|
1856
|
+
assistantText: entry.row.assistantText,
|
|
1857
|
+
startedAt: entry.row.at,
|
|
1858
|
+
}));
|
|
1859
|
+
const origin = taken.every((entry) => (entry.item.origin ?? 'live') === 'backfill') ? 'backfill' : 'live';
|
|
1860
|
+
log(
|
|
1861
|
+
`segment flush: trigger=${plan.trigger} turns=${turns.length} chars=${chars}` +
|
|
1862
|
+
` head=${head.item.turnNo} origin=${origin}`,
|
|
1863
|
+
);
|
|
1864
|
+
const outcome = await runExtraction(last.row.userText, head.item.turnId, head.item.turnNo, origin, { turns });
|
|
1653
1865
|
const ok =
|
|
1654
1866
|
outcome === 'ok' || outcome === 'skipped-off' || outcome === 'skipped-empty' || outcome === 'rules-mode';
|
|
1655
1867
|
if (ok) {
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1868
|
+
// Promotion is decided after the facts exist: a corrected turn's lessons enter
|
|
1869
|
+
// the classification queue here, which is the only live producer it has.
|
|
1870
|
+
for (const { item } of taken) feedClassifyFromTurn(item.turnId);
|
|
1871
|
+
removeExtractItems(taken.map((entry) => entry.item.turnId));
|
|
1872
|
+
sealSegment({
|
|
1873
|
+
trigger: plan.trigger,
|
|
1874
|
+
turnIds: taken.map((entry) => entry.item.turnId),
|
|
1875
|
+
turns: taken.length,
|
|
1876
|
+
chars,
|
|
1877
|
+
outcome,
|
|
1878
|
+
session: head.sessionId,
|
|
1879
|
+
});
|
|
1880
|
+
lastSegmentWait = -1;
|
|
1881
|
+
trace({ event: 'extract-queue', drained: taken.length, remaining: readExtractQueue().length, outcome, trigger: plan.trigger });
|
|
1668
1882
|
return true;
|
|
1669
1883
|
}
|
|
1670
1884
|
|
|
1671
1885
|
// A rules result caused by a broken model call is not a completed extraction.
|
|
1672
|
-
const attempts = (item.attempts ?? 0) + 1;
|
|
1886
|
+
const attempts = (head.item.attempts ?? 0) + 1;
|
|
1887
|
+
const turnIds = taken.map((entry) => entry.item.turnId);
|
|
1673
1888
|
if (attempts >= EXTRACT_MAX_ATTEMPTS) {
|
|
1674
|
-
log(`extract queue: giving up on
|
|
1675
|
-
|
|
1889
|
+
log(`extract queue: giving up on segment trigger=${plan.trigger} turns=${taken.length} after ${attempts}: ${outcome}`);
|
|
1890
|
+
removeExtractItems(turnIds);
|
|
1891
|
+
sealSegment({ trigger: plan.trigger, turnIds, turns: taken.length, chars, outcome: `given-up:${outcome}`, session: head.sessionId });
|
|
1892
|
+
trace({ event: 'extract-queue', drained: taken.length, remaining: readExtractQueue().length, outcome, dropped: true, trigger: plan.trigger });
|
|
1676
1893
|
return true;
|
|
1677
1894
|
}
|
|
1678
|
-
log(`extract queue: retry turn=${item.turnNo} attempt=${attempts} (${outcome})`);
|
|
1679
1895
|
// Bump in place, computed from a fresh read: an item enqueued during the model
|
|
1680
1896
|
// call must not be overwritten by a stale queue snapshot.
|
|
1681
|
-
bumpExtractAttempt(item.turnId);
|
|
1682
|
-
|
|
1897
|
+
for (const { item } of taken) bumpExtractAttempt(item.turnId);
|
|
1898
|
+
log(`extract queue: retry segment trigger=${plan.trigger} attempt=${attempts} turns=${taken.length} (${outcome})`);
|
|
1899
|
+
trace({ event: 'extract-queue', drained: taken.length, remaining: readExtractQueue().length, outcome, retry: true, trigger: plan.trigger });
|
|
1683
1900
|
return true;
|
|
1684
1901
|
} finally {
|
|
1685
1902
|
draining = false;
|
|
@@ -2758,6 +2975,25 @@ export function apply(ctx) {
|
|
|
2758
2975
|
return;
|
|
2759
2976
|
}
|
|
2760
2977
|
|
|
2978
|
+
// 0028 决定 2: compaction is the host saying "this left the window". A marker
|
|
2979
|
+
// only — events never call the model (it is callable inside a step), and the
|
|
2980
|
+
// next drain flushes what the marker covers. An errored compaction changed
|
|
2981
|
+
// nothing, so it must leave no marker: a false departure would write durable
|
|
2982
|
+
// memory for content that is still in the context.
|
|
2983
|
+
if (type === 'compaction/end') {
|
|
2984
|
+
if (event.data?.error) {
|
|
2985
|
+
log(`segment marker: compaction failed (${event.data.error}), window unchanged`);
|
|
2986
|
+
} else {
|
|
2987
|
+
setSegmentMarker({
|
|
2988
|
+
trigger: 'compaction',
|
|
2989
|
+
session: sessionId,
|
|
2990
|
+
at: Date.now(),
|
|
2991
|
+
compactionId: event.data?.compactionId ?? null,
|
|
2992
|
+
});
|
|
2993
|
+
}
|
|
2994
|
+
return;
|
|
2995
|
+
}
|
|
2996
|
+
|
|
2761
2997
|
if (type !== 'user/message') return;
|
|
2762
2998
|
|
|
2763
2999
|
const data = event.data ?? {};
|