github-issue-tower-defence-management 1.140.1 → 1.141.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/umino-project.yml +2 -1
- package/CHANGELOG.md +14 -0
- package/README.md +8 -1
- package/bin/adapter/entry-points/handlers/HandleScheduledEventUseCaseHandler.js +43 -2
- package/bin/adapter/entry-points/handlers/HandleScheduledEventUseCaseHandler.js.map +1 -1
- package/bin/adapter/entry-points/handlers/resetDegeneratedTmuxSessions.js +32 -0
- package/bin/adapter/entry-points/handlers/resetDegeneratedTmuxSessions.js.map +1 -0
- package/bin/adapter/repositories/FileSystemSessionAssistantTurnsRepository.js +135 -0
- package/bin/adapter/repositories/FileSystemSessionAssistantTurnsRepository.js.map +1 -0
- package/bin/adapter/repositories/FileSystemSessionDegenerationCooldownStateRepository.js +120 -0
- package/bin/adapter/repositories/FileSystemSessionDegenerationCooldownStateRepository.js.map +1 -0
- package/bin/domain/entities/OutputDegeneration.js +3 -0
- package/bin/domain/entities/OutputDegeneration.js.map +1 -0
- package/bin/domain/usecases/NotifySilentLiveSessionsUseCase.js +20 -5
- package/bin/domain/usecases/NotifySilentLiveSessionsUseCase.js.map +1 -1
- package/bin/domain/usecases/OutputDegenerationDetector.js +150 -0
- package/bin/domain/usecases/OutputDegenerationDetector.js.map +1 -0
- package/bin/domain/usecases/SessionOutputDegenerationRecoveryUseCase.js +84 -0
- package/bin/domain/usecases/SessionOutputDegenerationRecoveryUseCase.js.map +1 -0
- package/bin/domain/usecases/adapter-interfaces/SessionAssistantTurnsRepository.js +3 -0
- package/bin/domain/usecases/adapter-interfaces/SessionAssistantTurnsRepository.js.map +1 -0
- package/bin/domain/usecases/adapter-interfaces/SessionDegenerationCooldownStateRepository.js +3 -0
- package/bin/domain/usecases/adapter-interfaces/SessionDegenerationCooldownStateRepository.js.map +1 -0
- package/package.json +1 -1
- package/src/adapter/entry-points/handlers/HandleScheduledEventUseCaseHandler.ts +44 -0
- package/src/adapter/entry-points/handlers/notifySilentTmuxSessions.test.ts +33 -3
- package/src/adapter/entry-points/handlers/resetDegeneratedTmuxSessions.test.ts +69 -0
- package/src/adapter/entry-points/handlers/resetDegeneratedTmuxSessions.ts +71 -0
- package/src/adapter/repositories/FileSystemSessionAssistantTurnsRepository.test.ts +138 -0
- package/src/adapter/repositories/FileSystemSessionAssistantTurnsRepository.ts +113 -0
- package/src/adapter/repositories/FileSystemSessionDegenerationCooldownStateRepository.test.ts +85 -0
- package/src/adapter/repositories/FileSystemSessionDegenerationCooldownStateRepository.ts +105 -0
- package/src/domain/entities/OutputDegeneration.ts +10 -0
- package/src/domain/usecases/NotifySilentLiveSessionsUseCase.test.ts +120 -8
- package/src/domain/usecases/NotifySilentLiveSessionsUseCase.ts +25 -9
- package/src/domain/usecases/OutputDegenerationDetector.test.ts +167 -0
- package/src/domain/usecases/OutputDegenerationDetector.ts +169 -0
- package/src/domain/usecases/SessionOutputDegenerationRecoveryUseCase.test.ts +224 -0
- package/src/domain/usecases/SessionOutputDegenerationRecoveryUseCase.ts +142 -0
- package/src/domain/usecases/adapter-interfaces/SessionAssistantTurnsRepository.ts +6 -0
- package/src/domain/usecases/adapter-interfaces/SessionDegenerationCooldownStateRepository.ts +4 -0
- package/types/adapter/entry-points/handlers/HandleScheduledEventUseCaseHandler.d.ts.map +1 -1
- package/types/adapter/entry-points/handlers/resetDegeneratedTmuxSessions.d.ts +19 -0
- package/types/adapter/entry-points/handlers/resetDegeneratedTmuxSessions.d.ts.map +1 -0
- package/types/adapter/repositories/FileSystemSessionAssistantTurnsRepository.d.ts +10 -0
- package/types/adapter/repositories/FileSystemSessionAssistantTurnsRepository.d.ts.map +1 -0
- package/types/adapter/repositories/FileSystemSessionDegenerationCooldownStateRepository.d.ts +15 -0
- package/types/adapter/repositories/FileSystemSessionDegenerationCooldownStateRepository.d.ts.map +1 -0
- package/types/domain/entities/OutputDegeneration.d.ts +10 -0
- package/types/domain/entities/OutputDegeneration.d.ts.map +1 -0
- package/types/domain/usecases/NotifySilentLiveSessionsUseCase.d.ts.map +1 -1
- package/types/domain/usecases/OutputDegenerationDetector.d.ts +32 -0
- package/types/domain/usecases/OutputDegenerationDetector.d.ts.map +1 -0
- package/types/domain/usecases/SessionOutputDegenerationRecoveryUseCase.d.ts +31 -0
- package/types/domain/usecases/SessionOutputDegenerationRecoveryUseCase.d.ts.map +1 -0
- package/types/domain/usecases/adapter-interfaces/SessionAssistantTurnsRepository.d.ts +4 -0
- package/types/domain/usecases/adapter-interfaces/SessionAssistantTurnsRepository.d.ts.map +1 -0
- package/types/domain/usecases/adapter-interfaces/SessionDegenerationCooldownStateRepository.d.ts +8 -0
- package/types/domain/usecases/adapter-interfaces/SessionDegenerationCooldownStateRepository.d.ts.map +1 -0
|
@@ -331,7 +331,7 @@ describe('NotifySilentLiveSessionsUseCase', () => {
|
|
|
331
331
|
).toHaveBeenCalledWith(pullRequestSession, MAIN_STALLED_SECTION);
|
|
332
332
|
});
|
|
333
333
|
|
|
334
|
-
it('excludes a non-github-named session
|
|
334
|
+
it('excludes a non-github-named session that has no resolvable transcript so it is never notified', async () => {
|
|
335
335
|
mockSnapshotProvider.getSnapshot.mockResolvedValue(
|
|
336
336
|
snapshotWithSessions(['workbench']),
|
|
337
337
|
);
|
|
@@ -347,9 +347,9 @@ describe('NotifySilentLiveSessionsUseCase', () => {
|
|
|
347
347
|
|
|
348
348
|
await useCase.run(runParams());
|
|
349
349
|
|
|
350
|
-
expect(mockTranscriptResolver.resolveTranscriptPaths).toHaveBeenCalledWith(
|
|
351
|
-
|
|
352
|
-
);
|
|
350
|
+
expect(mockTranscriptResolver.resolveTranscriptPaths).toHaveBeenCalledWith([
|
|
351
|
+
sessionFor('workbench'),
|
|
352
|
+
]);
|
|
353
353
|
expect(
|
|
354
354
|
mockSubAgentActivityRepository.listSubAgentActivitiesBySessionName,
|
|
355
355
|
).toHaveBeenCalledWith([], new Map());
|
|
@@ -361,7 +361,7 @@ describe('NotifySilentLiveSessionsUseCase', () => {
|
|
|
361
361
|
).not.toHaveBeenCalled();
|
|
362
362
|
});
|
|
363
363
|
|
|
364
|
-
it('monitors
|
|
364
|
+
it('monitors the github-named session and excludes a non-github-named session that has no resolvable transcript', async () => {
|
|
365
365
|
mockSnapshotProvider.getSnapshot.mockResolvedValue(
|
|
366
366
|
snapshotWithSessions([GITHUB_SESSION, 'orchestrator']),
|
|
367
367
|
);
|
|
@@ -387,6 +387,7 @@ describe('NotifySilentLiveSessionsUseCase', () => {
|
|
|
387
387
|
|
|
388
388
|
expect(mockTranscriptResolver.resolveTranscriptPaths).toHaveBeenCalledWith([
|
|
389
389
|
sessionFor(GITHUB_SESSION),
|
|
390
|
+
sessionFor('orchestrator'),
|
|
390
391
|
]);
|
|
391
392
|
expect(
|
|
392
393
|
mockNotificationRepository.sendSelfCheckNotification,
|
|
@@ -691,6 +692,117 @@ describe('NotifySilentLiveSessionsUseCase', () => {
|
|
|
691
692
|
).not.toHaveBeenCalled();
|
|
692
693
|
});
|
|
693
694
|
|
|
695
|
+
describe('role-named resident leader and PM agent sessions', () => {
|
|
696
|
+
it('reminds a role-named leader agent session that has a resolvable transcript and is silent past the threshold', async () => {
|
|
697
|
+
setupLiveInteractiveSession('secretary');
|
|
698
|
+
mockSessionOutputActivityRepository.listSessionOutputActivities.mockResolvedValue(
|
|
699
|
+
[
|
|
700
|
+
{
|
|
701
|
+
sessionName: 'secretary',
|
|
702
|
+
lastOutputEpochSeconds:
|
|
703
|
+
nowEpochSeconds - DEFAULT_MAIN_SILENT_THRESHOLD_SECONDS,
|
|
704
|
+
},
|
|
705
|
+
],
|
|
706
|
+
);
|
|
707
|
+
|
|
708
|
+
await useCase.run(runParams());
|
|
709
|
+
|
|
710
|
+
expect(
|
|
711
|
+
mockMessageComposer.composeMainStalledSection,
|
|
712
|
+
).toHaveBeenCalledWith(DEFAULT_MAIN_SILENT_THRESHOLD_SECONDS);
|
|
713
|
+
expect(
|
|
714
|
+
mockNotificationRepository.sendSelfCheckNotification,
|
|
715
|
+
).toHaveBeenCalledWith('secretary', MAIN_STALLED_SECTION);
|
|
716
|
+
});
|
|
717
|
+
|
|
718
|
+
it('excludes a non-agent interactive session with no resolvable transcript (for example a viewer named sso_login)', async () => {
|
|
719
|
+
mockSnapshotProvider.getSnapshot.mockResolvedValue(
|
|
720
|
+
snapshotWithSessions(['sso_login']),
|
|
721
|
+
);
|
|
722
|
+
mockSessionOutputActivityRepository.listSessionOutputActivities.mockResolvedValue(
|
|
723
|
+
[
|
|
724
|
+
{
|
|
725
|
+
sessionName: 'sso_login',
|
|
726
|
+
lastOutputEpochSeconds:
|
|
727
|
+
nowEpochSeconds - DEFAULT_MAIN_SILENT_THRESHOLD_SECONDS,
|
|
728
|
+
},
|
|
729
|
+
],
|
|
730
|
+
);
|
|
731
|
+
|
|
732
|
+
await useCase.run(runParams());
|
|
733
|
+
|
|
734
|
+
expect(
|
|
735
|
+
mockTranscriptResolver.resolveTranscriptPaths,
|
|
736
|
+
).toHaveBeenCalledWith([sessionFor('sso_login')]);
|
|
737
|
+
expect(
|
|
738
|
+
mockMessageComposer.composeMainStalledSection,
|
|
739
|
+
).not.toHaveBeenCalled();
|
|
740
|
+
expect(
|
|
741
|
+
mockNotificationRepository.sendSelfCheckNotification,
|
|
742
|
+
).not.toHaveBeenCalled();
|
|
743
|
+
});
|
|
744
|
+
|
|
745
|
+
it('reminds both a github-named session and a role-named leader session in the same cycle', async () => {
|
|
746
|
+
mockSnapshotProvider.getSnapshot.mockResolvedValue(
|
|
747
|
+
snapshotWithSessions([GITHUB_SESSION, 'app']),
|
|
748
|
+
);
|
|
749
|
+
mockTranscriptResolver.resolveTranscriptPaths.mockReturnValue(
|
|
750
|
+
transcriptMapFor([GITHUB_SESSION, 'app']),
|
|
751
|
+
);
|
|
752
|
+
mockSessionOutputActivityRepository.listSessionOutputActivities.mockResolvedValue(
|
|
753
|
+
[
|
|
754
|
+
{
|
|
755
|
+
sessionName: GITHUB_SESSION,
|
|
756
|
+
lastOutputEpochSeconds:
|
|
757
|
+
nowEpochSeconds - DEFAULT_MAIN_SILENT_THRESHOLD_SECONDS,
|
|
758
|
+
},
|
|
759
|
+
{
|
|
760
|
+
sessionName: 'app',
|
|
761
|
+
lastOutputEpochSeconds:
|
|
762
|
+
nowEpochSeconds - DEFAULT_MAIN_SILENT_THRESHOLD_SECONDS,
|
|
763
|
+
},
|
|
764
|
+
],
|
|
765
|
+
);
|
|
766
|
+
|
|
767
|
+
await useCase.run(runParams());
|
|
768
|
+
|
|
769
|
+
expect(
|
|
770
|
+
mockNotificationRepository.sendSelfCheckNotification,
|
|
771
|
+
).toHaveBeenCalledTimes(2);
|
|
772
|
+
expect(
|
|
773
|
+
mockNotificationRepository.sendSelfCheckNotification,
|
|
774
|
+
).toHaveBeenCalledWith(GITHUB_SESSION, MAIN_STALLED_SECTION);
|
|
775
|
+
expect(
|
|
776
|
+
mockNotificationRepository.sendSelfCheckNotification,
|
|
777
|
+
).toHaveBeenCalledWith('app', MAIN_STALLED_SECTION);
|
|
778
|
+
});
|
|
779
|
+
|
|
780
|
+
it('parks a role-named leader session while its latest owner call is unanswered', async () => {
|
|
781
|
+
setupLiveInteractiveSession('uminopm');
|
|
782
|
+
mockSessionOutputActivityRepository.listSessionOutputActivities.mockResolvedValue(
|
|
783
|
+
[
|
|
784
|
+
{
|
|
785
|
+
sessionName: 'uminopm',
|
|
786
|
+
lastOutputEpochSeconds:
|
|
787
|
+
nowEpochSeconds - DEFAULT_MAIN_SILENT_THRESHOLD_SECONDS,
|
|
788
|
+
},
|
|
789
|
+
],
|
|
790
|
+
);
|
|
791
|
+
mockOwnerCallStatusProvider.listUnansweredOwnerCallEpochSecondsBySessionName.mockResolvedValue(
|
|
792
|
+
new Map([['uminopm', nowEpochSeconds - 60]]),
|
|
793
|
+
);
|
|
794
|
+
|
|
795
|
+
await useCase.run(runParams());
|
|
796
|
+
|
|
797
|
+
expect(
|
|
798
|
+
mockMessageComposer.composeMainStalledSection,
|
|
799
|
+
).not.toHaveBeenCalled();
|
|
800
|
+
expect(
|
|
801
|
+
mockNotificationRepository.sendSelfCheckNotification,
|
|
802
|
+
).not.toHaveBeenCalled();
|
|
803
|
+
});
|
|
804
|
+
});
|
|
805
|
+
|
|
694
806
|
it('notifies a persistent stall only once per silent episode instead of every cycle', async () => {
|
|
695
807
|
wireStatefulNotifiedLatch();
|
|
696
808
|
setupLiveInteractiveSession(GITHUB_SESSION);
|
|
@@ -1221,15 +1333,15 @@ describe('NotifySilentLiveSessionsUseCase', () => {
|
|
|
1221
1333
|
).not.toHaveBeenCalled();
|
|
1222
1334
|
});
|
|
1223
1335
|
|
|
1224
|
-
it('
|
|
1225
|
-
setupSilentMainSession('
|
|
1336
|
+
it('does not consult the hub-task resolver for a role-named leader session that has no hub task URL, and still reminds it', async () => {
|
|
1337
|
+
setupSilentMainSession('tdpm-cli');
|
|
1226
1338
|
|
|
1227
1339
|
await useCase.run(runParams({ activeHubTaskStatus: ACTIVE_STATUS }));
|
|
1228
1340
|
|
|
1229
1341
|
expect(mockHubTaskStatusResolver.getIssueByUrl).not.toHaveBeenCalled();
|
|
1230
1342
|
expect(
|
|
1231
1343
|
mockNotificationRepository.sendSelfCheckNotification,
|
|
1232
|
-
).
|
|
1344
|
+
).toHaveBeenCalledWith('tdpm-cli', MAIN_STALLED_SECTION);
|
|
1233
1345
|
});
|
|
1234
1346
|
|
|
1235
1347
|
it('does not call the resolver at all when the active status is unconfigured', async () => {
|
|
@@ -99,20 +99,36 @@ export class NotifySilentLiveSessionsUseCase {
|
|
|
99
99
|
await this.liveSessionProcessSnapshotProvider.getSnapshot();
|
|
100
100
|
const allInteractiveSessions =
|
|
101
101
|
this.resolveInteractiveLiveSessions.resolve(snapshot);
|
|
102
|
-
|
|
103
|
-
|
|
102
|
+
// Resolve the on-disk Claude transcript for every interactive session
|
|
103
|
+
// before any name-based narrowing. A resolvable transcript is the proof
|
|
104
|
+
// that a session is a live Claude agent session (its transcript, keyed by
|
|
105
|
+
// the session id, is actively present on disk), independent of how the tmux
|
|
106
|
+
// session is named.
|
|
107
|
+
const transcriptPathBySessionName =
|
|
108
|
+
this.interactiveLiveSessionTranscriptResolver.resolveTranscriptPaths(
|
|
109
|
+
allInteractiveSessions,
|
|
110
|
+
);
|
|
111
|
+
// Monitor a session when it is either named after a github.com issue or
|
|
112
|
+
// pull-request URL (the hub-task sessions) or has a resolvable Claude agent
|
|
113
|
+
// transcript (role-named resident leader / PM agent sessions such as app,
|
|
114
|
+
// tdpm-cli, secretary, and the per-project *pm sessions). Gating the
|
|
115
|
+
// broadened inclusion on the resolvable transcript rather than on merely
|
|
116
|
+
// being a tmux session keeps the fail-direction safe: a genuine non-agent
|
|
117
|
+
// interactive session — a login shell, or a viewer such as sso_login or a
|
|
118
|
+
// tdpm viewer — exposes no Claude transcript on disk, so it is still
|
|
119
|
+
// excluded and never reminded.
|
|
120
|
+
const interactiveSessions = allInteractiveSessions.filter(
|
|
121
|
+
(session) =>
|
|
122
|
+
isGitHubIssueOrPullRequestSessionName(session.sessionName) ||
|
|
123
|
+
transcriptPathBySessionName.has(session.sessionName),
|
|
104
124
|
);
|
|
105
|
-
const
|
|
125
|
+
const skippedNonAgentSessionCount =
|
|
106
126
|
allInteractiveSessions.length - interactiveSessions.length;
|
|
107
|
-
if (
|
|
127
|
+
if (skippedNonAgentSessionCount > 0) {
|
|
108
128
|
console.log(
|
|
109
|
-
`Silent live session notification: ignoring ${
|
|
129
|
+
`Silent live session notification: ignoring ${skippedNonAgentSessionCount} interactive session(s) that are neither named after a github.com issue or pull-request URL nor have a resolvable Claude agent transcript.`,
|
|
110
130
|
);
|
|
111
131
|
}
|
|
112
|
-
const transcriptPathBySessionName =
|
|
113
|
-
this.interactiveLiveSessionTranscriptResolver.resolveTranscriptPaths(
|
|
114
|
-
interactiveSessions,
|
|
115
|
-
);
|
|
116
132
|
|
|
117
133
|
// A session whose most recent assistant turn is a model refusal is
|
|
118
134
|
// excluded from ALL reminder candidates (main-stall and sub-agent
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import {
|
|
2
|
+
OutputDegenerationDetector,
|
|
3
|
+
OUTPUT_DEGENERATION_REPEAT_THRESHOLD,
|
|
4
|
+
OUTPUT_DEGENERATION_DOMINATION_FRACTION,
|
|
5
|
+
OUTPUT_DEGENERATION_MAX_TOKEN_LENGTH,
|
|
6
|
+
OUTPUT_DEGENERATION_ABSOLUTE_RUN_THRESHOLD,
|
|
7
|
+
OUTPUT_DEGENERATION_REP4_THRESHOLD,
|
|
8
|
+
OUTPUT_DEGENERATION_REP4_MIN_TOKENS,
|
|
9
|
+
OUTPUT_DEGENERATION_CROSS_TURN_WINDOW,
|
|
10
|
+
OUTPUT_DEGENERATION_CROSS_TURN_MIN_TURNS,
|
|
11
|
+
} from './OutputDegenerationDetector';
|
|
12
|
+
|
|
13
|
+
const repeat = (token: string, count: number): string =>
|
|
14
|
+
Array.from({ length: count }, () => token).join(' ');
|
|
15
|
+
|
|
16
|
+
describe('OutputDegenerationDetector', () => {
|
|
17
|
+
const detector = new OutputDegenerationDetector();
|
|
18
|
+
|
|
19
|
+
describe('ported constants match the validated Python values', () => {
|
|
20
|
+
it('exposes the exact thresholds', () => {
|
|
21
|
+
expect(OUTPUT_DEGENERATION_REPEAT_THRESHOLD).toBe(5);
|
|
22
|
+
expect(OUTPUT_DEGENERATION_DOMINATION_FRACTION).toBe(0.8);
|
|
23
|
+
expect(OUTPUT_DEGENERATION_MAX_TOKEN_LENGTH).toBe(32);
|
|
24
|
+
expect(OUTPUT_DEGENERATION_ABSOLUTE_RUN_THRESHOLD).toBe(15);
|
|
25
|
+
expect(OUTPUT_DEGENERATION_REP4_THRESHOLD).toBe(0.3);
|
|
26
|
+
expect(OUTPUT_DEGENERATION_REP4_MIN_TOKENS).toBe(40);
|
|
27
|
+
expect(OUTPUT_DEGENERATION_CROSS_TURN_WINDOW).toBe(10);
|
|
28
|
+
expect(OUTPUT_DEGENERATION_CROSS_TURN_MIN_TURNS).toBe(4);
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
describe('isIntraTurnDegeneration', () => {
|
|
33
|
+
it('fires on a turn dominated by a single short repeated token (positive)', () => {
|
|
34
|
+
expect(detector.isIntraTurnDegeneration(repeat('court', 40))).toBe(true);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('does not fire on a short run just below the repeat threshold (negative)', () => {
|
|
38
|
+
expect(detector.isIntraTurnDegeneration(repeat('court', 4))).toBe(false);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it('does not fire on healthy prose with no repetition', () => {
|
|
42
|
+
const healthy =
|
|
43
|
+
'I finished the migration and verified the results against the seed data. The build passed and the report is uploaded.';
|
|
44
|
+
expect(detector.isIntraTurnDegeneration(healthy)).toBe(false);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it('fires exactly at the domination-fraction boundary (run 5 of 6 tokens)', () => {
|
|
48
|
+
expect(
|
|
49
|
+
detector.isIntraTurnDegeneration('count count count count count alpha'),
|
|
50
|
+
).toBe(true);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('does not fire just past the domination-fraction boundary (run 5 of 7 tokens)', () => {
|
|
54
|
+
expect(
|
|
55
|
+
detector.isIntraTurnDegeneration(
|
|
56
|
+
'count count count count count alpha beta',
|
|
57
|
+
),
|
|
58
|
+
).toBe(false);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('fires on a large absolute run even when domination fails', () => {
|
|
62
|
+
const buried = `${repeat('court', 15)} ${repeat('distinct', 1)} one two three four five`;
|
|
63
|
+
expect(detector.isIntraTurnDegeneration(buried)).toBe(true);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('fires on a repeated phrase loop via duplicated 4-gram density', () => {
|
|
67
|
+
const phraseLoop = repeat('the quick brown fox', 15);
|
|
68
|
+
expect(detector.isIntraTurnDegeneration(phraseLoop)).toBe(true);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('does not fire on a non-dominating repeat run confined to a fenced code block', () => {
|
|
72
|
+
const variedProse = Array.from(
|
|
73
|
+
{ length: 100 },
|
|
74
|
+
(_unused, index) => `word${index}`,
|
|
75
|
+
).join(' ');
|
|
76
|
+
const withCodeBlock = `${variedProse}\n\`\`\`\n${repeat('court', 20)}\n\`\`\``;
|
|
77
|
+
expect(detector.isIntraTurnDegeneration(withCodeBlock)).toBe(false);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('ignores repeated long tokens above the max token length', () => {
|
|
81
|
+
const longToken = 'a'.repeat(OUTPUT_DEGENERATION_MAX_TOKEN_LENGTH + 1);
|
|
82
|
+
expect(detector.isIntraTurnDegeneration(repeat(longToken, 40))).toBe(
|
|
83
|
+
false,
|
|
84
|
+
);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('returns false for an empty turn', () => {
|
|
88
|
+
expect(detector.isIntraTurnDegeneration('')).toBe(false);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
describe('maxConsecutiveShortTokenRun', () => {
|
|
93
|
+
it('reports the dominating token and run length for logging', () => {
|
|
94
|
+
const result = detector.maxConsecutiveShortTokenRun(repeat('court', 12));
|
|
95
|
+
expect(result.token).toBe('court');
|
|
96
|
+
expect(result.run).toBe(12);
|
|
97
|
+
expect(result.total).toBe(12);
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
describe('detectCrossTurnDegeneration', () => {
|
|
102
|
+
const cleanEnding = (index: number): string =>
|
|
103
|
+
`Turn ${index}: I completed the step and reported the result.`;
|
|
104
|
+
const trailingTokenTurn = (index: number, token: string): string =>
|
|
105
|
+
`Turn ${index}: real message here.\n\n${token}`;
|
|
106
|
+
|
|
107
|
+
it('fires when the same trailing token recurs in 4 of the last 10 turns', () => {
|
|
108
|
+
const turns = [
|
|
109
|
+
trailingTokenTurn(1, 'court'),
|
|
110
|
+
cleanEnding(2),
|
|
111
|
+
trailingTokenTurn(3, 'court'),
|
|
112
|
+
cleanEnding(4),
|
|
113
|
+
trailingTokenTurn(5, 'court'),
|
|
114
|
+
cleanEnding(6),
|
|
115
|
+
trailingTokenTurn(7, 'court'),
|
|
116
|
+
cleanEnding(8),
|
|
117
|
+
cleanEnding(9),
|
|
118
|
+
cleanEnding(10),
|
|
119
|
+
];
|
|
120
|
+
const result = detector.detectCrossTurnDegeneration(turns);
|
|
121
|
+
expect(result).not.toBeNull();
|
|
122
|
+
expect(result?.token).toBe('court');
|
|
123
|
+
expect(result?.turnCount).toBe(4);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('does not fire when the trailing token recurs in only 3 of 10 turns', () => {
|
|
127
|
+
const turns = [
|
|
128
|
+
trailingTokenTurn(1, 'court'),
|
|
129
|
+
cleanEnding(2),
|
|
130
|
+
trailingTokenTurn(3, 'court'),
|
|
131
|
+
cleanEnding(4),
|
|
132
|
+
trailingTokenTurn(5, 'court'),
|
|
133
|
+
cleanEnding(6),
|
|
134
|
+
cleanEnding(7),
|
|
135
|
+
cleanEnding(8),
|
|
136
|
+
cleanEnding(9),
|
|
137
|
+
cleanEnding(10),
|
|
138
|
+
];
|
|
139
|
+
expect(detector.detectCrossTurnDegeneration(turns)).toBeNull();
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it('does not treat a bare URL or multi-word ending as a trailing token', () => {
|
|
143
|
+
const turns = [
|
|
144
|
+
'Done. See https://example.com/report',
|
|
145
|
+
'The work is complete',
|
|
146
|
+
'Result: 42',
|
|
147
|
+
'Done. See https://example.com/report',
|
|
148
|
+
];
|
|
149
|
+
expect(detector.detectCrossTurnDegeneration(turns)).toBeNull();
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it('only counts turns inside the cross-turn window', () => {
|
|
153
|
+
const withinWindow = Array.from({ length: 10 }, (_unused, index) =>
|
|
154
|
+
index < 3 ? trailingTokenTurn(index, 'court') : cleanEnding(index),
|
|
155
|
+
);
|
|
156
|
+
const beyondWindow = Array.from({ length: 5 }, (_unused, index) =>
|
|
157
|
+
trailingTokenTurn(100 + index, 'court'),
|
|
158
|
+
);
|
|
159
|
+
expect(
|
|
160
|
+
detector.detectCrossTurnDegeneration([
|
|
161
|
+
...withinWindow,
|
|
162
|
+
...beyondWindow,
|
|
163
|
+
]),
|
|
164
|
+
).toBeNull();
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
});
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { CrossTurnDegeneration } from '../entities/OutputDegeneration';
|
|
2
|
+
|
|
3
|
+
export const OUTPUT_DEGENERATION_REPEAT_THRESHOLD = 5;
|
|
4
|
+
export const OUTPUT_DEGENERATION_DOMINATION_FRACTION = 0.8;
|
|
5
|
+
export const OUTPUT_DEGENERATION_MAX_TOKEN_LENGTH = 32;
|
|
6
|
+
export const OUTPUT_DEGENERATION_ABSOLUTE_RUN_THRESHOLD = 15;
|
|
7
|
+
export const OUTPUT_DEGENERATION_REP4_THRESHOLD = 0.3;
|
|
8
|
+
export const OUTPUT_DEGENERATION_REP4_MIN_TOKENS = 40;
|
|
9
|
+
export const OUTPUT_DEGENERATION_CROSS_TURN_WINDOW = 10;
|
|
10
|
+
export const OUTPUT_DEGENERATION_CROSS_TURN_MIN_TURNS = 4;
|
|
11
|
+
|
|
12
|
+
export type ShortTokenRun = {
|
|
13
|
+
token: string | null;
|
|
14
|
+
run: number;
|
|
15
|
+
total: number;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const LIST_ITEM_LINE_PATTERN = /^\s*(?:[-*+]\s|\d+[.)]\s)/;
|
|
19
|
+
const TRAILING_ALPHA_TOKEN_PATTERN = /^[A-Za-z]+$/;
|
|
20
|
+
|
|
21
|
+
const splitTokens = (text: string): string[] =>
|
|
22
|
+
text.split(/\s+/).filter((token) => token.length > 0);
|
|
23
|
+
|
|
24
|
+
const splitLines = (text: string): string[] => text.split(/\r\n|\r|\n/);
|
|
25
|
+
|
|
26
|
+
export class OutputDegenerationDetector {
|
|
27
|
+
constructor(
|
|
28
|
+
private readonly repeatThreshold: number = OUTPUT_DEGENERATION_REPEAT_THRESHOLD,
|
|
29
|
+
private readonly dominationFraction: number = OUTPUT_DEGENERATION_DOMINATION_FRACTION,
|
|
30
|
+
private readonly maxTokenLength: number = OUTPUT_DEGENERATION_MAX_TOKEN_LENGTH,
|
|
31
|
+
private readonly absoluteRunThreshold: number = OUTPUT_DEGENERATION_ABSOLUTE_RUN_THRESHOLD,
|
|
32
|
+
private readonly rep4Threshold: number = OUTPUT_DEGENERATION_REP4_THRESHOLD,
|
|
33
|
+
private readonly rep4MinTokens: number = OUTPUT_DEGENERATION_REP4_MIN_TOKENS,
|
|
34
|
+
private readonly crossTurnWindow: number = OUTPUT_DEGENERATION_CROSS_TURN_WINDOW,
|
|
35
|
+
private readonly crossTurnMinTurns: number = OUTPUT_DEGENERATION_CROSS_TURN_MIN_TURNS,
|
|
36
|
+
) {}
|
|
37
|
+
|
|
38
|
+
maxConsecutiveShortTokenRun = (text: string): ShortTokenRun => {
|
|
39
|
+
let bestToken: string | null = null;
|
|
40
|
+
let bestRun = 0;
|
|
41
|
+
let totalTokens = 0;
|
|
42
|
+
let currentToken: string | null = null;
|
|
43
|
+
let currentRun = 0;
|
|
44
|
+
for (const token of splitTokens(text)) {
|
|
45
|
+
totalTokens += 1;
|
|
46
|
+
if (token.length > this.maxTokenLength) {
|
|
47
|
+
currentToken = null;
|
|
48
|
+
currentRun = 0;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (token === currentToken) {
|
|
52
|
+
currentRun += 1;
|
|
53
|
+
} else {
|
|
54
|
+
currentToken = token;
|
|
55
|
+
currentRun = 1;
|
|
56
|
+
}
|
|
57
|
+
if (currentRun > bestRun) {
|
|
58
|
+
bestRun = currentRun;
|
|
59
|
+
bestToken = currentToken;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return { token: bestToken, run: bestRun, total: totalTokens };
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
isIntraTurnDegeneration = (text: string): boolean => {
|
|
66
|
+
const { run, total } = this.maxConsecutiveShortTokenRun(text);
|
|
67
|
+
if (total === 0) {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
if (run >= this.repeatThreshold && run >= this.dominationFraction * total) {
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
const filtered = this.stripRepetitionExemptSpans(text);
|
|
74
|
+
const filteredRun = this.maxConsecutiveShortTokenRun(filtered);
|
|
75
|
+
if (filteredRun.run >= this.absoluteRunThreshold) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
if (
|
|
79
|
+
filteredRun.total >= this.rep4MinTokens &&
|
|
80
|
+
this.duplicateNgramFraction(filtered, 4) >= this.rep4Threshold
|
|
81
|
+
) {
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
return false;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
detectCrossTurnDegeneration = (
|
|
88
|
+
texts: string[],
|
|
89
|
+
): CrossTurnDegeneration | null => {
|
|
90
|
+
const counts = new Map<string, number>();
|
|
91
|
+
for (const text of texts.slice(0, this.crossTurnWindow)) {
|
|
92
|
+
const token = this.trailingSpuriousToken(text);
|
|
93
|
+
if (token === null) {
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
97
|
+
}
|
|
98
|
+
let bestToken: string | null = null;
|
|
99
|
+
let bestCount = 0;
|
|
100
|
+
for (const [token, count] of counts) {
|
|
101
|
+
if (count > bestCount) {
|
|
102
|
+
bestCount = count;
|
|
103
|
+
bestToken = token;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
if (bestToken !== null && bestCount >= this.crossTurnMinTurns) {
|
|
107
|
+
return { token: bestToken, turnCount: bestCount };
|
|
108
|
+
}
|
|
109
|
+
return null;
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
private stripRepetitionExemptSpans = (text: string): string => {
|
|
113
|
+
const keptLines: string[] = [];
|
|
114
|
+
let inFence = false;
|
|
115
|
+
for (const line of splitLines(text)) {
|
|
116
|
+
const stripped = line.trim();
|
|
117
|
+
if (stripped.startsWith('```') || stripped.startsWith('~~~')) {
|
|
118
|
+
inFence = !inFence;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
if (inFence) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (line.includes('|')) {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (LIST_ITEM_LINE_PATTERN.test(line)) {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
keptLines.push(line);
|
|
131
|
+
}
|
|
132
|
+
return keptLines.join('\n');
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
private duplicateNgramFraction = (text: string, n: number): number => {
|
|
136
|
+
const tokens = splitTokens(text).filter(
|
|
137
|
+
(token) => token.length <= this.maxTokenLength,
|
|
138
|
+
);
|
|
139
|
+
if (tokens.length < n + 1) {
|
|
140
|
+
return 0;
|
|
141
|
+
}
|
|
142
|
+
const ngrams: string[] = [];
|
|
143
|
+
for (let index = 0; index <= tokens.length - n; index += 1) {
|
|
144
|
+
ngrams.push(tokens.slice(index, index + n).join('\u0000'));
|
|
145
|
+
}
|
|
146
|
+
const total = ngrams.length;
|
|
147
|
+
if (total < 2) {
|
|
148
|
+
return 0;
|
|
149
|
+
}
|
|
150
|
+
const unique = new Set(ngrams).size;
|
|
151
|
+
return 1 - unique / total;
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
private trailingSpuriousToken = (text: string): string | null => {
|
|
155
|
+
const stripped = text.replace(/\s+$/, '');
|
|
156
|
+
if (stripped.length === 0) {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
const lines = splitLines(stripped);
|
|
160
|
+
const lastLine = lines[lines.length - 1].trim();
|
|
161
|
+
if (lastLine.length === 0 || lastLine.length > this.maxTokenLength) {
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
if (!TRAILING_ALPHA_TOKEN_PATTERN.test(lastLine)) {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
return lastLine;
|
|
168
|
+
};
|
|
169
|
+
}
|