@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-esbuild-bundle-worker-20260807182016

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +3472 -6
  2. package/LICENSE.md +15 -0
  3. package/README.md +1 -1
  4. package/docs/INVESTIGATION_WORKFLOW.md +314 -0
  5. package/lint-staged.config.mjs +3 -0
  6. package/package.json +40 -33
  7. package/scripts/download.ts +22 -1
  8. package/scripts/generate-wm-templates.ts +2 -1
  9. package/src/cli.ts +967 -155
  10. package/src/commands/__tests__/prepare.test.ts +11 -6
  11. package/src/commands/__tests__/run.test.ts +10 -8
  12. package/src/commands/clean.ts +147 -0
  13. package/src/commands/deterministic-ids.ts +126 -0
  14. package/src/commands/find-prohibited.ts +300 -0
  15. package/src/commands/investigate.ts +2437 -0
  16. package/src/commands/list-partial.ts +137 -0
  17. package/src/commands/obscure-thread-ids.ts +237 -0
  18. package/src/commands/partial-results.ts +198 -0
  19. package/src/commands/precompute-embeddings.ts +302 -0
  20. package/src/commands/prepare.ts +732 -105
  21. package/src/commands/reconcile.ts +383 -0
  22. package/src/commands/run.ts +1240 -121
  23. package/src/commands/sessions.ts +126 -0
  24. package/src/commands/sync.ts +152 -0
  25. package/src/commands/test-cleaned.ts +54 -0
  26. package/src/commands/test-incremental.ts +121 -0
  27. package/src/commands/test-messages.ts +96 -0
  28. package/src/commands/test-prohibited.ts +245 -0
  29. package/src/commands/test-thread.ts +137 -0
  30. package/src/commands/tokens.ts +625 -0
  31. package/src/config.ts +1253 -72
  32. package/src/data/types.ts +48 -6
  33. package/src/embeddings/cached-openai-embedding-model.ts +0 -1
  34. package/src/evaluation/longmemeval-metric.ts +186 -103
  35. package/src/processors/content-sanitizer.ts +153 -0
  36. package/src/processors/date-injector.ts +111 -0
  37. package/src/processors/index.ts +3 -0
  38. package/src/processors/observation-semantic-filter.ts +1142 -0
  39. package/src/retry-model.ts +5 -3
  40. package/src/storage/__tests__/benchmark-store.test.ts +18 -35
  41. package/src/storage/benchmark-store.ts +140 -153
  42. package/src/storage/benchmark-vector.ts +1 -1
  43. package/src/storage/index.ts +1 -0
  44. package/src/storage/persistable-inmemory.ts +133 -0
  45. package/src/test-utils/mock-model.ts +35 -0
@@ -15,9 +15,10 @@ vi.mock('@ai-sdk/openai', () => ({
15
15
  }));
16
16
 
17
17
  // Mock the DatasetLoader
18
- vi.mock('../../data/loader', () => ({
19
- DatasetLoader: vi.fn().mockImplementation(() => ({
20
- loadDataset: vi.fn().mockResolvedValue([
18
+ vi.mock('../../data/loader', () => {
19
+ // Use a class for constructor (Vitest v4 requirement)
20
+ class MockDatasetLoader {
21
+ loadDataset = vi.fn().mockResolvedValue([
21
22
  {
22
23
  question_id: 'test-q1',
23
24
  question_type: 'single-session-user',
@@ -54,9 +55,13 @@ vi.mock('../../data/loader', () => ({
54
55
  ],
55
56
  answer_session_ids: ['session-3'],
56
57
  },
57
- ] as LongMemEvalQuestion[]),
58
- })),
59
- }));
58
+ ] as LongMemEvalQuestion[]);
59
+ }
60
+
61
+ return {
62
+ DatasetLoader: MockDatasetLoader,
63
+ };
64
+ });
60
65
 
61
66
  // Mock chalk and ora to avoid console output in tests
62
67
  vi.mock('chalk', () => ({
@@ -28,11 +28,16 @@ vi.mock('@ai-sdk/openai', () => ({
28
28
  }));
29
29
 
30
30
  // Mock the LongMemEvalMetric
31
- vi.mock('../../evaluation/longmemeval-metric', () => ({
32
- LongMemEvalMetric: vi.fn().mockImplementation(() => ({
33
- measure: vi.fn().mockResolvedValue({ score: 1 }), // Always returns correct
34
- })),
35
- }));
31
+ vi.mock('../../evaluation/longmemeval-metric', () => {
32
+ // Use a class for constructor (Vitest v4 requirement)
33
+ class MockLongMemEvalMetric {
34
+ measure = vi.fn().mockResolvedValue({ score: 1 }); // Always returns correct
35
+ }
36
+
37
+ return {
38
+ LongMemEvalMetric: MockLongMemEvalMetric,
39
+ };
40
+ });
36
41
 
37
42
  // Mock chalk and ora
38
43
  vi.mock('chalk', () => ({
@@ -148,7 +153,6 @@ describe('RunCommand', () => {
148
153
  ],
149
154
  mastra_resources: [],
150
155
  mastra_workflow_snapshot: [],
151
- mastra_evals: [],
152
156
  mastra_traces: [],
153
157
  }),
154
158
  );
@@ -225,7 +229,6 @@ describe('RunCommand', () => {
225
229
  ],
226
230
  mastra_resources: [],
227
231
  mastra_workflow_snapshot: [],
228
- mastra_evals: [],
229
232
  mastra_traces: [],
230
233
  }),
231
234
  );
@@ -322,7 +325,6 @@ describe('RunCommand', () => {
322
325
  ],
323
326
  mastra_resources: [],
324
327
  mastra_workflow_snapshot: [],
325
- mastra_evals: [],
326
328
  mastra_traces: [],
327
329
  }),
328
330
  );
@@ -0,0 +1,147 @@
1
+ import chalk from 'chalk';
2
+ import ora from 'ora';
3
+ import { join } from 'path';
4
+ import { readdir, rm } from 'fs/promises';
5
+ import { existsSync } from 'fs';
6
+
7
+ import { DatasetLoader } from '../data/loader';
8
+ import type { DatasetType, MemoryConfigType } from '../data/types';
9
+
10
+ export interface CleanOptions {
11
+ dataset: DatasetType;
12
+ memoryConfig: MemoryConfigType;
13
+ preparedDataDir?: string;
14
+ offset?: number;
15
+ subset?: number;
16
+ questionId?: string;
17
+ dryRun?: boolean;
18
+ partial?: boolean; // Only clean partially prepared questions
19
+ }
20
+
21
+ export class CleanCommand {
22
+ private preparedDataDir: string;
23
+ private loader: DatasetLoader;
24
+
25
+ constructor() {
26
+ this.preparedDataDir = './prepared-data';
27
+ this.loader = new DatasetLoader();
28
+ }
29
+
30
+ async run(options: CleanOptions): Promise<void> {
31
+ const preparedDir = join(options.preparedDataDir || this.preparedDataDir, options.dataset, options.memoryConfig);
32
+
33
+ if (!existsSync(preparedDir)) {
34
+ console.log(chalk.yellow(`No prepared data found at: ${preparedDir}`));
35
+ return;
36
+ }
37
+
38
+ // Load original dataset to get correct question order
39
+ const spinner = ora('Loading dataset for ordering...').start();
40
+ const originalQuestions = await this.loader.loadDataset(options.dataset);
41
+ const questionIdOrder = new Map(originalQuestions.map((q, i) => [q.question_id, i]));
42
+ spinner.succeed(`Loaded ${originalQuestions.length} questions`);
43
+
44
+ // Get all prepared question directories
45
+ const questionDirs = await readdir(preparedDir);
46
+
47
+ // Sort by original dataset order
48
+ const sortedDirs = questionDirs
49
+ .filter(dir => questionIdOrder.has(dir))
50
+ .sort((a, b) => {
51
+ const orderA = questionIdOrder.get(a) ?? Infinity;
52
+ const orderB = questionIdOrder.get(b) ?? Infinity;
53
+ return orderA - orderB;
54
+ });
55
+
56
+ // Determine which to delete
57
+ let toDelete: string[] = [];
58
+
59
+ if (options.partial) {
60
+ // Only delete partially prepared questions (have progress.json but no meta.json, or failed)
61
+ for (const questionId of sortedDirs) {
62
+ const questionDir = join(preparedDir, questionId);
63
+ const progressPath = join(questionDir, 'progress.json');
64
+ const metaPath = join(questionDir, 'meta.json');
65
+
66
+ // Skip if fully prepared (has meta.json)
67
+ if (existsSync(metaPath)) {
68
+ continue;
69
+ }
70
+
71
+ // Include if has progress.json (partial or failed)
72
+ if (existsSync(progressPath)) {
73
+ toDelete.push(questionId);
74
+ }
75
+ }
76
+ } else if (options.questionId) {
77
+ // Delete specific question
78
+ if (sortedDirs.includes(options.questionId)) {
79
+ toDelete = [options.questionId];
80
+ } else {
81
+ console.log(chalk.yellow(`Question ${options.questionId} not found in prepared data`));
82
+ return;
83
+ }
84
+ } else {
85
+ // Apply offset and subset
86
+ const offset = options.offset || 0;
87
+ let selected = sortedDirs.slice(offset);
88
+
89
+ if (options.subset) {
90
+ selected = selected.slice(0, options.subset);
91
+ }
92
+
93
+ toDelete = selected;
94
+ }
95
+
96
+ if (toDelete.length === 0) {
97
+ console.log(chalk.yellow('No questions to delete'));
98
+ return;
99
+ }
100
+
101
+ // Show what will be deleted
102
+ const offset = options.offset || 0;
103
+ console.log(
104
+ chalk.yellow(
105
+ `\n${options.dryRun ? '[DRY RUN] Would delete' : 'Deleting'} ${toDelete.length} prepared question(s):`,
106
+ ),
107
+ );
108
+ console.log(chalk.gray(` Range: ${offset + 1}-${offset + toDelete.length} of ${sortedDirs.length} total\n`));
109
+
110
+ // Show first few and last few
111
+ const showCount = 3;
112
+ if (toDelete.length <= showCount * 2) {
113
+ for (const id of toDelete) {
114
+ const idx = questionIdOrder.get(id)! + 1;
115
+ console.log(chalk.gray(` ${idx}. ${id}`));
116
+ }
117
+ } else {
118
+ for (let i = 0; i < showCount; i++) {
119
+ const id = toDelete[i];
120
+ const idx = questionIdOrder.get(id)! + 1;
121
+ console.log(chalk.gray(` ${idx}. ${id}`));
122
+ }
123
+ console.log(chalk.gray(` ... (${toDelete.length - showCount * 2} more)`));
124
+ for (let i = toDelete.length - showCount; i < toDelete.length; i++) {
125
+ const id = toDelete[i];
126
+ const idx = questionIdOrder.get(id)! + 1;
127
+ console.log(chalk.gray(` ${idx}. ${id}`));
128
+ }
129
+ }
130
+
131
+ if (options.dryRun) {
132
+ console.log(chalk.cyan('\n[DRY RUN] No files deleted. Remove --dry-run to actually delete.'));
133
+ return;
134
+ }
135
+
136
+ // Delete the directories
137
+ const deleteSpinner = ora('Deleting...').start();
138
+ let deleted = 0;
139
+ for (const id of toDelete) {
140
+ const dirPath = join(preparedDir, id);
141
+ await rm(dirPath, { recursive: true, force: true });
142
+ deleted++;
143
+ deleteSpinner.text = `Deleting... ${deleted}/${toDelete.length}`;
144
+ }
145
+ deleteSpinner.succeed(`Deleted ${deleted} prepared question(s)`);
146
+ }
147
+ }
@@ -0,0 +1,126 @@
1
+ import { readdir, readFile, writeFile } from 'fs/promises';
2
+ import { join } from 'path';
3
+ import chalk from 'chalk';
4
+
5
+ interface DeterministicIdsOptions {
6
+ preparedDataDir?: string;
7
+ questionId?: string;
8
+ }
9
+
10
+ /**
11
+ * Makes message IDs deterministic for a single om.json file.
12
+ * Format: {thread_id}_msg_{index}
13
+ * @returns Number of IDs updated
14
+ */
15
+ export async function makeDeterministicIds(omJsonPath: string): Promise<number> {
16
+ const content = await readFile(omJsonPath, 'utf-8');
17
+ const data = JSON.parse(content);
18
+
19
+ if (!data.messages || !Array.isArray(data.messages)) {
20
+ return 0;
21
+ }
22
+
23
+ // Group messages by thread_id to get proper indices
24
+ const messagesByThread = new Map<string, Array<{ index: number; entry: [string, any] }>>();
25
+
26
+ for (let i = 0; i < data.messages.length; i++) {
27
+ const entry = data.messages[i] as [string, any];
28
+ const message = entry[1];
29
+ const threadId = message.thread_id;
30
+
31
+ if (!messagesByThread.has(threadId)) {
32
+ messagesByThread.set(threadId, []);
33
+ }
34
+ messagesByThread.get(threadId)!.push({ index: i, entry });
35
+ }
36
+
37
+ // Update IDs to be deterministic
38
+ let updatedCount = 0;
39
+
40
+ for (const [threadId, messages] of messagesByThread) {
41
+ // Sort by createdAt to ensure consistent ordering
42
+ messages.sort((a, b) => {
43
+ const dateA = new Date(a.entry[1].createdAt).getTime();
44
+ const dateB = new Date(b.entry[1].createdAt).getTime();
45
+ return dateA - dateB;
46
+ });
47
+
48
+ for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {
49
+ const { index, entry } = messages[msgIdx];
50
+ const newId = `${threadId}_msg_${msgIdx}`;
51
+ const oldId = entry[0];
52
+
53
+ if (oldId !== newId) {
54
+ // Update both the key and the id field
55
+ data.messages[index] = [newId, { ...entry[1], id: newId }];
56
+ updatedCount++;
57
+ }
58
+ }
59
+ }
60
+
61
+ if (updatedCount > 0) {
62
+ await writeFile(omJsonPath, JSON.stringify(data, null, 2));
63
+ }
64
+
65
+ return updatedCount;
66
+ }
67
+
68
+ /**
69
+ * Updates message IDs in prepared data to be deterministic.
70
+ * Format: {thread_id}_msg_{index}
71
+ */
72
+ export class DeterministicIdsCommand {
73
+ private preparedDataDir: string;
74
+ private questionId?: string;
75
+
76
+ constructor(options: DeterministicIdsOptions) {
77
+ this.preparedDataDir =
78
+ options.preparedDataDir || join(process.cwd(), 'prepared-data', 'longmemeval_s', 'observational-memory');
79
+ this.questionId = options.questionId;
80
+ }
81
+
82
+ async run(): Promise<void> {
83
+ console.log(chalk.blue('\n📋 Making message IDs deterministic...\n'));
84
+
85
+ // Get list of question directories
86
+ let questionDirs: string[];
87
+
88
+ if (this.questionId) {
89
+ questionDirs = [this.questionId];
90
+ } else {
91
+ questionDirs = await readdir(this.preparedDataDir);
92
+ questionDirs = questionDirs.filter(d => !d.startsWith('.'));
93
+ }
94
+
95
+ let totalUpdated = 0;
96
+ let totalMessages = 0;
97
+
98
+ for (const questionDir of questionDirs) {
99
+ const omJsonPath = join(this.preparedDataDir, questionDir, 'om.json');
100
+
101
+ try {
102
+ const updatedCount = await makeDeterministicIds(omJsonPath);
103
+
104
+ // Count total messages for stats
105
+ const content = await readFile(omJsonPath, 'utf-8');
106
+ const data = JSON.parse(content);
107
+ totalMessages += data.messages?.length || 0;
108
+
109
+ if (updatedCount > 0) {
110
+ console.log(chalk.green(` ✓ ${questionDir}: updated ${updatedCount} message IDs`));
111
+ totalUpdated += updatedCount;
112
+ } else {
113
+ console.log(chalk.gray(` - ${questionDir}: already deterministic`));
114
+ }
115
+ } catch (error: any) {
116
+ if (error.code === 'ENOENT') {
117
+ console.log(chalk.yellow(` ⚠ ${questionDir}: no om.json found`));
118
+ } else {
119
+ console.log(chalk.red(` ✗ ${questionDir}: ${error.message}`));
120
+ }
121
+ }
122
+ }
123
+
124
+ console.log(chalk.blue(`\n✓ Updated ${totalUpdated} of ${totalMessages} message IDs\n`));
125
+ }
126
+ }
@@ -0,0 +1,300 @@
1
+ import { google } from '@ai-sdk/google';
2
+ import { generateText } from 'ai';
3
+ import chalk from 'chalk';
4
+ import { DatasetLoader } from '../data/loader';
5
+
6
+ interface FindProhibitedOptions {
7
+ dataset: 'longmemeval_s' | 'longmemeval_m' | 'longmemeval_oracle';
8
+ questionId: string;
9
+ }
10
+
11
+ interface Turn {
12
+ role: string;
13
+ content: string;
14
+ }
15
+
16
+ interface SessionWithMeta {
17
+ sessionId: string;
18
+ session: Turn[];
19
+ date: string;
20
+ index: number;
21
+ }
22
+
23
+ /**
24
+ * Test if content triggers Gemini's PROHIBITED_CONTENT filter
25
+ */
26
+ async function testContent(content: string): Promise<{ blocked: boolean; error?: string }> {
27
+ try {
28
+ await generateText({
29
+ model: google('gemini-2.5-flash'),
30
+ allowSystemInMessages: false,
31
+ messages: [
32
+ { role: 'user', content },
33
+ { role: 'assistant', content: 'I understand.' },
34
+ ],
35
+ providerOptions: {
36
+ google: {
37
+ safetySettings: [
38
+ { category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'OFF' },
39
+ { category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'OFF' },
40
+ { category: 'HARM_CATEGORY_HARASSMENT', threshold: 'OFF' },
41
+ { category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'OFF' },
42
+ { category: 'HARM_CATEGORY_CIVIC_INTEGRITY', threshold: 'OFF' },
43
+ ],
44
+ },
45
+ },
46
+ });
47
+ return { blocked: false };
48
+ } catch (error: any) {
49
+ const errorStr = error?.message || String(error);
50
+ if (errorStr.includes('PROHIBITED_CONTENT') || errorStr.includes('blockReason')) {
51
+ return { blocked: true, error: errorStr };
52
+ }
53
+ // Re-throw non-prohibited errors
54
+ throw error;
55
+ }
56
+ }
57
+
58
+ /**
59
+ * Binary search to find prohibited messages within a session
60
+ */
61
+ async function findProhibitedInSession(
62
+ session: Array<{ role: string; content: string }>,
63
+ sessionId: string,
64
+ ): Promise<number[]> {
65
+ const prohibitedIndices: number[] = [];
66
+
67
+ // First, test the entire session
68
+ const fullContent = session.map(m => `${m.role}: ${m.content}`).join('\n\n');
69
+ const fullResult = await testContent(fullContent);
70
+
71
+ if (!fullResult.blocked) {
72
+ return []; // Session is clean
73
+ }
74
+
75
+ console.log(chalk.yellow(` Session ${sessionId} contains prohibited content, binary searching...`));
76
+
77
+ // Binary search to find the problematic message(s)
78
+ async function binarySearch(start: number, end: number): Promise<void> {
79
+ if (start > end) return;
80
+
81
+ if (start === end) {
82
+ // Single message - test it
83
+ const msg = session[start];
84
+ const result = await testContent(`${msg.role}: ${msg.content}`);
85
+ if (result.blocked) {
86
+ prohibitedIndices.push(start);
87
+ console.log(chalk.red(` Found prohibited message at index ${start}`));
88
+ }
89
+ return;
90
+ }
91
+
92
+ const mid = Math.floor((start + end) / 2);
93
+
94
+ // Test first half
95
+ const firstHalf = session.slice(start, mid + 1);
96
+ const firstContent = firstHalf.map(m => `${m.role}: ${m.content}`).join('\n\n');
97
+ const firstResult = await testContent(firstContent);
98
+
99
+ // Test second half
100
+ const secondHalf = session.slice(mid + 1, end + 1);
101
+ const secondContent = secondHalf.map(m => `${m.role}: ${m.content}`).join('\n\n');
102
+ const secondResult = await testContent(secondContent);
103
+
104
+ // Recurse into halves that contain prohibited content
105
+ if (firstResult.blocked) {
106
+ await binarySearch(start, mid);
107
+ }
108
+ if (secondResult.blocked) {
109
+ await binarySearch(mid + 1, end);
110
+ }
111
+ }
112
+
113
+ await binarySearch(0, session.length - 1);
114
+ return prohibitedIndices;
115
+ }
116
+
117
+ export class FindProhibitedCommand {
118
+ private loader: DatasetLoader;
119
+
120
+ constructor() {
121
+ this.loader = new DatasetLoader();
122
+ }
123
+
124
+ async run(options: FindProhibitedOptions): Promise<void> {
125
+ console.log(chalk.blue(`\n🔍 Finding prohibited content in question ${options.questionId}\n`));
126
+
127
+ // Load dataset
128
+ const questions = await this.loader.loadDataset(options.dataset);
129
+ const question = questions.find(q => q.question_id === options.questionId);
130
+
131
+ if (!question) {
132
+ console.error(chalk.red(`Question ${options.questionId} not found in ${options.dataset}`));
133
+ return;
134
+ }
135
+
136
+ // Get all sessions with metadata (using array format like prepare.ts)
137
+ const sessionsWithMeta: SessionWithMeta[] = question.haystack_sessions.map((session, index) => ({
138
+ session: session as Turn[],
139
+ sessionId: question.haystack_session_ids[index],
140
+ date: question.haystack_dates[index],
141
+ index,
142
+ }));
143
+
144
+ // Sort by date
145
+ sessionsWithMeta.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
146
+
147
+ console.log(chalk.gray(`Found ${sessionsWithMeta.length} sessions\n`));
148
+
149
+ // First, binary search through sessions to find which ones have prohibited content
150
+ const prohibitedSessions: SessionWithMeta[] = [];
151
+
152
+ console.log(chalk.cyan('Phase 1: Binary search to find sessions with prohibited content...\n'));
153
+
154
+ // Binary search through sessions to find which ones have prohibited content
155
+ async function binarySearchSessions(start: number, end: number): Promise<void> {
156
+ if (start > end) return;
157
+
158
+ if (start === end) {
159
+ // Single session - test it
160
+ const sessionMeta = sessionsWithMeta[start];
161
+ const fullContent = sessionMeta.session.map(m => `${m.role}: ${m.content}`).join('\n\n');
162
+ process.stdout.write(chalk.gray(` Testing session ${sessionMeta.sessionId}... `));
163
+ try {
164
+ const result = await testContent(fullContent);
165
+ if (result.blocked) {
166
+ console.log(chalk.red('BLOCKED'));
167
+ prohibitedSessions.push(sessionMeta);
168
+ } else {
169
+ console.log(chalk.green('OK'));
170
+ }
171
+ } catch (error: any) {
172
+ console.log(chalk.yellow(`ERROR: ${error.message?.slice(0, 50)}`));
173
+ }
174
+ return;
175
+ }
176
+
177
+ const mid = Math.floor((start + end) / 2);
178
+
179
+ // Test first half (all sessions combined)
180
+ const firstHalfSessions = sessionsWithMeta.slice(start, mid + 1);
181
+ const firstHalfContent = firstHalfSessions
182
+ .map(s => s.session.map(m => `${m.role}: ${m.content}`).join('\n\n'))
183
+ .join('\n\n---SESSION BREAK---\n\n');
184
+
185
+ console.log(chalk.gray(` Testing sessions ${start}-${mid} (${mid - start + 1} sessions)...`));
186
+ const firstResult = await testContent(firstHalfContent);
187
+
188
+ // Test second half
189
+ const secondHalfSessions = sessionsWithMeta.slice(mid + 1, end + 1);
190
+ const secondHalfContent = secondHalfSessions
191
+ .map(s => s.session.map(m => `${m.role}: ${m.content}`).join('\n\n'))
192
+ .join('\n\n---SESSION BREAK---\n\n');
193
+
194
+ console.log(chalk.gray(` Testing sessions ${mid + 1}-${end} (${end - mid} sessions)...`));
195
+ const secondResult = await testContent(secondHalfContent);
196
+
197
+ // Recurse into halves that contain prohibited content
198
+ if (firstResult.blocked) {
199
+ console.log(chalk.yellow(` Sessions ${start}-${mid} contain prohibited content, drilling down...`));
200
+ await binarySearchSessions(start, mid);
201
+ }
202
+ if (secondResult.blocked) {
203
+ console.log(chalk.yellow(` Sessions ${mid + 1}-${end} contain prohibited content, drilling down...`));
204
+ await binarySearchSessions(mid + 1, end);
205
+ }
206
+ }
207
+
208
+ // First test ALL sessions combined to see if there's any prohibited content
209
+ const allContent = sessionsWithMeta
210
+ .map(s => s.session.map(m => `${m.role}: ${m.content}`).join('\n\n'))
211
+ .join('\n\n---SESSION BREAK---\n\n');
212
+
213
+ console.log(chalk.gray(` Testing all ${sessionsWithMeta.length} sessions combined...`));
214
+ const allResult = await testContent(allContent);
215
+
216
+ if (!allResult.blocked) {
217
+ console.log(chalk.green('\n✅ No prohibited content found in any session!'));
218
+ return;
219
+ }
220
+
221
+ console.log(chalk.yellow(`\n Prohibited content detected! Starting binary search...\n`));
222
+ await binarySearchSessions(0, sessionsWithMeta.length - 1);
223
+
224
+ if (prohibitedSessions.length === 0) {
225
+ console.log(chalk.green('\n✅ No prohibited content found in any session!'));
226
+ return;
227
+ }
228
+
229
+ console.log(chalk.yellow(`\nFound ${prohibitedSessions.length} session(s) with prohibited content\n`));
230
+
231
+ // Phase 2: Find exact messages in each prohibited session
232
+ console.log(chalk.cyan('Phase 2: Finding exact prohibited messages...\n'));
233
+
234
+ const results: Array<{
235
+ sessionId: string;
236
+ date: string;
237
+ messageIndices: number[];
238
+ messages: Array<{ index: number; role: string; content: string }>;
239
+ }> = [];
240
+
241
+ for (const sessionMeta of prohibitedSessions) {
242
+ console.log(chalk.blue(`\nSearching session ${sessionMeta.sessionId} (${sessionMeta.date})...`));
243
+
244
+ const prohibitedIndices = await findProhibitedInSession(sessionMeta.session, sessionMeta.sessionId);
245
+
246
+ if (prohibitedIndices.length > 0) {
247
+ results.push({
248
+ sessionId: sessionMeta.sessionId,
249
+ date: sessionMeta.date,
250
+ messageIndices: prohibitedIndices,
251
+ messages: prohibitedIndices.map(idx => ({
252
+ index: idx,
253
+ role: sessionMeta.session[idx].role,
254
+ content: sessionMeta.session[idx].content,
255
+ })),
256
+ });
257
+ }
258
+ }
259
+
260
+ // Print summary
261
+ console.log(chalk.blue('\n' + '='.repeat(80)));
262
+ console.log(chalk.blue('SUMMARY'));
263
+ console.log(chalk.blue('='.repeat(80) + '\n'));
264
+
265
+ console.log(chalk.white(`Question ID: ${options.questionId}`));
266
+ console.log(chalk.white(`Total sessions: ${sessionsWithMeta.length}`));
267
+ console.log(chalk.white(`Sessions with prohibited content: ${prohibitedSessions.length}`));
268
+ console.log(chalk.white(`Total prohibited messages: ${results.reduce((sum, r) => sum + r.messages.length, 0)}\n`));
269
+
270
+ for (const result of results) {
271
+ console.log(chalk.yellow(`\nSession: ${result.sessionId} (${result.date})`));
272
+ console.log(chalk.yellow(`Message indices: ${result.messageIndices.join(', ')}`));
273
+
274
+ for (const msg of result.messages) {
275
+ console.log(chalk.red(`\n [${msg.index}] ${msg.role}:`));
276
+ // Truncate long content
277
+ const preview = msg.content.length > 500 ? msg.content.slice(0, 500) + '...' : msg.content;
278
+ console.log(chalk.gray(` ${preview.replace(/\n/g, '\n ')}`));
279
+ }
280
+ }
281
+
282
+ // Output as JSON for programmatic use
283
+ console.log(chalk.blue('\n' + '='.repeat(80)));
284
+ console.log(chalk.blue('JSON OUTPUT'));
285
+ console.log(chalk.blue('='.repeat(80) + '\n'));
286
+
287
+ const jsonOutput = {
288
+ questionId: options.questionId,
289
+ dataset: options.dataset,
290
+ totalSessions: sessionsWithMeta.length,
291
+ prohibitedSessions: results.map(r => ({
292
+ sessionId: r.sessionId,
293
+ date: r.date,
294
+ messageIndices: r.messageIndices,
295
+ })),
296
+ };
297
+
298
+ console.log(JSON.stringify(jsonOutput, null, 2));
299
+ }
300
+ }