@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
|
@@ -15,9 +15,10 @@ vi.mock('@ai-sdk/openai', () => ({
|
|
|
15
15
|
}));
|
|
16
16
|
|
|
17
17
|
// Mock the DatasetLoader
|
|
18
|
-
vi.mock('../../data/loader', () =>
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
vi.mock('../../data/loader', () => {
|
|
19
|
+
// Use a class for constructor (Vitest v4 requirement)
|
|
20
|
+
class MockDatasetLoader {
|
|
21
|
+
loadDataset = vi.fn().mockResolvedValue([
|
|
21
22
|
{
|
|
22
23
|
question_id: 'test-q1',
|
|
23
24
|
question_type: 'single-session-user',
|
|
@@ -54,9 +55,13 @@ vi.mock('../../data/loader', () => ({
|
|
|
54
55
|
],
|
|
55
56
|
answer_session_ids: ['session-3'],
|
|
56
57
|
},
|
|
57
|
-
] as LongMemEvalQuestion[])
|
|
58
|
-
}
|
|
59
|
-
|
|
58
|
+
] as LongMemEvalQuestion[]);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return {
|
|
62
|
+
DatasetLoader: MockDatasetLoader,
|
|
63
|
+
};
|
|
64
|
+
});
|
|
60
65
|
|
|
61
66
|
// Mock chalk and ora to avoid console output in tests
|
|
62
67
|
vi.mock('chalk', () => ({
|
|
@@ -28,11 +28,16 @@ vi.mock('@ai-sdk/openai', () => ({
|
|
|
28
28
|
}));
|
|
29
29
|
|
|
30
30
|
// Mock the LongMemEvalMetric
|
|
31
|
-
vi.mock('../../evaluation/longmemeval-metric', () =>
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
}
|
|
31
|
+
vi.mock('../../evaluation/longmemeval-metric', () => {
|
|
32
|
+
// Use a class for constructor (Vitest v4 requirement)
|
|
33
|
+
class MockLongMemEvalMetric {
|
|
34
|
+
measure = vi.fn().mockResolvedValue({ score: 1 }); // Always returns correct
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
return {
|
|
38
|
+
LongMemEvalMetric: MockLongMemEvalMetric,
|
|
39
|
+
};
|
|
40
|
+
});
|
|
36
41
|
|
|
37
42
|
// Mock chalk and ora
|
|
38
43
|
vi.mock('chalk', () => ({
|
|
@@ -148,7 +153,6 @@ describe('RunCommand', () => {
|
|
|
148
153
|
],
|
|
149
154
|
mastra_resources: [],
|
|
150
155
|
mastra_workflow_snapshot: [],
|
|
151
|
-
mastra_evals: [],
|
|
152
156
|
mastra_traces: [],
|
|
153
157
|
}),
|
|
154
158
|
);
|
|
@@ -225,7 +229,6 @@ describe('RunCommand', () => {
|
|
|
225
229
|
],
|
|
226
230
|
mastra_resources: [],
|
|
227
231
|
mastra_workflow_snapshot: [],
|
|
228
|
-
mastra_evals: [],
|
|
229
232
|
mastra_traces: [],
|
|
230
233
|
}),
|
|
231
234
|
);
|
|
@@ -322,7 +325,6 @@ describe('RunCommand', () => {
|
|
|
322
325
|
],
|
|
323
326
|
mastra_resources: [],
|
|
324
327
|
mastra_workflow_snapshot: [],
|
|
325
|
-
mastra_evals: [],
|
|
326
328
|
mastra_traces: [],
|
|
327
329
|
}),
|
|
328
330
|
);
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import { join } from 'path';
|
|
4
|
+
import { readdir, rm } from 'fs/promises';
|
|
5
|
+
import { existsSync } from 'fs';
|
|
6
|
+
|
|
7
|
+
import { DatasetLoader } from '../data/loader';
|
|
8
|
+
import type { DatasetType, MemoryConfigType } from '../data/types';
|
|
9
|
+
|
|
10
|
+
export interface CleanOptions {
|
|
11
|
+
dataset: DatasetType;
|
|
12
|
+
memoryConfig: MemoryConfigType;
|
|
13
|
+
preparedDataDir?: string;
|
|
14
|
+
offset?: number;
|
|
15
|
+
subset?: number;
|
|
16
|
+
questionId?: string;
|
|
17
|
+
dryRun?: boolean;
|
|
18
|
+
partial?: boolean; // Only clean partially prepared questions
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class CleanCommand {
|
|
22
|
+
private preparedDataDir: string;
|
|
23
|
+
private loader: DatasetLoader;
|
|
24
|
+
|
|
25
|
+
constructor() {
|
|
26
|
+
this.preparedDataDir = './prepared-data';
|
|
27
|
+
this.loader = new DatasetLoader();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async run(options: CleanOptions): Promise<void> {
|
|
31
|
+
const preparedDir = join(options.preparedDataDir || this.preparedDataDir, options.dataset, options.memoryConfig);
|
|
32
|
+
|
|
33
|
+
if (!existsSync(preparedDir)) {
|
|
34
|
+
console.log(chalk.yellow(`No prepared data found at: ${preparedDir}`));
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Load original dataset to get correct question order
|
|
39
|
+
const spinner = ora('Loading dataset for ordering...').start();
|
|
40
|
+
const originalQuestions = await this.loader.loadDataset(options.dataset);
|
|
41
|
+
const questionIdOrder = new Map(originalQuestions.map((q, i) => [q.question_id, i]));
|
|
42
|
+
spinner.succeed(`Loaded ${originalQuestions.length} questions`);
|
|
43
|
+
|
|
44
|
+
// Get all prepared question directories
|
|
45
|
+
const questionDirs = await readdir(preparedDir);
|
|
46
|
+
|
|
47
|
+
// Sort by original dataset order
|
|
48
|
+
const sortedDirs = questionDirs
|
|
49
|
+
.filter(dir => questionIdOrder.has(dir))
|
|
50
|
+
.sort((a, b) => {
|
|
51
|
+
const orderA = questionIdOrder.get(a) ?? Infinity;
|
|
52
|
+
const orderB = questionIdOrder.get(b) ?? Infinity;
|
|
53
|
+
return orderA - orderB;
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
// Determine which to delete
|
|
57
|
+
let toDelete: string[] = [];
|
|
58
|
+
|
|
59
|
+
if (options.partial) {
|
|
60
|
+
// Only delete partially prepared questions (have progress.json but no meta.json, or failed)
|
|
61
|
+
for (const questionId of sortedDirs) {
|
|
62
|
+
const questionDir = join(preparedDir, questionId);
|
|
63
|
+
const progressPath = join(questionDir, 'progress.json');
|
|
64
|
+
const metaPath = join(questionDir, 'meta.json');
|
|
65
|
+
|
|
66
|
+
// Skip if fully prepared (has meta.json)
|
|
67
|
+
if (existsSync(metaPath)) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Include if has progress.json (partial or failed)
|
|
72
|
+
if (existsSync(progressPath)) {
|
|
73
|
+
toDelete.push(questionId);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
} else if (options.questionId) {
|
|
77
|
+
// Delete specific question
|
|
78
|
+
if (sortedDirs.includes(options.questionId)) {
|
|
79
|
+
toDelete = [options.questionId];
|
|
80
|
+
} else {
|
|
81
|
+
console.log(chalk.yellow(`Question ${options.questionId} not found in prepared data`));
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
} else {
|
|
85
|
+
// Apply offset and subset
|
|
86
|
+
const offset = options.offset || 0;
|
|
87
|
+
let selected = sortedDirs.slice(offset);
|
|
88
|
+
|
|
89
|
+
if (options.subset) {
|
|
90
|
+
selected = selected.slice(0, options.subset);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
toDelete = selected;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (toDelete.length === 0) {
|
|
97
|
+
console.log(chalk.yellow('No questions to delete'));
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Show what will be deleted
|
|
102
|
+
const offset = options.offset || 0;
|
|
103
|
+
console.log(
|
|
104
|
+
chalk.yellow(
|
|
105
|
+
`\n${options.dryRun ? '[DRY RUN] Would delete' : 'Deleting'} ${toDelete.length} prepared question(s):`,
|
|
106
|
+
),
|
|
107
|
+
);
|
|
108
|
+
console.log(chalk.gray(` Range: ${offset + 1}-${offset + toDelete.length} of ${sortedDirs.length} total\n`));
|
|
109
|
+
|
|
110
|
+
// Show first few and last few
|
|
111
|
+
const showCount = 3;
|
|
112
|
+
if (toDelete.length <= showCount * 2) {
|
|
113
|
+
for (const id of toDelete) {
|
|
114
|
+
const idx = questionIdOrder.get(id)! + 1;
|
|
115
|
+
console.log(chalk.gray(` ${idx}. ${id}`));
|
|
116
|
+
}
|
|
117
|
+
} else {
|
|
118
|
+
for (let i = 0; i < showCount; i++) {
|
|
119
|
+
const id = toDelete[i];
|
|
120
|
+
const idx = questionIdOrder.get(id)! + 1;
|
|
121
|
+
console.log(chalk.gray(` ${idx}. ${id}`));
|
|
122
|
+
}
|
|
123
|
+
console.log(chalk.gray(` ... (${toDelete.length - showCount * 2} more)`));
|
|
124
|
+
for (let i = toDelete.length - showCount; i < toDelete.length; i++) {
|
|
125
|
+
const id = toDelete[i];
|
|
126
|
+
const idx = questionIdOrder.get(id)! + 1;
|
|
127
|
+
console.log(chalk.gray(` ${idx}. ${id}`));
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (options.dryRun) {
|
|
132
|
+
console.log(chalk.cyan('\n[DRY RUN] No files deleted. Remove --dry-run to actually delete.'));
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Delete the directories
|
|
137
|
+
const deleteSpinner = ora('Deleting...').start();
|
|
138
|
+
let deleted = 0;
|
|
139
|
+
for (const id of toDelete) {
|
|
140
|
+
const dirPath = join(preparedDir, id);
|
|
141
|
+
await rm(dirPath, { recursive: true, force: true });
|
|
142
|
+
deleted++;
|
|
143
|
+
deleteSpinner.text = `Deleting... ${deleted}/${toDelete.length}`;
|
|
144
|
+
}
|
|
145
|
+
deleteSpinner.succeed(`Deleted ${deleted} prepared question(s)`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { readdir, readFile, writeFile } from 'fs/promises';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import chalk from 'chalk';
|
|
4
|
+
|
|
5
|
+
interface DeterministicIdsOptions {
|
|
6
|
+
preparedDataDir?: string;
|
|
7
|
+
questionId?: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Makes message IDs deterministic for a single om.json file.
|
|
12
|
+
* Format: {thread_id}_msg_{index}
|
|
13
|
+
* @returns Number of IDs updated
|
|
14
|
+
*/
|
|
15
|
+
export async function makeDeterministicIds(omJsonPath: string): Promise<number> {
|
|
16
|
+
const content = await readFile(omJsonPath, 'utf-8');
|
|
17
|
+
const data = JSON.parse(content);
|
|
18
|
+
|
|
19
|
+
if (!data.messages || !Array.isArray(data.messages)) {
|
|
20
|
+
return 0;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// Group messages by thread_id to get proper indices
|
|
24
|
+
const messagesByThread = new Map<string, Array<{ index: number; entry: [string, any] }>>();
|
|
25
|
+
|
|
26
|
+
for (let i = 0; i < data.messages.length; i++) {
|
|
27
|
+
const entry = data.messages[i] as [string, any];
|
|
28
|
+
const message = entry[1];
|
|
29
|
+
const threadId = message.thread_id;
|
|
30
|
+
|
|
31
|
+
if (!messagesByThread.has(threadId)) {
|
|
32
|
+
messagesByThread.set(threadId, []);
|
|
33
|
+
}
|
|
34
|
+
messagesByThread.get(threadId)!.push({ index: i, entry });
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Update IDs to be deterministic
|
|
38
|
+
let updatedCount = 0;
|
|
39
|
+
|
|
40
|
+
for (const [threadId, messages] of messagesByThread) {
|
|
41
|
+
// Sort by createdAt to ensure consistent ordering
|
|
42
|
+
messages.sort((a, b) => {
|
|
43
|
+
const dateA = new Date(a.entry[1].createdAt).getTime();
|
|
44
|
+
const dateB = new Date(b.entry[1].createdAt).getTime();
|
|
45
|
+
return dateA - dateB;
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {
|
|
49
|
+
const { index, entry } = messages[msgIdx];
|
|
50
|
+
const newId = `${threadId}_msg_${msgIdx}`;
|
|
51
|
+
const oldId = entry[0];
|
|
52
|
+
|
|
53
|
+
if (oldId !== newId) {
|
|
54
|
+
// Update both the key and the id field
|
|
55
|
+
data.messages[index] = [newId, { ...entry[1], id: newId }];
|
|
56
|
+
updatedCount++;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (updatedCount > 0) {
|
|
62
|
+
await writeFile(omJsonPath, JSON.stringify(data, null, 2));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return updatedCount;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Updates message IDs in prepared data to be deterministic.
|
|
70
|
+
* Format: {thread_id}_msg_{index}
|
|
71
|
+
*/
|
|
72
|
+
export class DeterministicIdsCommand {
|
|
73
|
+
private preparedDataDir: string;
|
|
74
|
+
private questionId?: string;
|
|
75
|
+
|
|
76
|
+
constructor(options: DeterministicIdsOptions) {
|
|
77
|
+
this.preparedDataDir =
|
|
78
|
+
options.preparedDataDir || join(process.cwd(), 'prepared-data', 'longmemeval_s', 'observational-memory');
|
|
79
|
+
this.questionId = options.questionId;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async run(): Promise<void> {
|
|
83
|
+
console.log(chalk.blue('\n📋 Making message IDs deterministic...\n'));
|
|
84
|
+
|
|
85
|
+
// Get list of question directories
|
|
86
|
+
let questionDirs: string[];
|
|
87
|
+
|
|
88
|
+
if (this.questionId) {
|
|
89
|
+
questionDirs = [this.questionId];
|
|
90
|
+
} else {
|
|
91
|
+
questionDirs = await readdir(this.preparedDataDir);
|
|
92
|
+
questionDirs = questionDirs.filter(d => !d.startsWith('.'));
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
let totalUpdated = 0;
|
|
96
|
+
let totalMessages = 0;
|
|
97
|
+
|
|
98
|
+
for (const questionDir of questionDirs) {
|
|
99
|
+
const omJsonPath = join(this.preparedDataDir, questionDir, 'om.json');
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
const updatedCount = await makeDeterministicIds(omJsonPath);
|
|
103
|
+
|
|
104
|
+
// Count total messages for stats
|
|
105
|
+
const content = await readFile(omJsonPath, 'utf-8');
|
|
106
|
+
const data = JSON.parse(content);
|
|
107
|
+
totalMessages += data.messages?.length || 0;
|
|
108
|
+
|
|
109
|
+
if (updatedCount > 0) {
|
|
110
|
+
console.log(chalk.green(` ✓ ${questionDir}: updated ${updatedCount} message IDs`));
|
|
111
|
+
totalUpdated += updatedCount;
|
|
112
|
+
} else {
|
|
113
|
+
console.log(chalk.gray(` - ${questionDir}: already deterministic`));
|
|
114
|
+
}
|
|
115
|
+
} catch (error: any) {
|
|
116
|
+
if (error.code === 'ENOENT') {
|
|
117
|
+
console.log(chalk.yellow(` ⚠ ${questionDir}: no om.json found`));
|
|
118
|
+
} else {
|
|
119
|
+
console.log(chalk.red(` ✗ ${questionDir}: ${error.message}`));
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
console.log(chalk.blue(`\n✓ Updated ${totalUpdated} of ${totalMessages} message IDs\n`));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
import { google } from '@ai-sdk/google';
|
|
2
|
+
import { generateText } from 'ai';
|
|
3
|
+
import chalk from 'chalk';
|
|
4
|
+
import { DatasetLoader } from '../data/loader';
|
|
5
|
+
|
|
6
|
+
interface FindProhibitedOptions {
|
|
7
|
+
dataset: 'longmemeval_s' | 'longmemeval_m' | 'longmemeval_oracle';
|
|
8
|
+
questionId: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
interface Turn {
|
|
12
|
+
role: string;
|
|
13
|
+
content: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface SessionWithMeta {
|
|
17
|
+
sessionId: string;
|
|
18
|
+
session: Turn[];
|
|
19
|
+
date: string;
|
|
20
|
+
index: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Test if content triggers Gemini's PROHIBITED_CONTENT filter
|
|
25
|
+
*/
|
|
26
|
+
async function testContent(content: string): Promise<{ blocked: boolean; error?: string }> {
|
|
27
|
+
try {
|
|
28
|
+
await generateText({
|
|
29
|
+
model: google('gemini-2.5-flash'),
|
|
30
|
+
messages: [
|
|
31
|
+
{ role: 'user', content },
|
|
32
|
+
{ role: 'assistant', content: 'I understand.' },
|
|
33
|
+
],
|
|
34
|
+
providerOptions: {
|
|
35
|
+
google: {
|
|
36
|
+
safetySettings: [
|
|
37
|
+
{ category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'OFF' },
|
|
38
|
+
{ category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'OFF' },
|
|
39
|
+
{ category: 'HARM_CATEGORY_HARASSMENT', threshold: 'OFF' },
|
|
40
|
+
{ category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'OFF' },
|
|
41
|
+
{ category: 'HARM_CATEGORY_CIVIC_INTEGRITY', threshold: 'OFF' },
|
|
42
|
+
],
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
return { blocked: false };
|
|
47
|
+
} catch (error: any) {
|
|
48
|
+
const errorStr = error?.message || String(error);
|
|
49
|
+
if (errorStr.includes('PROHIBITED_CONTENT') || errorStr.includes('blockReason')) {
|
|
50
|
+
return { blocked: true, error: errorStr };
|
|
51
|
+
}
|
|
52
|
+
// Re-throw non-prohibited errors
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Binary search to find prohibited messages within a session
|
|
59
|
+
*/
|
|
60
|
+
async function findProhibitedInSession(
|
|
61
|
+
session: Array<{ role: string; content: string }>,
|
|
62
|
+
sessionId: string,
|
|
63
|
+
): Promise<number[]> {
|
|
64
|
+
const prohibitedIndices: number[] = [];
|
|
65
|
+
|
|
66
|
+
// First, test the entire session
|
|
67
|
+
const fullContent = session.map(m => `${m.role}: ${m.content}`).join('\n\n');
|
|
68
|
+
const fullResult = await testContent(fullContent);
|
|
69
|
+
|
|
70
|
+
if (!fullResult.blocked) {
|
|
71
|
+
return []; // Session is clean
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
console.log(chalk.yellow(` Session ${sessionId} contains prohibited content, binary searching...`));
|
|
75
|
+
|
|
76
|
+
// Binary search to find the problematic message(s)
|
|
77
|
+
async function binarySearch(start: number, end: number): Promise<void> {
|
|
78
|
+
if (start > end) return;
|
|
79
|
+
|
|
80
|
+
if (start === end) {
|
|
81
|
+
// Single message - test it
|
|
82
|
+
const msg = session[start];
|
|
83
|
+
const result = await testContent(`${msg.role}: ${msg.content}`);
|
|
84
|
+
if (result.blocked) {
|
|
85
|
+
prohibitedIndices.push(start);
|
|
86
|
+
console.log(chalk.red(` Found prohibited message at index ${start}`));
|
|
87
|
+
}
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const mid = Math.floor((start + end) / 2);
|
|
92
|
+
|
|
93
|
+
// Test first half
|
|
94
|
+
const firstHalf = session.slice(start, mid + 1);
|
|
95
|
+
const firstContent = firstHalf.map(m => `${m.role}: ${m.content}`).join('\n\n');
|
|
96
|
+
const firstResult = await testContent(firstContent);
|
|
97
|
+
|
|
98
|
+
// Test second half
|
|
99
|
+
const secondHalf = session.slice(mid + 1, end + 1);
|
|
100
|
+
const secondContent = secondHalf.map(m => `${m.role}: ${m.content}`).join('\n\n');
|
|
101
|
+
const secondResult = await testContent(secondContent);
|
|
102
|
+
|
|
103
|
+
// Recurse into halves that contain prohibited content
|
|
104
|
+
if (firstResult.blocked) {
|
|
105
|
+
await binarySearch(start, mid);
|
|
106
|
+
}
|
|
107
|
+
if (secondResult.blocked) {
|
|
108
|
+
await binarySearch(mid + 1, end);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
await binarySearch(0, session.length - 1);
|
|
113
|
+
return prohibitedIndices;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export class FindProhibitedCommand {
|
|
117
|
+
private loader: DatasetLoader;
|
|
118
|
+
|
|
119
|
+
constructor() {
|
|
120
|
+
this.loader = new DatasetLoader();
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async run(options: FindProhibitedOptions): Promise<void> {
|
|
124
|
+
console.log(chalk.blue(`\n🔍 Finding prohibited content in question ${options.questionId}\n`));
|
|
125
|
+
|
|
126
|
+
// Load dataset
|
|
127
|
+
const questions = await this.loader.loadDataset(options.dataset);
|
|
128
|
+
const question = questions.find(q => q.question_id === options.questionId);
|
|
129
|
+
|
|
130
|
+
if (!question) {
|
|
131
|
+
console.error(chalk.red(`Question ${options.questionId} not found in ${options.dataset}`));
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Get all sessions with metadata (using array format like prepare.ts)
|
|
136
|
+
const sessionsWithMeta: SessionWithMeta[] = question.haystack_sessions.map((session, index) => ({
|
|
137
|
+
session: session as Turn[],
|
|
138
|
+
sessionId: question.haystack_session_ids[index],
|
|
139
|
+
date: question.haystack_dates[index],
|
|
140
|
+
index,
|
|
141
|
+
}));
|
|
142
|
+
|
|
143
|
+
// Sort by date
|
|
144
|
+
sessionsWithMeta.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
|
|
145
|
+
|
|
146
|
+
console.log(chalk.gray(`Found ${sessionsWithMeta.length} sessions\n`));
|
|
147
|
+
|
|
148
|
+
// First, binary search through sessions to find which ones have prohibited content
|
|
149
|
+
const prohibitedSessions: SessionWithMeta[] = [];
|
|
150
|
+
|
|
151
|
+
console.log(chalk.cyan('Phase 1: Binary search to find sessions with prohibited content...\n'));
|
|
152
|
+
|
|
153
|
+
// Binary search through sessions to find which ones have prohibited content
|
|
154
|
+
async function binarySearchSessions(start: number, end: number): Promise<void> {
|
|
155
|
+
if (start > end) return;
|
|
156
|
+
|
|
157
|
+
if (start === end) {
|
|
158
|
+
// Single session - test it
|
|
159
|
+
const sessionMeta = sessionsWithMeta[start];
|
|
160
|
+
const fullContent = sessionMeta.session.map(m => `${m.role}: ${m.content}`).join('\n\n');
|
|
161
|
+
process.stdout.write(chalk.gray(` Testing session ${sessionMeta.sessionId}... `));
|
|
162
|
+
try {
|
|
163
|
+
const result = await testContent(fullContent);
|
|
164
|
+
if (result.blocked) {
|
|
165
|
+
console.log(chalk.red('BLOCKED'));
|
|
166
|
+
prohibitedSessions.push(sessionMeta);
|
|
167
|
+
} else {
|
|
168
|
+
console.log(chalk.green('OK'));
|
|
169
|
+
}
|
|
170
|
+
} catch (error: any) {
|
|
171
|
+
console.log(chalk.yellow(`ERROR: ${error.message?.slice(0, 50)}`));
|
|
172
|
+
}
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const mid = Math.floor((start + end) / 2);
|
|
177
|
+
|
|
178
|
+
// Test first half (all sessions combined)
|
|
179
|
+
const firstHalfSessions = sessionsWithMeta.slice(start, mid + 1);
|
|
180
|
+
const firstHalfContent = firstHalfSessions
|
|
181
|
+
.map(s => s.session.map(m => `${m.role}: ${m.content}`).join('\n\n'))
|
|
182
|
+
.join('\n\n---SESSION BREAK---\n\n');
|
|
183
|
+
|
|
184
|
+
console.log(chalk.gray(` Testing sessions ${start}-${mid} (${mid - start + 1} sessions)...`));
|
|
185
|
+
const firstResult = await testContent(firstHalfContent);
|
|
186
|
+
|
|
187
|
+
// Test second half
|
|
188
|
+
const secondHalfSessions = sessionsWithMeta.slice(mid + 1, end + 1);
|
|
189
|
+
const secondHalfContent = secondHalfSessions
|
|
190
|
+
.map(s => s.session.map(m => `${m.role}: ${m.content}`).join('\n\n'))
|
|
191
|
+
.join('\n\n---SESSION BREAK---\n\n');
|
|
192
|
+
|
|
193
|
+
console.log(chalk.gray(` Testing sessions ${mid + 1}-${end} (${end - mid} sessions)...`));
|
|
194
|
+
const secondResult = await testContent(secondHalfContent);
|
|
195
|
+
|
|
196
|
+
// Recurse into halves that contain prohibited content
|
|
197
|
+
if (firstResult.blocked) {
|
|
198
|
+
console.log(chalk.yellow(` Sessions ${start}-${mid} contain prohibited content, drilling down...`));
|
|
199
|
+
await binarySearchSessions(start, mid);
|
|
200
|
+
}
|
|
201
|
+
if (secondResult.blocked) {
|
|
202
|
+
console.log(chalk.yellow(` Sessions ${mid + 1}-${end} contain prohibited content, drilling down...`));
|
|
203
|
+
await binarySearchSessions(mid + 1, end);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// First test ALL sessions combined to see if there's any prohibited content
|
|
208
|
+
const allContent = sessionsWithMeta
|
|
209
|
+
.map(s => s.session.map(m => `${m.role}: ${m.content}`).join('\n\n'))
|
|
210
|
+
.join('\n\n---SESSION BREAK---\n\n');
|
|
211
|
+
|
|
212
|
+
console.log(chalk.gray(` Testing all ${sessionsWithMeta.length} sessions combined...`));
|
|
213
|
+
const allResult = await testContent(allContent);
|
|
214
|
+
|
|
215
|
+
if (!allResult.blocked) {
|
|
216
|
+
console.log(chalk.green('\n✅ No prohibited content found in any session!'));
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
console.log(chalk.yellow(`\n Prohibited content detected! Starting binary search...\n`));
|
|
221
|
+
await binarySearchSessions(0, sessionsWithMeta.length - 1);
|
|
222
|
+
|
|
223
|
+
if (prohibitedSessions.length === 0) {
|
|
224
|
+
console.log(chalk.green('\n✅ No prohibited content found in any session!'));
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
console.log(chalk.yellow(`\nFound ${prohibitedSessions.length} session(s) with prohibited content\n`));
|
|
229
|
+
|
|
230
|
+
// Phase 2: Find exact messages in each prohibited session
|
|
231
|
+
console.log(chalk.cyan('Phase 2: Finding exact prohibited messages...\n'));
|
|
232
|
+
|
|
233
|
+
const results: Array<{
|
|
234
|
+
sessionId: string;
|
|
235
|
+
date: string;
|
|
236
|
+
messageIndices: number[];
|
|
237
|
+
messages: Array<{ index: number; role: string; content: string }>;
|
|
238
|
+
}> = [];
|
|
239
|
+
|
|
240
|
+
for (const sessionMeta of prohibitedSessions) {
|
|
241
|
+
console.log(chalk.blue(`\nSearching session ${sessionMeta.sessionId} (${sessionMeta.date})...`));
|
|
242
|
+
|
|
243
|
+
const prohibitedIndices = await findProhibitedInSession(sessionMeta.session, sessionMeta.sessionId);
|
|
244
|
+
|
|
245
|
+
if (prohibitedIndices.length > 0) {
|
|
246
|
+
results.push({
|
|
247
|
+
sessionId: sessionMeta.sessionId,
|
|
248
|
+
date: sessionMeta.date,
|
|
249
|
+
messageIndices: prohibitedIndices,
|
|
250
|
+
messages: prohibitedIndices.map(idx => ({
|
|
251
|
+
index: idx,
|
|
252
|
+
role: sessionMeta.session[idx].role,
|
|
253
|
+
content: sessionMeta.session[idx].content,
|
|
254
|
+
})),
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// Print summary
|
|
260
|
+
console.log(chalk.blue('\n' + '='.repeat(80)));
|
|
261
|
+
console.log(chalk.blue('SUMMARY'));
|
|
262
|
+
console.log(chalk.blue('='.repeat(80) + '\n'));
|
|
263
|
+
|
|
264
|
+
console.log(chalk.white(`Question ID: ${options.questionId}`));
|
|
265
|
+
console.log(chalk.white(`Total sessions: ${sessionsWithMeta.length}`));
|
|
266
|
+
console.log(chalk.white(`Sessions with prohibited content: ${prohibitedSessions.length}`));
|
|
267
|
+
console.log(chalk.white(`Total prohibited messages: ${results.reduce((sum, r) => sum + r.messages.length, 0)}\n`));
|
|
268
|
+
|
|
269
|
+
for (const result of results) {
|
|
270
|
+
console.log(chalk.yellow(`\nSession: ${result.sessionId} (${result.date})`));
|
|
271
|
+
console.log(chalk.yellow(`Message indices: ${result.messageIndices.join(', ')}`));
|
|
272
|
+
|
|
273
|
+
for (const msg of result.messages) {
|
|
274
|
+
console.log(chalk.red(`\n [${msg.index}] ${msg.role}:`));
|
|
275
|
+
// Truncate long content
|
|
276
|
+
const preview = msg.content.length > 500 ? msg.content.slice(0, 500) + '...' : msg.content;
|
|
277
|
+
console.log(chalk.gray(` ${preview.replace(/\n/g, '\n ')}`));
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// Output as JSON for programmatic use
|
|
282
|
+
console.log(chalk.blue('\n' + '='.repeat(80)));
|
|
283
|
+
console.log(chalk.blue('JSON OUTPUT'));
|
|
284
|
+
console.log(chalk.blue('='.repeat(80) + '\n'));
|
|
285
|
+
|
|
286
|
+
const jsonOutput = {
|
|
287
|
+
questionId: options.questionId,
|
|
288
|
+
dataset: options.dataset,
|
|
289
|
+
totalSessions: sessionsWithMeta.length,
|
|
290
|
+
prohibitedSessions: results.map(r => ({
|
|
291
|
+
sessionId: r.sessionId,
|
|
292
|
+
date: r.date,
|
|
293
|
+
messageIndices: r.messageIndices,
|
|
294
|
+
})),
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
console.log(JSON.stringify(jsonOutput, null, 2));
|
|
298
|
+
}
|
|
299
|
+
}
|