@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import { DatasetLoader } from '../data/loader.js';
|
|
3
|
+
import { LongMemEvalQuestion } from '../data/types.js';
|
|
4
|
+
|
|
5
|
+
type DatasetType = 'longmemeval_s' | 'longmemeval_m' | 'longmemeval_oracle' | 'sample_data';
|
|
6
|
+
|
|
7
|
+
interface SessionsOptions {
|
|
8
|
+
dataset: DatasetType;
|
|
9
|
+
questionId: string;
|
|
10
|
+
showAll?: boolean;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
interface Message {
|
|
14
|
+
role: string;
|
|
15
|
+
content: string;
|
|
16
|
+
session_id?: string;
|
|
17
|
+
date?: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export class SessionsCommand {
|
|
21
|
+
async run(options: SessionsOptions): Promise<void> {
|
|
22
|
+
const { dataset: datasetName, questionId, showAll } = options;
|
|
23
|
+
|
|
24
|
+
// Load dataset
|
|
25
|
+
const loader = new DatasetLoader();
|
|
26
|
+
const dataset = await loader.loadDataset(datasetName);
|
|
27
|
+
|
|
28
|
+
// Find the question
|
|
29
|
+
const question = dataset.find((q: LongMemEvalQuestion) => q.question_id === questionId);
|
|
30
|
+
if (!question) {
|
|
31
|
+
console.error(chalk.red(`Question ${questionId} not found in dataset ${datasetName}`));
|
|
32
|
+
console.log(chalk.gray('Available question IDs (first 20):'));
|
|
33
|
+
dataset.slice(0, 20).forEach((q: LongMemEvalQuestion) => {
|
|
34
|
+
console.log(chalk.gray(` ${q.question_id}`));
|
|
35
|
+
});
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Display question info
|
|
40
|
+
console.log(chalk.bold.cyan('\n═══════════════════════════════════════════════════════════════'));
|
|
41
|
+
console.log(chalk.bold.white(`Question: ${questionId}`));
|
|
42
|
+
console.log(chalk.bold.cyan('═══════════════════════════════════════════════════════════════\n'));
|
|
43
|
+
|
|
44
|
+
console.log(chalk.yellow('Q:'), question.question);
|
|
45
|
+
console.log(chalk.green('A:'), question.answer);
|
|
46
|
+
console.log(chalk.gray('Type:'), question.question_type);
|
|
47
|
+
console.log(chalk.gray('Date:'), question.question_date);
|
|
48
|
+
|
|
49
|
+
if (question.improved_question) {
|
|
50
|
+
console.log(chalk.magenta('Improved Q:'), question.improved_question);
|
|
51
|
+
}
|
|
52
|
+
if (question.improved_answer) {
|
|
53
|
+
console.log(chalk.magenta('Improved A:'), question.improved_answer);
|
|
54
|
+
}
|
|
55
|
+
if (question.improvement_note) {
|
|
56
|
+
console.log(chalk.gray('Note:'), question.improvement_note);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Get answer session IDs (those starting with "answer_")
|
|
60
|
+
const answerSessionIds = (question.haystack_session_ids || []).filter((id: string) => id.startsWith('answer_'));
|
|
61
|
+
|
|
62
|
+
const sessionIdsToShow = showAll ? question.haystack_session_ids || [] : answerSessionIds;
|
|
63
|
+
|
|
64
|
+
if (sessionIdsToShow.length === 0) {
|
|
65
|
+
console.log(chalk.yellow('\nNo sessions found for this question.'));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
console.log(chalk.bold.cyan(`\n─── ${showAll ? 'All' : 'Answer'} Sessions (${sessionIdsToShow.length}) ───\n`));
|
|
70
|
+
|
|
71
|
+
// Get sessions from haystack_sessions
|
|
72
|
+
const haystackSessions = question.haystack_sessions || [];
|
|
73
|
+
|
|
74
|
+
for (const sessionId of sessionIdsToShow) {
|
|
75
|
+
// haystack_sessions is parallel to haystack_session_ids - find by index
|
|
76
|
+
const allSessionIds = question.haystack_session_ids || [];
|
|
77
|
+
const sessionIndex = allSessionIds.indexOf(sessionId);
|
|
78
|
+
const session = sessionIndex >= 0 ? haystackSessions[sessionIndex] : null;
|
|
79
|
+
|
|
80
|
+
if (!session || !Array.isArray(session)) {
|
|
81
|
+
console.log(chalk.gray(`Session ${sessionId}: (not found in haystack_sessions)`));
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Get date from parallel haystack_dates array
|
|
86
|
+
const haystackDates = question.haystack_dates || [];
|
|
87
|
+
const sessionDate = haystackDates[sessionIndex] || 'Unknown date';
|
|
88
|
+
|
|
89
|
+
console.log(chalk.bold.white(`\n┌─ Session: ${sessionId}`));
|
|
90
|
+
console.log(chalk.gray(`│ Date: ${sessionDate}`));
|
|
91
|
+
console.log(chalk.gray('│'));
|
|
92
|
+
|
|
93
|
+
for (const msg of session as Message[]) {
|
|
94
|
+
const role = msg.role;
|
|
95
|
+
const content = msg.content || '';
|
|
96
|
+
const roleColor = role === 'user' ? chalk.cyan : chalk.green;
|
|
97
|
+
const roleLabel = role === 'user' ? '👤 User' : '🤖 Assistant';
|
|
98
|
+
|
|
99
|
+
// Truncate long messages for readability
|
|
100
|
+
const maxLen = 500;
|
|
101
|
+
const displayContent =
|
|
102
|
+
content.length > maxLen ? content.substring(0, maxLen) + chalk.gray('... (truncated)') : content;
|
|
103
|
+
|
|
104
|
+
console.log(chalk.gray('│'));
|
|
105
|
+
console.log(roleColor(`│ ${roleLabel}:`));
|
|
106
|
+
|
|
107
|
+
// Indent message content
|
|
108
|
+
const lines = displayContent.split('\n');
|
|
109
|
+
for (const line of lines) {
|
|
110
|
+
console.log(chalk.gray('│ ') + line);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
console.log(chalk.gray('│'));
|
|
115
|
+
console.log(chalk.gray('└─────────────────────────────────────────'));
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Show summary of other sessions if not showing all
|
|
119
|
+
if (!showAll && question.haystack_session_ids) {
|
|
120
|
+
const otherCount = question.haystack_session_ids.length - answerSessionIds.length;
|
|
121
|
+
if (otherCount > 0) {
|
|
122
|
+
console.log(chalk.gray(`\n(${otherCount} other haystack sessions not shown. Use --all to see all sessions)`));
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { readdir, readFile, writeFile } from 'fs/promises';
|
|
4
|
+
import { existsSync } from 'fs';
|
|
5
|
+
|
|
6
|
+
import { DatasetLoader } from '../data/loader';
|
|
7
|
+
import type { LongMemEvalQuestion, DatasetType } from '../data/types';
|
|
8
|
+
|
|
9
|
+
export interface SyncOptions {
|
|
10
|
+
dataset: DatasetType;
|
|
11
|
+
memoryConfig: string;
|
|
12
|
+
preparedDataDir: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export class SyncCommand {
|
|
16
|
+
async run(options: SyncOptions): Promise<void> {
|
|
17
|
+
const { dataset, memoryConfig, preparedDataDir } = options;
|
|
18
|
+
|
|
19
|
+
// Load the source dataset
|
|
20
|
+
console.log(chalk.gray(`Loading dataset: ${dataset}...`));
|
|
21
|
+
const loader = new DatasetLoader();
|
|
22
|
+
const questions = await loader.loadDataset(dataset);
|
|
23
|
+
|
|
24
|
+
// Create a map for quick lookup by question_id
|
|
25
|
+
const questionMap = new Map<string, LongMemEvalQuestion>();
|
|
26
|
+
for (const q of questions) {
|
|
27
|
+
questionMap.set(q.question_id, q);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
console.log(chalk.gray(`Loaded ${questions.length} questions from dataset\n`));
|
|
31
|
+
|
|
32
|
+
// Find prepared data directory
|
|
33
|
+
const preparedDir = join(preparedDataDir, dataset, memoryConfig);
|
|
34
|
+
|
|
35
|
+
if (!existsSync(preparedDir)) {
|
|
36
|
+
console.error(chalk.red(`No prepared data found at: ${preparedDir}`));
|
|
37
|
+
console.error(chalk.gray(`Run 'longmemeval prepare' first`));
|
|
38
|
+
process.exit(1);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Get all question directories
|
|
42
|
+
const questionDirs = await readdir(preparedDir);
|
|
43
|
+
|
|
44
|
+
let updated = 0;
|
|
45
|
+
let skipped = 0;
|
|
46
|
+
let notFound = 0;
|
|
47
|
+
|
|
48
|
+
for (const questionDir of questionDirs) {
|
|
49
|
+
const metaPath = join(preparedDir, questionDir, 'meta.json');
|
|
50
|
+
|
|
51
|
+
if (!existsSync(metaPath)) {
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Load existing meta.json
|
|
56
|
+
const meta = JSON.parse(await readFile(metaPath, 'utf-8'));
|
|
57
|
+
const questionId = meta.questionId;
|
|
58
|
+
|
|
59
|
+
// Find the source question
|
|
60
|
+
const sourceQuestion = questionMap.get(questionId);
|
|
61
|
+
|
|
62
|
+
if (!sourceQuestion) {
|
|
63
|
+
console.log(chalk.yellow(`⚠ Question not found in dataset: ${questionId}`));
|
|
64
|
+
notFound++;
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Check if there are updates to sync
|
|
69
|
+
const hasImprovedQuestion = sourceQuestion.improved_question !== undefined;
|
|
70
|
+
const hasImprovedAnswer = sourceQuestion.improved_answer !== undefined;
|
|
71
|
+
const hasImprovementNote = sourceQuestion.improvement_note !== undefined;
|
|
72
|
+
const hasRequiresRetry = sourceQuestion.requires_retry !== undefined;
|
|
73
|
+
const hasFailureCategory = sourceQuestion.failure_category !== undefined;
|
|
74
|
+
|
|
75
|
+
const needsUpdate =
|
|
76
|
+
meta.improvedQuestion !== sourceQuestion.improved_question ||
|
|
77
|
+
meta.improvedAnswer !== sourceQuestion.improved_answer ||
|
|
78
|
+
meta.improvementNote !== sourceQuestion.improvement_note ||
|
|
79
|
+
meta.requiresRetry !== sourceQuestion.requires_retry ||
|
|
80
|
+
meta.failureCategory !== sourceQuestion.failure_category;
|
|
81
|
+
|
|
82
|
+
if (!needsUpdate) {
|
|
83
|
+
skipped++;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Update meta with improved fields
|
|
88
|
+
if (hasImprovedQuestion) {
|
|
89
|
+
meta.improvedQuestion = sourceQuestion.improved_question;
|
|
90
|
+
} else {
|
|
91
|
+
delete meta.improvedQuestion;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (hasImprovedAnswer) {
|
|
95
|
+
meta.improvedAnswer = sourceQuestion.improved_answer;
|
|
96
|
+
} else {
|
|
97
|
+
delete meta.improvedAnswer;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (hasImprovementNote) {
|
|
101
|
+
meta.improvementNote = sourceQuestion.improvement_note;
|
|
102
|
+
} else {
|
|
103
|
+
delete meta.improvementNote;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (hasRequiresRetry) {
|
|
107
|
+
meta.requiresRetry = sourceQuestion.requires_retry;
|
|
108
|
+
} else {
|
|
109
|
+
delete meta.requiresRetry;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (hasFailureCategory) {
|
|
113
|
+
meta.failureCategory = sourceQuestion.failure_category;
|
|
114
|
+
} else {
|
|
115
|
+
delete meta.failureCategory;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Write updated meta.json
|
|
119
|
+
await writeFile(metaPath, JSON.stringify(meta, null, 2));
|
|
120
|
+
|
|
121
|
+
// Log the update
|
|
122
|
+
if (hasImprovedQuestion || hasImprovedAnswer || hasImprovementNote || hasRequiresRetry || hasFailureCategory) {
|
|
123
|
+
console.log(
|
|
124
|
+
chalk.green(`✓ ${questionId}`),
|
|
125
|
+
hasImprovedQuestion ? chalk.cyan('(improved_question)') : '',
|
|
126
|
+
hasImprovedAnswer ? chalk.magenta('(improved_answer)') : '',
|
|
127
|
+
hasImprovementNote ? chalk.yellow('(improvement_note)') : '',
|
|
128
|
+
hasRequiresRetry ? chalk.blue('(requires_retry)') : '',
|
|
129
|
+
hasFailureCategory ? chalk.red(`(${sourceQuestion.failure_category})`) : '',
|
|
130
|
+
);
|
|
131
|
+
} else {
|
|
132
|
+
console.log(chalk.gray(`○ ${questionId} (cleared improved fields)`));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
updated++;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// Summary
|
|
139
|
+
console.log(chalk.bold('\n─────────────────────────────────'));
|
|
140
|
+
console.log(chalk.bold('Sync Summary:'));
|
|
141
|
+
console.log(chalk.green(` Updated: ${updated}`));
|
|
142
|
+
console.log(chalk.gray(` Skipped (no changes): ${skipped}`));
|
|
143
|
+
if (notFound > 0) {
|
|
144
|
+
console.log(chalk.yellow(` Not found in dataset: ${notFound}`));
|
|
145
|
+
}
|
|
146
|
+
console.log(chalk.bold('─────────────────────────────────\n'));
|
|
147
|
+
|
|
148
|
+
// Show count of questions with improved fields in dataset
|
|
149
|
+
const withImproved = questions.filter(q => q.improved_question || q.improved_answer).length;
|
|
150
|
+
console.log(chalk.gray(`Dataset has ${withImproved} questions with improved_question or improved_answer fields`));
|
|
151
|
+
}
|
|
152
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test if removing the problematic thread fixes the blocking.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { google } from '@ai-sdk/google';
|
|
6
|
+
import { generateText } from 'ai';
|
|
7
|
+
import { readFileSync } from 'fs';
|
|
8
|
+
|
|
9
|
+
const safetySettings = [
|
|
10
|
+
{ category: 'HARM_CATEGORY_HATE_SPEECH' as const, threshold: 'OFF' as const },
|
|
11
|
+
{ category: 'HARM_CATEGORY_DANGEROUS_CONTENT' as const, threshold: 'OFF' as const },
|
|
12
|
+
{ category: 'HARM_CATEGORY_HARASSMENT' as const, threshold: 'OFF' as const },
|
|
13
|
+
{ category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT' as const, threshold: 'OFF' as const },
|
|
14
|
+
{ category: 'HARM_CATEGORY_CIVIC_INTEGRITY' as const, threshold: 'OFF' as const },
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
async function testContent(content: string, label: string): Promise<boolean> {
|
|
18
|
+
try {
|
|
19
|
+
const model = google('gemini-2.5-flash');
|
|
20
|
+
await generateText({
|
|
21
|
+
model,
|
|
22
|
+
prompt: content.slice(0, 50000),
|
|
23
|
+
providerOptions: {
|
|
24
|
+
google: { safetySettings },
|
|
25
|
+
},
|
|
26
|
+
maxOutputTokens: 10,
|
|
27
|
+
});
|
|
28
|
+
console.log(`${label}: ✅ OK`);
|
|
29
|
+
return true;
|
|
30
|
+
} catch (error: any) {
|
|
31
|
+
const errorStr = String(error) + String(error.cause || '');
|
|
32
|
+
if (errorStr.includes('PROHIBITED_CONTENT')) {
|
|
33
|
+
console.log(`${label}: 🚫 BLOCKED`);
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
console.log(`${label}: ⚠️ Other error: ${error.message?.slice(0, 100)}`);
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function main() {
|
|
42
|
+
// Test original prompt
|
|
43
|
+
const original = JSON.parse(readFileSync('/tmp/om-prohibited-1768418150770.json', 'utf-8'));
|
|
44
|
+
console.log('Testing original prompt...');
|
|
45
|
+
await testContent(original.prompt, 'Original prompt');
|
|
46
|
+
|
|
47
|
+
// Test cleaned prompt (without problematic thread)
|
|
48
|
+
const cleaned = readFileSync('/tmp/prompt-without-problematic.txt', 'utf-8');
|
|
49
|
+
console.log('\nTesting cleaned prompt (without sharegpt_eYSgdp6_0)...');
|
|
50
|
+
await testContent(cleaned, 'Cleaned prompt');
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
main().catch(console.error);
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Incrementally build up messages, skipping ones that cause blocking.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { google } from '@ai-sdk/google';
|
|
6
|
+
import { generateText } from 'ai';
|
|
7
|
+
import { readFileSync } from 'fs';
|
|
8
|
+
|
|
9
|
+
const safetySettings = [
|
|
10
|
+
{ category: 'HARM_CATEGORY_HATE_SPEECH' as const, threshold: 'OFF' as const },
|
|
11
|
+
{ category: 'HARM_CATEGORY_DANGEROUS_CONTENT' as const, threshold: 'OFF' as const },
|
|
12
|
+
{ category: 'HARM_CATEGORY_HARASSMENT' as const, threshold: 'OFF' as const },
|
|
13
|
+
{ category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT' as const, threshold: 'OFF' as const },
|
|
14
|
+
{ category: 'HARM_CATEGORY_CIVIC_INTEGRITY' as const, threshold: 'OFF' as const },
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
async function testContent(content: string): Promise<boolean> {
|
|
18
|
+
try {
|
|
19
|
+
const model = google('gemini-2.5-flash');
|
|
20
|
+
await generateText({
|
|
21
|
+
model,
|
|
22
|
+
prompt: content,
|
|
23
|
+
providerOptions: { google: { safetySettings } },
|
|
24
|
+
maxOutputTokens: 10,
|
|
25
|
+
});
|
|
26
|
+
return true; // OK
|
|
27
|
+
} catch (error: any) {
|
|
28
|
+
const errorStr = String(error) + String(error.cause || '');
|
|
29
|
+
return !errorStr.includes('PROHIBITED_CONTENT');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function parseMessages(content: string): string[] {
|
|
34
|
+
const parts = content.split('---');
|
|
35
|
+
const messages: string[] = [];
|
|
36
|
+
let current = '';
|
|
37
|
+
|
|
38
|
+
for (const part of parts) {
|
|
39
|
+
const trimmed = part.trim();
|
|
40
|
+
if (trimmed.startsWith('**User') || trimmed.startsWith('**Assistant')) {
|
|
41
|
+
if (current) messages.push(current);
|
|
42
|
+
current = trimmed;
|
|
43
|
+
} else if (current) {
|
|
44
|
+
current += '\n---\n' + trimmed;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (current) messages.push(current);
|
|
48
|
+
return messages;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function main() {
|
|
52
|
+
const content = readFileSync('/tmp/problematic-thread.txt', 'utf-8');
|
|
53
|
+
const messages = parseMessages(content);
|
|
54
|
+
|
|
55
|
+
console.log(`Found ${messages.length} messages\n`);
|
|
56
|
+
|
|
57
|
+
// Phase 1: Build up incrementally, skipping problematic messages
|
|
58
|
+
console.log('=== Phase 1: Incremental build (skip problematic) ===\n');
|
|
59
|
+
|
|
60
|
+
const included: number[] = [];
|
|
61
|
+
const skipped: number[] = [];
|
|
62
|
+
|
|
63
|
+
for (let i = 0; i < messages.length; i++) {
|
|
64
|
+
// Try adding this message
|
|
65
|
+
const testSet = [...included, i];
|
|
66
|
+
const combined = testSet.map(idx => messages[idx]).join('\n\n---\n\n');
|
|
67
|
+
const ok = await testContent(combined);
|
|
68
|
+
|
|
69
|
+
const preview = messages[i].slice(0, 60).replace(/\n/g, ' ');
|
|
70
|
+
|
|
71
|
+
if (ok) {
|
|
72
|
+
included.push(i);
|
|
73
|
+
console.log(`Message ${i}: ✅ included "${preview}..."`);
|
|
74
|
+
} else {
|
|
75
|
+
skipped.push(i);
|
|
76
|
+
console.log(`Message ${i}: 🚫 SKIPPED "${preview}..."`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
console.log(`\n--- Summary ---`);
|
|
81
|
+
console.log(`Included: ${included.length} messages: [${included.join(', ')}]`);
|
|
82
|
+
console.log(`Skipped: ${skipped.length} messages: [${skipped.join(', ')}]`);
|
|
83
|
+
|
|
84
|
+
// Phase 2: For each skipped message, show full content
|
|
85
|
+
if (skipped.length > 0) {
|
|
86
|
+
console.log(`\n=== Phase 2: Skipped message details ===\n`);
|
|
87
|
+
for (const idx of skipped) {
|
|
88
|
+
console.log(`--- Message ${idx} (BLOCKED) ---`);
|
|
89
|
+
console.log(messages[idx]);
|
|
90
|
+
console.log();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Phase 3: Test if we can include any skipped messages by removing others
|
|
95
|
+
if (skipped.length > 1) {
|
|
96
|
+
console.log(`\n=== Phase 3: Can we include some skipped by removing others? ===\n`);
|
|
97
|
+
|
|
98
|
+
for (const tryInclude of skipped) {
|
|
99
|
+
for (const tryRemove of skipped) {
|
|
100
|
+
if (tryInclude === tryRemove) continue;
|
|
101
|
+
|
|
102
|
+
const testSet = [...included, tryInclude].filter(i => i !== tryRemove);
|
|
103
|
+
const combined = testSet.map(idx => messages[idx]).join('\n\n---\n\n');
|
|
104
|
+
const ok = await testContent(combined);
|
|
105
|
+
|
|
106
|
+
if (ok) {
|
|
107
|
+
console.log(`✅ Can include ${tryInclude} if we remove ${tryRemove}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Final test: verify the included set works
|
|
114
|
+
console.log(`\n=== Final verification ===`);
|
|
115
|
+
const finalCombined = included.map(idx => messages[idx]).join('\n\n---\n\n');
|
|
116
|
+
const finalOk = await testContent(finalCombined);
|
|
117
|
+
console.log(`Included messages only: ${finalOk ? '✅ PASSES' : '🚫 FAILS'}`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
main().catch(console.error);
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test each message individually to find exact trigger(s).
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { google } from '@ai-sdk/google';
|
|
6
|
+
import { generateText } from 'ai';
|
|
7
|
+
import { readFileSync } from 'fs';
|
|
8
|
+
|
|
9
|
+
const safetySettings = [
|
|
10
|
+
{ category: 'HARM_CATEGORY_HATE_SPEECH' as const, threshold: 'OFF' as const },
|
|
11
|
+
{ category: 'HARM_CATEGORY_DANGEROUS_CONTENT' as const, threshold: 'OFF' as const },
|
|
12
|
+
{ category: 'HARM_CATEGORY_HARASSMENT' as const, threshold: 'OFF' as const },
|
|
13
|
+
{ category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT' as const, threshold: 'OFF' as const },
|
|
14
|
+
{ category: 'HARM_CATEGORY_CIVIC_INTEGRITY' as const, threshold: 'OFF' as const },
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
async function testContent(content: string): Promise<boolean> {
|
|
18
|
+
try {
|
|
19
|
+
const model = google('gemini-2.5-flash');
|
|
20
|
+
await generateText({
|
|
21
|
+
model,
|
|
22
|
+
prompt: content,
|
|
23
|
+
providerOptions: { google: { safetySettings } },
|
|
24
|
+
maxOutputTokens: 10,
|
|
25
|
+
});
|
|
26
|
+
return true; // OK
|
|
27
|
+
} catch (error: any) {
|
|
28
|
+
const errorStr = String(error) + String(error.cause || '');
|
|
29
|
+
return !errorStr.includes('PROHIBITED_CONTENT');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function parseMessages(content: string): string[] {
|
|
34
|
+
const parts = content.split('---');
|
|
35
|
+
const messages: string[] = [];
|
|
36
|
+
let current = '';
|
|
37
|
+
|
|
38
|
+
for (const part of parts) {
|
|
39
|
+
const trimmed = part.trim();
|
|
40
|
+
if (trimmed.startsWith('**User') || trimmed.startsWith('**Assistant')) {
|
|
41
|
+
if (current) messages.push(current);
|
|
42
|
+
current = trimmed;
|
|
43
|
+
} else if (current) {
|
|
44
|
+
current += '\n---\n' + trimmed;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (current) messages.push(current);
|
|
48
|
+
return messages;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function main() {
|
|
52
|
+
const content = readFileSync('/tmp/problematic-thread.txt', 'utf-8');
|
|
53
|
+
const messages = parseMessages(content);
|
|
54
|
+
|
|
55
|
+
console.log(`Testing ${messages.length} messages individually...\n`);
|
|
56
|
+
|
|
57
|
+
// Test each message alone
|
|
58
|
+
console.log('=== Individual Message Tests ===');
|
|
59
|
+
for (let i = 0; i < messages.length; i++) {
|
|
60
|
+
const ok = await testContent(messages[i]);
|
|
61
|
+
const preview = messages[i].slice(0, 80).replace(/\n/g, ' ');
|
|
62
|
+
console.log(`Message ${i}: ${ok ? '✅' : '🚫'} ${preview}...`);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Test cumulative (0 to N)
|
|
66
|
+
console.log('\n=== Cumulative Tests (messages 0 to N) ===');
|
|
67
|
+
for (let i = 0; i < messages.length; i++) {
|
|
68
|
+
const combined = messages.slice(0, i + 1).join('\n\n---\n\n');
|
|
69
|
+
const ok = await testContent(combined);
|
|
70
|
+
console.log(`Messages 0-${i}: ${ok ? '✅' : '🚫'}`);
|
|
71
|
+
if (!ok) {
|
|
72
|
+
console.log(` ^ First failure at message ${i}`);
|
|
73
|
+
console.log(` Content: ${messages[i].slice(0, 150).replace(/\n/g, ' ')}...`);
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Test removing specific messages to find which are essential to trigger
|
|
79
|
+
console.log('\n=== Testing which messages are essential triggers ===');
|
|
80
|
+
const allContent = messages.join('\n\n---\n\n');
|
|
81
|
+
const allOk = await testContent(allContent);
|
|
82
|
+
console.log(`All messages combined: ${allOk ? '✅' : '🚫'}`);
|
|
83
|
+
|
|
84
|
+
if (!allOk) {
|
|
85
|
+
// Test removing each message one at a time
|
|
86
|
+
console.log('\nRemoving one message at a time:');
|
|
87
|
+
for (let i = 0; i < messages.length; i++) {
|
|
88
|
+
const without = messages.filter((_, idx) => idx !== i).join('\n\n---\n\n');
|
|
89
|
+
const ok = await testContent(without);
|
|
90
|
+
console.log(`Without message ${i}: ${ok ? '✅ FIXES IT' : '🚫 still blocked'}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
main().catch(console.error);
|