network-ai 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/QUICKSTART.md +260 -0
- package/README.md +604 -0
- package/SKILL.md +568 -0
- package/dist/adapters/adapter-registry.d.ts +94 -0
- package/dist/adapters/adapter-registry.d.ts.map +1 -0
- package/dist/adapters/adapter-registry.js +355 -0
- package/dist/adapters/adapter-registry.js.map +1 -0
- package/dist/adapters/agno-adapter.d.ts +112 -0
- package/dist/adapters/agno-adapter.d.ts.map +1 -0
- package/dist/adapters/agno-adapter.js +140 -0
- package/dist/adapters/agno-adapter.js.map +1 -0
- package/dist/adapters/autogen-adapter.d.ts +67 -0
- package/dist/adapters/autogen-adapter.d.ts.map +1 -0
- package/dist/adapters/autogen-adapter.js +141 -0
- package/dist/adapters/autogen-adapter.js.map +1 -0
- package/dist/adapters/base-adapter.d.ts +51 -0
- package/dist/adapters/base-adapter.d.ts.map +1 -0
- package/dist/adapters/base-adapter.js +103 -0
- package/dist/adapters/base-adapter.js.map +1 -0
- package/dist/adapters/crewai-adapter.d.ts +72 -0
- package/dist/adapters/crewai-adapter.d.ts.map +1 -0
- package/dist/adapters/crewai-adapter.js +148 -0
- package/dist/adapters/crewai-adapter.js.map +1 -0
- package/dist/adapters/custom-adapter.d.ts +74 -0
- package/dist/adapters/custom-adapter.d.ts.map +1 -0
- package/dist/adapters/custom-adapter.js +142 -0
- package/dist/adapters/custom-adapter.js.map +1 -0
- package/dist/adapters/dspy-adapter.d.ts +70 -0
- package/dist/adapters/dspy-adapter.d.ts.map +1 -0
- package/dist/adapters/dspy-adapter.js +127 -0
- package/dist/adapters/dspy-adapter.js.map +1 -0
- package/dist/adapters/haystack-adapter.d.ts +83 -0
- package/dist/adapters/haystack-adapter.d.ts.map +1 -0
- package/dist/adapters/haystack-adapter.js +149 -0
- package/dist/adapters/haystack-adapter.js.map +1 -0
- package/dist/adapters/index.d.ts +47 -0
- package/dist/adapters/index.d.ts.map +1 -0
- package/dist/adapters/index.js +56 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/langchain-adapter.d.ts +51 -0
- package/dist/adapters/langchain-adapter.d.ts.map +1 -0
- package/dist/adapters/langchain-adapter.js +134 -0
- package/dist/adapters/langchain-adapter.js.map +1 -0
- package/dist/adapters/llamaindex-adapter.d.ts +89 -0
- package/dist/adapters/llamaindex-adapter.d.ts.map +1 -0
- package/dist/adapters/llamaindex-adapter.js +135 -0
- package/dist/adapters/llamaindex-adapter.js.map +1 -0
- package/dist/adapters/mcp-adapter.d.ts +90 -0
- package/dist/adapters/mcp-adapter.d.ts.map +1 -0
- package/dist/adapters/mcp-adapter.js +200 -0
- package/dist/adapters/mcp-adapter.js.map +1 -0
- package/dist/adapters/openai-assistants-adapter.d.ts +94 -0
- package/dist/adapters/openai-assistants-adapter.d.ts.map +1 -0
- package/dist/adapters/openai-assistants-adapter.js +130 -0
- package/dist/adapters/openai-assistants-adapter.js.map +1 -0
- package/dist/adapters/openclaw-adapter.d.ts +21 -0
- package/dist/adapters/openclaw-adapter.d.ts.map +1 -0
- package/dist/adapters/openclaw-adapter.js +140 -0
- package/dist/adapters/openclaw-adapter.js.map +1 -0
- package/dist/adapters/semantic-kernel-adapter.d.ts +73 -0
- package/dist/adapters/semantic-kernel-adapter.d.ts.map +1 -0
- package/dist/adapters/semantic-kernel-adapter.js +123 -0
- package/dist/adapters/semantic-kernel-adapter.js.map +1 -0
- package/dist/index.d.ts +379 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1428 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/blackboard-validator.d.ts +205 -0
- package/dist/lib/blackboard-validator.d.ts.map +1 -0
- package/dist/lib/blackboard-validator.js +756 -0
- package/dist/lib/blackboard-validator.js.map +1 -0
- package/dist/lib/locked-blackboard.d.ts +174 -0
- package/dist/lib/locked-blackboard.d.ts.map +1 -0
- package/dist/lib/locked-blackboard.js +654 -0
- package/dist/lib/locked-blackboard.js.map +1 -0
- package/dist/lib/swarm-utils.d.ts +136 -0
- package/dist/lib/swarm-utils.d.ts.map +1 -0
- package/dist/lib/swarm-utils.js +510 -0
- package/dist/lib/swarm-utils.js.map +1 -0
- package/dist/security.d.ts +269 -0
- package/dist/security.d.ts.map +1 -0
- package/dist/security.js +713 -0
- package/dist/security.js.map +1 -0
- package/package.json +84 -0
- package/scripts/blackboard.py +819 -0
- package/scripts/check_permission.py +331 -0
- package/scripts/revoke_token.py +243 -0
- package/scripts/swarm_guard.py +1140 -0
- package/scripts/validate_token.py +97 -0
- package/types/agent-adapter.d.ts +244 -0
- package/types/openclaw-core.d.ts +52 -0
|
@@ -0,0 +1,756 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* BlackboardValidator + QualityGateAgent
|
|
4
|
+
*
|
|
5
|
+
* Two-layer content validation for the SharedBlackboard:
|
|
6
|
+
*
|
|
7
|
+
* Layer 1 -- BlackboardValidator (rule-based, deterministic, fast)
|
|
8
|
+
* Validates structure, completeness, and basic quality of tasks,
|
|
9
|
+
* results, and code before they enter the blackboard.
|
|
10
|
+
*
|
|
11
|
+
* Layer 2 -- QualityGateAgent (AI-assisted, optional)
|
|
12
|
+
* A special review agent that can inspect pending entries,
|
|
13
|
+
* run deeper analysis, detect hallucinations, and approve/reject.
|
|
14
|
+
*
|
|
15
|
+
* Together they prevent bad code, incomplete results, and hallucinated
|
|
16
|
+
* data from poisoning the shared state that other agents depend on.
|
|
17
|
+
*
|
|
18
|
+
* @module BlackboardValidator
|
|
19
|
+
*/
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
exports.QualityGateAgent = exports.BlackboardValidator = void 0;
|
|
22
|
+
// ============================================================================
|
|
23
|
+
// DEFAULT CONFIGURATION
|
|
24
|
+
// ============================================================================
|
|
25
|
+
const DEFAULT_CONFIG = {
|
|
26
|
+
minInstructionLength: 10,
|
|
27
|
+
maxInstructionLength: 50000,
|
|
28
|
+
requireConstraints: false,
|
|
29
|
+
requireExpectedOutput: false,
|
|
30
|
+
minResultFields: 1,
|
|
31
|
+
maxErrorRate: 0.5,
|
|
32
|
+
minCodeLines: 1,
|
|
33
|
+
maxCommentRatio: 0.8,
|
|
34
|
+
detectHallucinations: true,
|
|
35
|
+
rejectPlaceholders: true,
|
|
36
|
+
customRules: [],
|
|
37
|
+
};
|
|
38
|
+
// ============================================================================
|
|
39
|
+
// LAYER 1: BLACKBOARD VALIDATOR -- Rule-based quality gates
|
|
40
|
+
// ============================================================================
|
|
41
|
+
class BlackboardValidator {
|
|
42
|
+
config;
|
|
43
|
+
constructor(config) {
|
|
44
|
+
this.config = { ...DEFAULT_CONFIG, ...config, customRules: [...(config?.customRules ?? DEFAULT_CONFIG.customRules)] };
|
|
45
|
+
}
|
|
46
|
+
// ---------- Public API ----------
|
|
47
|
+
/**
|
|
48
|
+
* Validate any entry by auto-detecting its type from the key prefix.
|
|
49
|
+
*/
|
|
50
|
+
validate(key, value, metadata) {
|
|
51
|
+
const entryType = this.detectEntryType(key, value);
|
|
52
|
+
switch (entryType) {
|
|
53
|
+
case 'task':
|
|
54
|
+
return this.validateTask(key, value);
|
|
55
|
+
case 'result':
|
|
56
|
+
return this.validateResult(key, value, metadata);
|
|
57
|
+
case 'code':
|
|
58
|
+
return this.validateCode(key, value);
|
|
59
|
+
default:
|
|
60
|
+
return this.validateGeneric(key, value);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Validate a task payload before dispatching.
|
|
65
|
+
*/
|
|
66
|
+
validateTask(key, value) {
|
|
67
|
+
const issues = [];
|
|
68
|
+
const rulesApplied = [];
|
|
69
|
+
const obj = value;
|
|
70
|
+
if (!obj || typeof obj !== 'object') {
|
|
71
|
+
return this.makeResult(false, 0, [
|
|
72
|
+
{ rule: 'task.structure', severity: 'error', message: 'Task value must be an object' },
|
|
73
|
+
], ['task.structure']);
|
|
74
|
+
}
|
|
75
|
+
// --- Rule: Instruction quality ---
|
|
76
|
+
rulesApplied.push('task.instruction');
|
|
77
|
+
const instruction = obj.instruction ?? '';
|
|
78
|
+
if (typeof instruction !== 'string' || instruction.trim().length === 0) {
|
|
79
|
+
issues.push({
|
|
80
|
+
rule: 'task.instruction',
|
|
81
|
+
severity: 'error',
|
|
82
|
+
message: 'Task must have a non-empty instruction',
|
|
83
|
+
field: 'instruction',
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
if (instruction.length < this.config.minInstructionLength) {
|
|
88
|
+
issues.push({
|
|
89
|
+
rule: 'task.instruction.length',
|
|
90
|
+
severity: 'error',
|
|
91
|
+
message: `Instruction too short (${instruction.length} chars, minimum ${this.config.minInstructionLength})`,
|
|
92
|
+
field: 'instruction',
|
|
93
|
+
suggestion: 'Provide more specific details about what the task should accomplish',
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
if (instruction.length > this.config.maxInstructionLength) {
|
|
97
|
+
issues.push({
|
|
98
|
+
rule: 'task.instruction.length',
|
|
99
|
+
severity: 'error',
|
|
100
|
+
message: `Instruction too long (${instruction.length} chars, maximum ${this.config.maxInstructionLength})`,
|
|
101
|
+
field: 'instruction',
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
// Detect vague instructions
|
|
105
|
+
rulesApplied.push('task.instruction.quality');
|
|
106
|
+
const vaguePatterns = /^(do it|fix it|make it work|do something|todo|tbd|placeholder|asdf|test123)/i;
|
|
107
|
+
if (vaguePatterns.test(instruction.trim())) {
|
|
108
|
+
issues.push({
|
|
109
|
+
rule: 'task.instruction.quality',
|
|
110
|
+
severity: 'error',
|
|
111
|
+
message: 'Instruction appears to be a placeholder or too vague',
|
|
112
|
+
field: 'instruction',
|
|
113
|
+
suggestion: 'Provide a clear, specific instruction describing the task objective',
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
// --- Rule: Constraints ---
|
|
118
|
+
if (this.config.requireConstraints) {
|
|
119
|
+
rulesApplied.push('task.constraints');
|
|
120
|
+
if (!obj.constraints || !Array.isArray(obj.constraints) || obj.constraints.length === 0) {
|
|
121
|
+
issues.push({
|
|
122
|
+
rule: 'task.constraints',
|
|
123
|
+
severity: 'warning',
|
|
124
|
+
message: 'Task has no constraints defined -- results may be unbounded',
|
|
125
|
+
field: 'constraints',
|
|
126
|
+
suggestion: 'Add constraints like time limits, scope boundaries, or quality thresholds',
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
// --- Rule: Expected output ---
|
|
131
|
+
if (this.config.requireExpectedOutput) {
|
|
132
|
+
rulesApplied.push('task.expectedOutput');
|
|
133
|
+
if (!obj.expectedOutput) {
|
|
134
|
+
issues.push({
|
|
135
|
+
rule: 'task.expectedOutput',
|
|
136
|
+
severity: 'warning',
|
|
137
|
+
message: 'Task has no expectedOutput defined -- validation of results will be weaker',
|
|
138
|
+
field: 'expectedOutput',
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
// --- Custom rules ---
|
|
143
|
+
this.applyCustomRules('task', key, value, issues, rulesApplied);
|
|
144
|
+
return this.makeResult(!issues.some(i => i.severity === 'error'), this.calculateScore(issues), issues, rulesApplied);
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Validate a result/output before caching.
|
|
148
|
+
*/
|
|
149
|
+
validateResult(key, value, metadata) {
|
|
150
|
+
const issues = [];
|
|
151
|
+
const rulesApplied = [];
|
|
152
|
+
// --- Rule: Non-null result ---
|
|
153
|
+
rulesApplied.push('result.existence');
|
|
154
|
+
if (value === null || value === undefined) {
|
|
155
|
+
issues.push({
|
|
156
|
+
rule: 'result.existence',
|
|
157
|
+
severity: 'error',
|
|
158
|
+
message: 'Result value is null or undefined',
|
|
159
|
+
});
|
|
160
|
+
return this.makeResult(false, 0, issues, rulesApplied);
|
|
161
|
+
}
|
|
162
|
+
// --- Rule: Result structure ---
|
|
163
|
+
rulesApplied.push('result.structure');
|
|
164
|
+
if (typeof value === 'object' && !Array.isArray(value)) {
|
|
165
|
+
const obj = value;
|
|
166
|
+
const fieldCount = Object.keys(obj).length;
|
|
167
|
+
if (fieldCount < this.config.minResultFields) {
|
|
168
|
+
issues.push({
|
|
169
|
+
rule: 'result.structure',
|
|
170
|
+
severity: 'warning',
|
|
171
|
+
message: `Result has very few fields (${fieldCount}), expected at least ${this.config.minResultFields}`,
|
|
172
|
+
suggestion: 'Ensure the result contains all expected output data',
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
// --- Rule: Error result check ---
|
|
176
|
+
rulesApplied.push('result.error_check');
|
|
177
|
+
if (obj.error && !obj.data && !obj.result) {
|
|
178
|
+
issues.push({
|
|
179
|
+
rule: 'result.error_check',
|
|
180
|
+
severity: 'error',
|
|
181
|
+
message: 'Result contains only an error -- no useful data',
|
|
182
|
+
field: 'error',
|
|
183
|
+
suggestion: 'Retry the task or handle the error before caching',
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
// --- Rule: Placeholder/dummy data detection ---
|
|
188
|
+
if (this.config.rejectPlaceholders) {
|
|
189
|
+
rulesApplied.push('result.placeholders');
|
|
190
|
+
const serialized = JSON.stringify(value);
|
|
191
|
+
const placeholderPatterns = [
|
|
192
|
+
/lorem ipsum/i,
|
|
193
|
+
/foo\s*bar\s*baz/i,
|
|
194
|
+
/example\.com/i,
|
|
195
|
+
/TODO|FIXME|HACK|XXX/,
|
|
196
|
+
/placeholder/i,
|
|
197
|
+
/dummy[_\s]?data/i,
|
|
198
|
+
/sample[_\s]?data/i,
|
|
199
|
+
/test123|abc123/i,
|
|
200
|
+
/\b0{5,}\b/, // Long runs of zeros
|
|
201
|
+
/\b1234567\b/, // Sequential numbers
|
|
202
|
+
];
|
|
203
|
+
for (const pattern of placeholderPatterns) {
|
|
204
|
+
if (pattern.test(serialized)) {
|
|
205
|
+
issues.push({
|
|
206
|
+
rule: 'result.placeholders',
|
|
207
|
+
severity: 'error',
|
|
208
|
+
message: `Result contains placeholder data (matched: ${pattern.source})`,
|
|
209
|
+
suggestion: 'Ensure the result contains real, production-ready data',
|
|
210
|
+
});
|
|
211
|
+
break; // One flag is enough
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// --- Rule: Hallucination detection ---
|
|
216
|
+
if (this.config.detectHallucinations) {
|
|
217
|
+
rulesApplied.push('result.hallucination');
|
|
218
|
+
const hallucinationIssues = this.detectHallucinations(value, metadata);
|
|
219
|
+
issues.push(...hallucinationIssues);
|
|
220
|
+
}
|
|
221
|
+
// --- Custom rules ---
|
|
222
|
+
this.applyCustomRules('result', key, value, issues, rulesApplied);
|
|
223
|
+
return this.makeResult(!issues.some(i => i.severity === 'error'), this.calculateScore(issues), issues, rulesApplied);
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Validate code content before it enters the blackboard.
|
|
227
|
+
*/
|
|
228
|
+
validateCode(key, value) {
|
|
229
|
+
const issues = [];
|
|
230
|
+
const rulesApplied = [];
|
|
231
|
+
// Extract code string from various formats
|
|
232
|
+
const code = this.extractCode(value);
|
|
233
|
+
if (!code) {
|
|
234
|
+
rulesApplied.push('code.extraction');
|
|
235
|
+
issues.push({
|
|
236
|
+
rule: 'code.extraction',
|
|
237
|
+
severity: 'error',
|
|
238
|
+
message: 'Could not extract code content from value',
|
|
239
|
+
suggestion: 'Value should be a string or an object with a "code", "content", or "source" field',
|
|
240
|
+
});
|
|
241
|
+
return this.makeResult(false, 0, issues, rulesApplied);
|
|
242
|
+
}
|
|
243
|
+
const lines = code.split('\n');
|
|
244
|
+
// --- Rule: Non-trivial code ---
|
|
245
|
+
rulesApplied.push('code.length');
|
|
246
|
+
const codeLines = lines.filter(l => l.trim().length > 0);
|
|
247
|
+
if (codeLines.length < this.config.minCodeLines) {
|
|
248
|
+
issues.push({
|
|
249
|
+
rule: 'code.length',
|
|
250
|
+
severity: 'warning',
|
|
251
|
+
message: `Code is very short (${codeLines.length} non-empty lines)`,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
// --- Rule: Syntax marker checks ---
|
|
255
|
+
rulesApplied.push('code.syntax');
|
|
256
|
+
const syntaxIssues = this.checkCodeSyntax(code);
|
|
257
|
+
issues.push(...syntaxIssues);
|
|
258
|
+
// --- Rule: Comment ratio ---
|
|
259
|
+
rulesApplied.push('code.comment_ratio');
|
|
260
|
+
const commentLines = lines.filter(l => /^\s*(\/\/|#|\/\*|\*|"""|''')/.test(l));
|
|
261
|
+
const ratio = codeLines.length > 0 ? commentLines.length / codeLines.length : 0;
|
|
262
|
+
if (ratio > this.config.maxCommentRatio && codeLines.length > 5) {
|
|
263
|
+
issues.push({
|
|
264
|
+
rule: 'code.comment_ratio',
|
|
265
|
+
severity: 'warning',
|
|
266
|
+
message: `High comment-to-code ratio (${(ratio * 100).toFixed(0)}%) -- may be mostly comments`,
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
// --- Rule: Dangerous patterns ---
|
|
270
|
+
rulesApplied.push('code.dangerous_patterns');
|
|
271
|
+
const dangerousPatterns = [
|
|
272
|
+
{ pattern: /eval\s*\(/, name: 'eval()' },
|
|
273
|
+
{ pattern: /exec\s*\(/, name: 'exec()' },
|
|
274
|
+
{ pattern: /rm\s+-rf\s+\//, name: 'rm -rf /' },
|
|
275
|
+
{ pattern: /DROP\s+TABLE|DROP\s+DATABASE/i, name: 'SQL DROP statements' },
|
|
276
|
+
{ pattern: /process\.env\.\w+/, name: 'Direct env var access' },
|
|
277
|
+
{ pattern: /child_process/, name: 'child_process import' },
|
|
278
|
+
{ pattern: /require\s*\(\s*['"]child_process/, name: 'child_process require' },
|
|
279
|
+
{ pattern: /\.exec\s*\(\s*['"`].*\$\{/, name: 'Command injection via template literal' },
|
|
280
|
+
{ pattern: /(?:password|secret|api_key|token)\s*[:=]\s*['"][^'"]+['"]/i, name: 'Hardcoded credentials' },
|
|
281
|
+
];
|
|
282
|
+
for (const { pattern, name } of dangerousPatterns) {
|
|
283
|
+
if (pattern.test(code)) {
|
|
284
|
+
issues.push({
|
|
285
|
+
rule: 'code.dangerous_patterns',
|
|
286
|
+
severity: 'error',
|
|
287
|
+
message: `Code contains dangerous pattern: ${name}`,
|
|
288
|
+
suggestion: 'Remove this pattern or provide explicit justification',
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
// --- Rule: Placeholder code detection ---
|
|
293
|
+
if (this.config.rejectPlaceholders) {
|
|
294
|
+
rulesApplied.push('code.placeholders');
|
|
295
|
+
const placeholderCode = [
|
|
296
|
+
/\/\/\s*TODO:?\s*implement/i,
|
|
297
|
+
/pass\s*#\s*TODO/i,
|
|
298
|
+
/throw\s+new\s+Error\s*\(\s*['"]Not implemented['"]/i,
|
|
299
|
+
/raise\s+NotImplementedError/i,
|
|
300
|
+
/console\.log\s*\(\s*['"]hello world['"]/i,
|
|
301
|
+
];
|
|
302
|
+
for (const pattern of placeholderCode) {
|
|
303
|
+
if (pattern.test(code)) {
|
|
304
|
+
issues.push({
|
|
305
|
+
rule: 'code.placeholders',
|
|
306
|
+
severity: 'warning',
|
|
307
|
+
message: 'Code contains placeholder/stub patterns -- may be incomplete',
|
|
308
|
+
suggestion: 'Ensure all functions are fully implemented before submission',
|
|
309
|
+
});
|
|
310
|
+
break;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
// --- Custom rules ---
|
|
315
|
+
this.applyCustomRules('code', key, value, issues, rulesApplied);
|
|
316
|
+
return this.makeResult(!issues.some(i => i.severity === 'error'), this.calculateScore(issues), issues, rulesApplied);
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Validate a generic entry (not task, result, or code).
|
|
320
|
+
*/
|
|
321
|
+
validateGeneric(key, value) {
|
|
322
|
+
const issues = [];
|
|
323
|
+
const rulesApplied = [];
|
|
324
|
+
rulesApplied.push('generic.non_null');
|
|
325
|
+
if (value === null || value === undefined) {
|
|
326
|
+
issues.push({
|
|
327
|
+
rule: 'generic.non_null',
|
|
328
|
+
severity: 'error',
|
|
329
|
+
message: 'Value must not be null or undefined',
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
// Run custom rules that apply to 'any'
|
|
333
|
+
this.applyCustomRules('any', key, value, issues, rulesApplied);
|
|
334
|
+
return this.makeResult(!issues.some(i => i.severity === 'error'), this.calculateScore(issues), issues, rulesApplied);
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Register a custom validation rule at runtime.
|
|
338
|
+
*/
|
|
339
|
+
addRule(rule) {
|
|
340
|
+
this.config.customRules.push(rule);
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Update configuration at runtime.
|
|
344
|
+
*/
|
|
345
|
+
updateConfig(patch) {
|
|
346
|
+
Object.assign(this.config, patch);
|
|
347
|
+
}
|
|
348
|
+
// ---------- Private helpers ----------
|
|
349
|
+
detectEntryType(key, value) {
|
|
350
|
+
// Key-prefix-based detection
|
|
351
|
+
if (/^task:/i.test(key))
|
|
352
|
+
return 'task';
|
|
353
|
+
if (/^result:|^output:/i.test(key))
|
|
354
|
+
return 'result';
|
|
355
|
+
if (/^code:|^source:|^file:/i.test(key))
|
|
356
|
+
return 'code';
|
|
357
|
+
// Value-shape-based fallback
|
|
358
|
+
if (typeof value === 'object' && value !== null) {
|
|
359
|
+
const obj = value;
|
|
360
|
+
if ('instruction' in obj)
|
|
361
|
+
return 'task';
|
|
362
|
+
if ('code' in obj || 'source' in obj) {
|
|
363
|
+
const codeField = (obj.code ?? obj.source);
|
|
364
|
+
if (typeof codeField === 'string' && codeField.includes('\n') && codeField.length > 50)
|
|
365
|
+
return 'code';
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
return 'generic';
|
|
369
|
+
}
|
|
370
|
+
extractCode(value) {
|
|
371
|
+
if (typeof value === 'string')
|
|
372
|
+
return value;
|
|
373
|
+
if (typeof value === 'object' && value !== null) {
|
|
374
|
+
const obj = value;
|
|
375
|
+
for (const field of ['code', 'source', 'content', 'body', 'text']) {
|
|
376
|
+
if (typeof obj[field] === 'string')
|
|
377
|
+
return obj[field];
|
|
378
|
+
}
|
|
379
|
+
// Array of files
|
|
380
|
+
if (Array.isArray(obj.files)) {
|
|
381
|
+
return obj.files
|
|
382
|
+
.map(f => f.content ?? '')
|
|
383
|
+
.filter(Boolean)
|
|
384
|
+
.join('\n\n');
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
return null;
|
|
388
|
+
}
|
|
389
|
+
checkCodeSyntax(code) {
|
|
390
|
+
const issues = [];
|
|
391
|
+
// Unmatched brackets/braces/parens
|
|
392
|
+
const opens = { '{': 0, '[': 0, '(': 0 };
|
|
393
|
+
const closes = { '}': '{', ']': '[', ')': '(' };
|
|
394
|
+
let inString = false;
|
|
395
|
+
let stringChar = '';
|
|
396
|
+
for (let i = 0; i < code.length; i++) {
|
|
397
|
+
const ch = code[i];
|
|
398
|
+
// Skip string contents
|
|
399
|
+
if (inString) {
|
|
400
|
+
if (ch === stringChar && code[i - 1] !== '\\')
|
|
401
|
+
inString = false;
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
if (ch === '"' || ch === "'" || ch === '`') {
|
|
405
|
+
inString = true;
|
|
406
|
+
stringChar = ch;
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
// Skip single-line comments
|
|
410
|
+
if (ch === '/' && code[i + 1] === '/') {
|
|
411
|
+
while (i < code.length && code[i] !== '\n')
|
|
412
|
+
i++;
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
if (ch in opens)
|
|
416
|
+
opens[ch]++;
|
|
417
|
+
if (ch in closes)
|
|
418
|
+
opens[closes[ch]]--;
|
|
419
|
+
}
|
|
420
|
+
for (const [bracket, count] of Object.entries(opens)) {
|
|
421
|
+
if (count !== 0) {
|
|
422
|
+
const matchMap = { '{': '}', '[': ']', '(': ')' };
|
|
423
|
+
issues.push({
|
|
424
|
+
rule: 'code.syntax',
|
|
425
|
+
severity: 'error',
|
|
426
|
+
message: `Unmatched bracket: ${count > 0 ? 'missing ' + matchMap[bracket] : 'extra ' + bracket} (${Math.abs(count)} unmatched)`,
|
|
427
|
+
suggestion: 'Check bracket/brace/parenthesis matching',
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return issues;
|
|
432
|
+
}
|
|
433
|
+
detectHallucinations(value, metadata) {
|
|
434
|
+
const issues = [];
|
|
435
|
+
const serialized = JSON.stringify(value);
|
|
436
|
+
// Pattern 1: Fake URLs with realistic-looking but invalid domains
|
|
437
|
+
const fakeUrlPattern = /https?:\/\/(?:www\.)?[a-z]+(?:api|service|endpoint|docs)\.[a-z]{2,}\//gi;
|
|
438
|
+
const urls = serialized.match(fakeUrlPattern) ?? [];
|
|
439
|
+
for (const url of urls) {
|
|
440
|
+
// Flag obviously fake API endpoints
|
|
441
|
+
if (/example-api|fake-service|test-endpoint|placeholder-url/i.test(url)) {
|
|
442
|
+
issues.push({
|
|
443
|
+
rule: 'result.hallucination',
|
|
444
|
+
severity: 'warning',
|
|
445
|
+
message: `Potentially hallucinated URL detected: ${url}`,
|
|
446
|
+
suggestion: 'Verify all URLs are real and accessible',
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
// Pattern 2: Suspicious precision in numeric data (too many decimal places)
|
|
451
|
+
const suspiciousNumbers = serialized.match(/"[^"]*":\s*\d+\.\d{10,}/g);
|
|
452
|
+
if (suspiciousNumbers && suspiciousNumbers.length > 3) {
|
|
453
|
+
issues.push({
|
|
454
|
+
rule: 'result.hallucination',
|
|
455
|
+
severity: 'info',
|
|
456
|
+
message: `Multiple values with unusual precision (${suspiciousNumbers.length} values with 10+ decimal places)`,
|
|
457
|
+
suggestion: 'Verify numeric data comes from a real source -- excessive precision may indicate hallucination',
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
// Pattern 3: Contradictory data within the same result
|
|
461
|
+
if (typeof value === 'object' && value !== null) {
|
|
462
|
+
const obj = value;
|
|
463
|
+
// Revenue > total but expenses also > total
|
|
464
|
+
if (typeof obj.revenue === 'number' && typeof obj.expenses === 'number' && typeof obj.total === 'number') {
|
|
465
|
+
if (obj.revenue > obj.total && obj.expenses > obj.total) {
|
|
466
|
+
issues.push({
|
|
467
|
+
rule: 'result.hallucination',
|
|
468
|
+
severity: 'warning',
|
|
469
|
+
message: 'Contradictory numeric data: both revenue and expenses exceed total',
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
// Success: true but error field present
|
|
474
|
+
if (obj.success === true && obj.error && typeof obj.error === 'string' && obj.error.length > 0) {
|
|
475
|
+
issues.push({
|
|
476
|
+
rule: 'result.hallucination',
|
|
477
|
+
severity: 'warning',
|
|
478
|
+
message: 'Contradictory state: success=true but error field contains a message',
|
|
479
|
+
});
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
// Pattern 4: Repetitive content (copy-paste hallucination)
|
|
483
|
+
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
|
484
|
+
const values = Object.values(value)
|
|
485
|
+
.filter(v => typeof v === 'string' && v.length > 20);
|
|
486
|
+
const unique = new Set(values.map(v => v.toLowerCase().trim()));
|
|
487
|
+
if (values.length >= 3 && unique.size < values.length * 0.5) {
|
|
488
|
+
issues.push({
|
|
489
|
+
rule: 'result.hallucination',
|
|
490
|
+
severity: 'warning',
|
|
491
|
+
message: `Highly repetitive content: ${values.length} string fields but only ${unique.size} unique values`,
|
|
492
|
+
suggestion: 'Check if the agent is copying the same output across multiple fields',
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
// Pattern 5: Fabricated references (papers, docs)
|
|
497
|
+
const fakeRefPatterns = [
|
|
498
|
+
/arXiv:\d{4}\.\d{5,}/g, // Fake arXiv IDs
|
|
499
|
+
/doi:\s*10\.\d{4}/g, // Fake DOIs
|
|
500
|
+
/ISBN\s*\d{10,13}/g, // Fake ISBNs
|
|
501
|
+
];
|
|
502
|
+
for (const pattern of fakeRefPatterns) {
|
|
503
|
+
const matches = serialized.match(pattern);
|
|
504
|
+
if (matches && matches.length > 0) {
|
|
505
|
+
issues.push({
|
|
506
|
+
rule: 'result.hallucination',
|
|
507
|
+
severity: 'info',
|
|
508
|
+
message: `Result contains ${matches.length} academic reference(s) -- verify they are real`,
|
|
509
|
+
suggestion: 'AI models commonly hallucinate paper titles, DOIs, and arXiv IDs',
|
|
510
|
+
});
|
|
511
|
+
break;
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
return issues;
|
|
515
|
+
}
|
|
516
|
+
applyCustomRules(entryType, key, value, issues, rulesApplied) {
|
|
517
|
+
for (const rule of this.config.customRules) {
|
|
518
|
+
if (rule.appliesTo.includes(entryType) || rule.appliesTo.includes('any')) {
|
|
519
|
+
rulesApplied.push(`custom:${rule.name}`);
|
|
520
|
+
const issue = rule.validate(key, value);
|
|
521
|
+
if (issue)
|
|
522
|
+
issues.push(issue);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
calculateScore(issues) {
|
|
527
|
+
let score = 1.0;
|
|
528
|
+
for (const issue of issues) {
|
|
529
|
+
switch (issue.severity) {
|
|
530
|
+
case 'error':
|
|
531
|
+
score -= 0.3;
|
|
532
|
+
break;
|
|
533
|
+
case 'warning':
|
|
534
|
+
score -= 0.1;
|
|
535
|
+
break;
|
|
536
|
+
case 'info':
|
|
537
|
+
score -= 0.02;
|
|
538
|
+
break;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
return Math.max(0, Math.min(1, score));
|
|
542
|
+
}
|
|
543
|
+
makeResult(passed, score, issues, rulesApplied) {
|
|
544
|
+
return {
|
|
545
|
+
passed,
|
|
546
|
+
score,
|
|
547
|
+
issues,
|
|
548
|
+
rulesApplied,
|
|
549
|
+
timestamp: new Date().toISOString(),
|
|
550
|
+
recoverable: issues.every(i => i.severity !== 'error' || i.suggestion !== undefined),
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
exports.BlackboardValidator = BlackboardValidator;
|
|
555
|
+
// ============================================================================
|
|
556
|
+
// LAYER 2: QUALITY GATE AGENT -- AI-assisted review
|
|
557
|
+
// ============================================================================
|
|
558
|
+
class QualityGateAgent {
|
|
559
|
+
validator;
|
|
560
|
+
quarantine = new Map();
|
|
561
|
+
reviewCallback;
|
|
562
|
+
metrics = {
|
|
563
|
+
totalChecked: 0,
|
|
564
|
+
approved: 0,
|
|
565
|
+
rejected: 0,
|
|
566
|
+
quarantined: 0,
|
|
567
|
+
aiReviewed: 0,
|
|
568
|
+
};
|
|
569
|
+
/** Quality score threshold: entries below this go to AI review or quarantine */
|
|
570
|
+
qualityThreshold;
|
|
571
|
+
/** Score below which entries are auto-rejected (no AI review) */
|
|
572
|
+
autoRejectThreshold;
|
|
573
|
+
/** Whether to invoke AI review for borderline entries */
|
|
574
|
+
aiReviewEnabled;
|
|
575
|
+
constructor(options) {
|
|
576
|
+
this.validator = new BlackboardValidator(options?.validationConfig);
|
|
577
|
+
this.qualityThreshold = options?.qualityThreshold ?? 0.7;
|
|
578
|
+
this.autoRejectThreshold = options?.autoRejectThreshold ?? 0.3;
|
|
579
|
+
this.reviewCallback = options?.aiReviewCallback;
|
|
580
|
+
this.aiReviewEnabled = !!options?.aiReviewCallback;
|
|
581
|
+
}
|
|
582
|
+
/**
|
|
583
|
+
* Gate an entry -- validate, optionally send for AI review, and decide.
|
|
584
|
+
*
|
|
585
|
+
* Call this before writing to the blackboard. Returns a decision:
|
|
586
|
+
* - 'approve': safe to write
|
|
587
|
+
* - 'reject': do not write, return error to submitting agent
|
|
588
|
+
* - 'quarantine': stored separately for human/senior-agent review
|
|
589
|
+
* - 'needs_review': requires AI review (only if callback is set)
|
|
590
|
+
*/
|
|
591
|
+
async gate(key, value, sourceAgent, metadata) {
|
|
592
|
+
this.metrics.totalChecked++;
|
|
593
|
+
// Layer 1: Rule-based validation
|
|
594
|
+
const validation = this.validator.validate(key, value, metadata);
|
|
595
|
+
const reviewNotes = [];
|
|
596
|
+
// Auto-reject: too many hard errors
|
|
597
|
+
if (validation.score < this.autoRejectThreshold) {
|
|
598
|
+
this.metrics.rejected++;
|
|
599
|
+
reviewNotes.push(`Auto-rejected: score ${validation.score.toFixed(2)} below threshold ${this.autoRejectThreshold}`);
|
|
600
|
+
return {
|
|
601
|
+
decision: 'reject',
|
|
602
|
+
validation,
|
|
603
|
+
reviewNotes,
|
|
604
|
+
};
|
|
605
|
+
}
|
|
606
|
+
// Clean pass: no issues, high quality
|
|
607
|
+
if (validation.passed && validation.score >= this.qualityThreshold) {
|
|
608
|
+
this.metrics.approved++;
|
|
609
|
+
reviewNotes.push(`Approved: score ${validation.score.toFixed(2)}, ${validation.rulesApplied.length} rules passed`);
|
|
610
|
+
return {
|
|
611
|
+
decision: 'approve',
|
|
612
|
+
validation,
|
|
613
|
+
reviewNotes,
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
// Borderline: send for AI review if available
|
|
617
|
+
if (this.aiReviewEnabled && this.reviewCallback) {
|
|
618
|
+
this.metrics.aiReviewed++;
|
|
619
|
+
reviewNotes.push(`Borderline score ${validation.score.toFixed(2)} -- sending for AI review`);
|
|
620
|
+
try {
|
|
621
|
+
const entryType = this.detectEntryType(key, value);
|
|
622
|
+
const aiResult = await this.reviewCallback(key, value, entryType, {
|
|
623
|
+
sourceAgent,
|
|
624
|
+
validation,
|
|
625
|
+
});
|
|
626
|
+
reviewNotes.push(`AI review: ${aiResult.approved ? 'APPROVED' : 'REJECTED'} (confidence: ${aiResult.confidence.toFixed(2)})`);
|
|
627
|
+
reviewNotes.push(`AI feedback: ${aiResult.feedback}`);
|
|
628
|
+
if (aiResult.suggestedFixes) {
|
|
629
|
+
reviewNotes.push(`Suggested fixes: ${aiResult.suggestedFixes.join('; ')}`);
|
|
630
|
+
}
|
|
631
|
+
if (aiResult.approved && aiResult.confidence >= 0.6) {
|
|
632
|
+
this.metrics.approved++;
|
|
633
|
+
return {
|
|
634
|
+
decision: 'approve',
|
|
635
|
+
validation,
|
|
636
|
+
reviewNotes,
|
|
637
|
+
reviewedBy: 'ai_reviewer',
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
else {
|
|
641
|
+
// AI rejected or low confidence -- quarantine
|
|
642
|
+
const qKey = this.addToQuarantine(key, value, validation.issues, sourceAgent);
|
|
643
|
+
this.metrics.quarantined++;
|
|
644
|
+
return {
|
|
645
|
+
decision: 'quarantine',
|
|
646
|
+
validation,
|
|
647
|
+
quarantineKey: qKey,
|
|
648
|
+
reviewNotes,
|
|
649
|
+
reviewedBy: 'ai_reviewer',
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
catch (err) {
|
|
654
|
+
reviewNotes.push(`AI review failed: ${err instanceof Error ? err.message : 'unknown error'}`);
|
|
655
|
+
// Fall through to quarantine
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
// No AI review available or AI review failed -- quarantine or reject based on severity
|
|
659
|
+
if (validation.passed) {
|
|
660
|
+
// Passed rules but low quality score -- quarantine for review
|
|
661
|
+
const qKey = this.addToQuarantine(key, value, validation.issues, sourceAgent);
|
|
662
|
+
this.metrics.quarantined++;
|
|
663
|
+
reviewNotes.push(`Quarantined: passed rules but score ${validation.score.toFixed(2)} below quality threshold ${this.qualityThreshold}`);
|
|
664
|
+
return {
|
|
665
|
+
decision: 'quarantine',
|
|
666
|
+
validation,
|
|
667
|
+
quarantineKey: qKey,
|
|
668
|
+
reviewNotes,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
else {
|
|
672
|
+
// Hard rule failures -- reject
|
|
673
|
+
this.metrics.rejected++;
|
|
674
|
+
reviewNotes.push('Rejected: failed validation rules');
|
|
675
|
+
return {
|
|
676
|
+
decision: 'reject',
|
|
677
|
+
validation,
|
|
678
|
+
reviewNotes,
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
/**
|
|
683
|
+
* Get all quarantined entries for manual review.
|
|
684
|
+
*/
|
|
685
|
+
getQuarantined() {
|
|
686
|
+
return Array.from(this.quarantine.entries()).map(([id, entry]) => ({
|
|
687
|
+
quarantineId: id,
|
|
688
|
+
...entry,
|
|
689
|
+
}));
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* Approve a quarantined entry -- returns the value for writing to the blackboard.
|
|
693
|
+
*/
|
|
694
|
+
approveQuarantined(quarantineId) {
|
|
695
|
+
const entry = this.quarantine.get(quarantineId);
|
|
696
|
+
if (!entry)
|
|
697
|
+
return null;
|
|
698
|
+
this.quarantine.delete(quarantineId);
|
|
699
|
+
this.metrics.approved++;
|
|
700
|
+
this.metrics.quarantined--;
|
|
701
|
+
return { key: entry.key, value: entry.value };
|
|
702
|
+
}
|
|
703
|
+
/**
|
|
704
|
+
* Reject and discard a quarantined entry.
|
|
705
|
+
*/
|
|
706
|
+
rejectQuarantined(quarantineId) {
|
|
707
|
+
if (!this.quarantine.has(quarantineId))
|
|
708
|
+
return false;
|
|
709
|
+
this.quarantine.delete(quarantineId);
|
|
710
|
+
this.metrics.rejected++;
|
|
711
|
+
this.metrics.quarantined--;
|
|
712
|
+
return true;
|
|
713
|
+
}
|
|
714
|
+
/**
|
|
715
|
+
* Get quality gate metrics.
|
|
716
|
+
*/
|
|
717
|
+
getMetrics() {
|
|
718
|
+
return { ...this.metrics };
|
|
719
|
+
}
|
|
720
|
+
/**
|
|
721
|
+
* Get the underlying validator for direct access (e.g., adding custom rules).
|
|
722
|
+
*/
|
|
723
|
+
getValidator() {
|
|
724
|
+
return this.validator;
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* Set or change the AI review callback at runtime.
|
|
728
|
+
*/
|
|
729
|
+
setAIReviewCallback(callback) {
|
|
730
|
+
this.reviewCallback = callback;
|
|
731
|
+
this.aiReviewEnabled = true;
|
|
732
|
+
}
|
|
733
|
+
// ---------- Private helpers ----------
|
|
734
|
+
addToQuarantine(key, value, issues, submittedBy) {
|
|
735
|
+
const id = `quarantine_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
736
|
+
this.quarantine.set(id, {
|
|
737
|
+
key,
|
|
738
|
+
value,
|
|
739
|
+
issues,
|
|
740
|
+
submittedBy,
|
|
741
|
+
timestamp: new Date().toISOString(),
|
|
742
|
+
});
|
|
743
|
+
return id;
|
|
744
|
+
}
|
|
745
|
+
detectEntryType(key, value) {
|
|
746
|
+
if (/^task:.*:pending|^task:.*:instruction/i.test(key))
|
|
747
|
+
return 'task';
|
|
748
|
+
if (/^task:.*:result|^result:|^output:/i.test(key))
|
|
749
|
+
return 'result';
|
|
750
|
+
if (/^code:|^source:|^file:/i.test(key))
|
|
751
|
+
return 'code';
|
|
752
|
+
return 'generic';
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
exports.QualityGateAgent = QualityGateAgent;
|
|
756
|
+
//# sourceMappingURL=blackboard-validator.js.map
|