@evomap/evolver-core 2.0.0-beta.7 → 2.0.0-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/algo/conversationSniffer.js +20 -7
- package/dist/assetstore/localJsonl.js +2 -2
- package/dist/hub/sanitize.js +3 -2
- package/dist/strategy/constraintAblation.d.ts +64 -0
- package/dist/strategy/constraintAblation.js +3074 -0
- package/dist/strategy/index.d.ts +2 -1
- package/dist/strategy/index.js +2 -1
- package/package.json +1 -1
|
@@ -4,9 +4,13 @@ import { redactString } from '../hub/sanitize.js';
|
|
|
4
4
|
// occur in nearly every Chinese dev session, and matching them bare made this fallback the DOMINANT distill
|
|
5
5
|
// path — 105 of 118 drafts in one bulk ingest (#562). English gets specificity from \b word boundaries;
|
|
6
6
|
// Chinese has no \b, so specificity must come from the phrase itself.
|
|
7
|
-
const REUSABLE_RE = /\b(reusable|repeatable|workflow|playbook|runbook|capability|procedure|pattern|recipe|documented|future runs?|next time|can reuse|reuse this)\b
|
|
7
|
+
const REUSABLE_RE = /\b(reusable|repeatable|workflow|playbook|runbook|capability|procedure|pattern|recipe|documented|future runs?|next time|can reuse|reuse this)\b|复用|可重用|工作流(?!程)|方法论|沉淀/i;
|
|
8
8
|
const PROOF_RE = /\b(validated|verified|passed|green|success(?:ful|fully)?|succeeded|works?|completed|published|uploaded|recorded:true|exit code:?\s*0|all tests passed)\b|(?:验证|校验|测试|检查|构建|编译|运行|执行|部署|发布)(?:都|均|全部)?(?:通过|成功)|全部通过|跑通|已(?:验证|发布|上线)/i;
|
|
9
9
|
const FAILURE_RE = /\b(failed|failure|error|exception|traceback|exit code:?\s*[1-9]|not working|unable to)\b|失败|错误|报错/i;
|
|
10
|
+
const CHINESE_NEGATED_PROOF_PREFIX_SOURCE = '(?:尚未|并未|还没有|尚没有|并没有|没有|未能|无法|不能|不曾|并非|未|没)';
|
|
11
|
+
const CHINESE_NEGATED_PROOF_SOURCE = `(?:${CHINESE_NEGATED_PROOF_PREFIX_SOURCE}\\s*(?:验证|校验|测试|检查|构建|编译|运行|执行|部署|发布)(?:都|均|全部)?\\s*(?:通过|成功)|(?:验证|校验|测试|检查|构建|编译|运行|执行|部署|发布)(?:仍|还|尚)?(?:没有|未能|未)(?:都|均|全部)?\\s*(?:通过|成功))`;
|
|
12
|
+
const CHINESE_NEGATED_PROOF_RE = new RegExp(CHINESE_NEGATED_PROOF_SOURCE, 'i');
|
|
13
|
+
const CHINESE_NEGATED_PROOF_GLOBAL_RE = new RegExp(CHINESE_NEGATED_PROOF_SOURCE, 'gi');
|
|
10
14
|
const EXIT_ZERO_RE = /\bexit code:?\s*0\b/i;
|
|
11
15
|
const EXIT_NON_ZERO_RE = /\bexit code:?\s*[1-9]\d*\b/i;
|
|
12
16
|
const DOMAIN_TOKENS = [
|
|
@@ -58,15 +62,24 @@ function domainSignals(text, toolCalls) {
|
|
|
58
62
|
function isToolTurn(turn) {
|
|
59
63
|
return turn.role === 'tool' || Boolean(turn.toolName);
|
|
60
64
|
}
|
|
65
|
+
function hasPositiveProof(text) {
|
|
66
|
+
return PROOF_RE.test(text.replace(CHINESE_NEGATED_PROOF_GLOBAL_RE, ''));
|
|
67
|
+
}
|
|
68
|
+
function hasFailure(text) {
|
|
69
|
+
return FAILURE_RE.test(text) || CHINESE_NEGATED_PROOF_RE.test(text);
|
|
70
|
+
}
|
|
61
71
|
function classifyToolOutcome(turn) {
|
|
62
72
|
const text = turnText(turn);
|
|
63
73
|
if (EXIT_NON_ZERO_RE.test(text))
|
|
64
74
|
return 'failure';
|
|
75
|
+
// A semantic "not yet successful" result is not proof even when the command process itself exited zero.
|
|
76
|
+
if (CHINESE_NEGATED_PROOF_RE.test(text))
|
|
77
|
+
return 'failure';
|
|
65
78
|
if (EXIT_ZERO_RE.test(text))
|
|
66
79
|
return 'success';
|
|
67
|
-
if (
|
|
80
|
+
if (hasPositiveProof(text) && !FAILURE_RE.test(text))
|
|
68
81
|
return 'success';
|
|
69
|
-
if (
|
|
82
|
+
if (hasFailure(text))
|
|
70
83
|
return 'failure';
|
|
71
84
|
return 'unknown';
|
|
72
85
|
}
|
|
@@ -85,7 +98,7 @@ function pickEvidence(turns, matcher, max = 3) {
|
|
|
85
98
|
const evidence = [];
|
|
86
99
|
for (const turn of turns) {
|
|
87
100
|
const text = turnText(turn);
|
|
88
|
-
if (!text || !matcher.test(text))
|
|
101
|
+
if (!text || !(typeof matcher === 'function' ? matcher(text) : matcher.test(text)))
|
|
89
102
|
continue;
|
|
90
103
|
evidence.push(cleanText(text, 180));
|
|
91
104
|
if (evidence.length >= max)
|
|
@@ -97,7 +110,7 @@ function pickSummary(turns, reusableEvidence, proofEvidence) {
|
|
|
97
110
|
const assistant = turns
|
|
98
111
|
.filter((turn) => turn.role === 'assistant' && !turn.isMeta)
|
|
99
112
|
.map(turnText)
|
|
100
|
-
.find((text) => text.length >= 40 && (REUSABLE_RE.test(text) ||
|
|
113
|
+
.find((text) => text.length >= 40 && (REUSABLE_RE.test(text) || hasPositiveProof(text)));
|
|
101
114
|
return cleanText(assistant ?? reusableEvidence[0] ?? proofEvidence[0] ?? 'Verified reusable conversation capability.', 180);
|
|
102
115
|
}
|
|
103
116
|
export function sniffConversationCapabilities(turns, opts = {}) {
|
|
@@ -108,10 +121,10 @@ export function sniffConversationCapabilities(turns, opts = {}) {
|
|
|
108
121
|
if (!allText.trim())
|
|
109
122
|
return [];
|
|
110
123
|
const reusableEvidence = pickEvidence(usableTurns, REUSABLE_RE);
|
|
111
|
-
const proofEvidence = pickEvidence(usableTurns,
|
|
124
|
+
const proofEvidence = pickEvidence(usableTurns, hasPositiveProof);
|
|
112
125
|
if (reusableEvidence.length === 0 || proofEvidence.length === 0)
|
|
113
126
|
return [];
|
|
114
|
-
const failureOnly =
|
|
127
|
+
const failureOnly = hasFailure(allText) && !hasPositiveProof(allText);
|
|
115
128
|
if (failureOnly)
|
|
116
129
|
return [];
|
|
117
130
|
if (!hasTerminalSuccessfulToolRun(usableTurns))
|
|
@@ -180,8 +180,8 @@ export class LocalJsonlProvider {
|
|
|
180
180
|
if (q.gene && r.gene !== q.gene)
|
|
181
181
|
continue;
|
|
182
182
|
if (q.signalsAny && q.signalsAny.length > 0) {
|
|
183
|
-
const sig = new Set(signalsOf(r));
|
|
184
|
-
if (!q.signalsAny.some((
|
|
183
|
+
const sig = new Set(signalsOf(r).map((signal) => signal.trim().toLowerCase()));
|
|
184
|
+
if (!q.signalsAny.some((signal) => sig.has(signal.trim().toLowerCase())))
|
|
185
185
|
continue;
|
|
186
186
|
}
|
|
187
187
|
if (q.text) {
|
package/dist/hub/sanitize.js
CHANGED
|
@@ -16,11 +16,12 @@ const REDACT_PATTERNS = [
|
|
|
16
16
|
/api[_-]?key[=:]\s*["']?[A-Za-z0-9-._~+/]{16,}["']?/gi,
|
|
17
17
|
/secret[=:]\s*["']?[A-Za-z0-9-._~+/]{16,}["']?/gi,
|
|
18
18
|
/password[=:]\s*["']?[^\s"',;)}\]]{6,}["']?/gi,
|
|
19
|
-
// GitHub tokens (ghp_, gho_, ghu_, ghs_, github_pat_)
|
|
19
|
+
// GitHub tokens (ghp_, gho_, ghu_, ghs_, ghr_, github_pat_)
|
|
20
20
|
/ghp_[A-Za-z0-9]{36,}/g,
|
|
21
21
|
/gho_[A-Za-z0-9]{36,}/g,
|
|
22
22
|
/ghu_[A-Za-z0-9]{36,}/g,
|
|
23
23
|
/ghs_[A-Za-z0-9]{36,}/g,
|
|
24
|
+
/ghr_[A-Za-z0-9]{36,}/g,
|
|
24
25
|
/github_pat_[A-Za-z0-9_]{22,}/g,
|
|
25
26
|
// AWS access keys
|
|
26
27
|
/AKIA[0-9A-Z]{16}/g,
|
|
@@ -139,7 +140,7 @@ const LEAK_SCANNERS = [
|
|
|
139
140
|
{ type: 'api_key', pattern: /sk-proj-[A-Za-z0-9-_]{20,}/g, suggest: 'process.env.OPENAI_API_KEY' },
|
|
140
141
|
{ type: 'api_key', pattern: /sk-ant-[A-Za-z0-9-_]{20,}/g, suggest: 'process.env.ANTHROPIC_API_KEY' },
|
|
141
142
|
{ type: 'api_key', pattern: /AKIA[0-9A-Z]{16}/g, suggest: 'process.env.AWS_ACCESS_KEY_ID' },
|
|
142
|
-
{ type: 'github_token', pattern: /
|
|
143
|
+
{ type: 'github_token', pattern: /gh(?:p|o|u|s|r)_[A-Za-z0-9]{36,}/g, suggest: 'process.env.GITHUB_TOKEN' },
|
|
143
144
|
{ type: 'github_token', pattern: /github_pat_[A-Za-z0-9_]{22,}/g, suggest: 'process.env.GITHUB_TOKEN' },
|
|
144
145
|
{ type: 'npm_token', pattern: /npm_[A-Za-z0-9]{36,}/g, suggest: 'process.env.NPM_TOKEN' },
|
|
145
146
|
{ type: 'slack_token', pattern: /xox[baprsv]-[A-Za-z0-9-]{10,}/g, suggest: 'process.env.SLACK_TOKEN' },
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
export type ConstraintKind = 'must' | 'must_not';
|
|
2
|
+
export type ConstraintTraceSource = 'plan' | 'task' | 'trace';
|
|
3
|
+
export type ConstraintSeverity = 'low' | 'medium' | 'high';
|
|
4
|
+
export type SensitiveClass = 'credential' | 'email' | 'filesystem_path';
|
|
5
|
+
export type TaskSuccessStatus = 'success' | 'failure' | 'unknown';
|
|
6
|
+
export type TaskSuccessSource = 'oracle' | 'task_status' | 'manual' | 'trace';
|
|
7
|
+
export type SensitivitySuccessComparison = 'success_constraint_sensitive' | 'success_constraint_insensitive' | 'failure_constraint_sensitive' | 'failure_constraint_insensitive' | 'unknown_success';
|
|
8
|
+
export interface ConstraintTrace {
|
|
9
|
+
source: ConstraintTraceSource;
|
|
10
|
+
text: string;
|
|
11
|
+
traceId?: string;
|
|
12
|
+
}
|
|
13
|
+
export interface ExtractedConstraint {
|
|
14
|
+
id: string;
|
|
15
|
+
kind: ConstraintKind;
|
|
16
|
+
textHash: string;
|
|
17
|
+
redactedText: string;
|
|
18
|
+
source: ConstraintTraceSource;
|
|
19
|
+
traceId?: string;
|
|
20
|
+
sensitiveClasses: SensitiveClass[];
|
|
21
|
+
}
|
|
22
|
+
export interface ConstraintAblatedPrompt {
|
|
23
|
+
originalPromptHash: string;
|
|
24
|
+
ablatedPromptHash: string;
|
|
25
|
+
removedConstraintIds: string[];
|
|
26
|
+
redactedPreview?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface ConstraintViolation {
|
|
29
|
+
constraintId: string;
|
|
30
|
+
kind: ConstraintKind;
|
|
31
|
+
severity: ConstraintSeverity;
|
|
32
|
+
evidenceHash: string;
|
|
33
|
+
matchedTerms: string[];
|
|
34
|
+
}
|
|
35
|
+
export interface TaskSuccessLabel {
|
|
36
|
+
status: TaskSuccessStatus;
|
|
37
|
+
source: TaskSuccessSource;
|
|
38
|
+
}
|
|
39
|
+
export interface ConstraintAblationScore {
|
|
40
|
+
source: 'constraint_ablation_replay';
|
|
41
|
+
sensitivity: number;
|
|
42
|
+
ablationCount: number;
|
|
43
|
+
baselineViolationCount: number;
|
|
44
|
+
ablatedViolationCount: number;
|
|
45
|
+
mustViolationCount: number;
|
|
46
|
+
mustNotViolationCount: number;
|
|
47
|
+
taskSuccess: TaskSuccessLabel;
|
|
48
|
+
comparison: SensitivitySuccessComparison;
|
|
49
|
+
}
|
|
50
|
+
export interface ConstraintAblationScoreInput {
|
|
51
|
+
baselineViolations: readonly ConstraintViolation[];
|
|
52
|
+
ablatedViolations: readonly ConstraintViolation[];
|
|
53
|
+
removedConstraintIds: readonly string[];
|
|
54
|
+
ablationCount: number;
|
|
55
|
+
taskSuccess: TaskSuccessLabel;
|
|
56
|
+
sensitivityThreshold?: number;
|
|
57
|
+
}
|
|
58
|
+
export declare function redactConstraintText(text: string): string;
|
|
59
|
+
export declare function extractConstraints(traces: readonly ConstraintTrace[]): ExtractedConstraint[];
|
|
60
|
+
export declare function buildConstraintAblatedPrompts(prompt: string, constraints: readonly ExtractedConstraint[], opts?: {
|
|
61
|
+
includeRedactedPreview?: boolean;
|
|
62
|
+
}): ConstraintAblatedPrompt[];
|
|
63
|
+
export declare function detectConstraintViolations(output: string, constraints: readonly ExtractedConstraint[]): ConstraintViolation[];
|
|
64
|
+
export declare function computeConstraintAblationScore(input: ConstraintAblationScoreInput): ConstraintAblationScore;
|