@evomap/evolver-core 2.0.0-beta.8 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/algo/candidateAssembly.d.ts +7 -2
- package/dist/algo/candidateAssembly.js +81 -13
- package/dist/algo/capabilityCandidates.js +10 -0
- package/dist/algo/conversationSniffer.js +20 -7
- package/dist/algo/cycleEngine.d.ts +23 -2
- package/dist/algo/cycleEngine.js +236 -36
- package/dist/algo/exploration.d.ts +7 -0
- package/dist/algo/exploration.js +16 -3
- package/dist/algo/geneHealth.d.ts +2 -2
- package/dist/algo/geneHealth.js +5 -4
- package/dist/algo/geneSelection.d.ts +63 -5
- package/dist/algo/geneSelection.js +330 -30
- package/dist/algo/index.d.ts +2 -0
- package/dist/algo/index.js +2 -0
- package/dist/algo/orchestrator.d.ts +8 -1
- package/dist/algo/orchestrator.js +15 -3
- package/dist/algo/publishEligibility.d.ts +34 -0
- package/dist/algo/publishEligibility.js +52 -0
- package/dist/algo/ucb1.d.ts +53 -0
- package/dist/algo/ucb1.js +156 -0
- package/dist/assetstore/assetSidecarRecords.d.ts +7 -2
- package/dist/assetstore/assetSidecarRecords.js +238 -0
- package/dist/assetstore/assetSidecarRecovery.js +2 -2
- package/dist/assetstore/assetStoreHealth.d.ts +6 -0
- package/dist/assetstore/assetStoreHealth.js +56 -14
- package/dist/assetstore/assetStoreStorage.d.ts +1 -1
- package/dist/assetstore/assetStoreStorage.js +27 -9
- package/dist/assetstore/assetSyncLedger.d.ts +86 -0
- package/dist/assetstore/assetSyncLedger.js +683 -4
- package/dist/assetstore/index.d.ts +1 -0
- package/dist/assetstore/index.js +1 -0
- package/dist/assetstore/localAssetStoreSnapshot.d.ts +51 -0
- package/dist/assetstore/localAssetStoreSnapshot.js +329 -0
- package/dist/assetstore/localJsonl.d.ts +4 -2
- package/dist/assetstore/localJsonl.js +91 -11
- package/dist/assetstore/provenance.d.ts +80 -4
- package/dist/assetstore/provenance.js +313 -5
- package/dist/assetstore/provider.d.ts +58 -1
- package/dist/assetstore/provider.js +97 -6
- package/dist/benchmark/antiGeneBenchmark.d.ts +2 -0
- package/dist/benchmark/antiGeneBenchmark.js +4 -3
- package/dist/benchmark/antiGeneRollout.d.ts +2 -0
- package/dist/benchmark/antiGeneRollout.js +4 -3
- package/dist/benchmark/index.d.ts +2 -1
- package/dist/benchmark/index.js +2 -1
- package/dist/benchmark/selectionFlatAbstention.d.ts +152 -0
- package/dist/benchmark/selectionFlatAbstention.js +481 -0
- package/dist/bootstrap/envFingerprint.d.ts +9 -0
- package/dist/bootstrap/envFingerprint.js +5 -0
- package/dist/bootstrap/index.d.ts +2 -1
- package/dist/bootstrap/index.js +2 -1
- package/dist/bootstrap/v1EnvCompat.d.ts +111 -0
- package/dist/bootstrap/v1EnvCompat.js +280 -0
- package/dist/events/eventArchive.d.ts +2 -0
- package/dist/events/eventArchive.js +13 -3
- package/dist/events/eventSchema.d.ts +7 -7
- package/dist/events/eventStore.d.ts +2 -0
- package/dist/events/eventStore.js +5 -1
- package/dist/events/ingest.d.ts +1 -0
- package/dist/events/ingest.js +1 -0
- package/dist/events/paths.d.ts +3 -1
- package/dist/events/paths.js +4 -0
- package/dist/events/public.d.ts +2 -2
- package/dist/events/public.js +2 -2
- package/dist/events/reports.d.ts +2 -0
- package/dist/events/reports.js +4 -0
- package/dist/exec/autoExec.d.ts +43 -2
- package/dist/exec/autoExec.js +76 -9
- package/dist/exec/autonomousCycle.d.ts +22 -4
- package/dist/exec/autonomousCycle.js +64 -13
- package/dist/exec/claudeBridge.d.ts +32 -7
- package/dist/exec/claudeBridge.js +281 -29
- package/dist/exec/prompt.js +5 -1
- package/dist/exec/runnerRegistry.d.ts +68 -26
- package/dist/exec/runnerRegistry.js +307 -72
- package/dist/exec/selfPr.js +1 -7
- package/dist/feedback/envelope.d.ts +61 -0
- package/dist/feedback/envelope.js +168 -0
- package/dist/feedback/index.d.ts +1 -0
- package/dist/feedback/index.js +1 -0
- package/dist/hooks/hooks.js +1 -0
- package/dist/hub/assetCallLog.d.ts +35 -1
- package/dist/hub/assetCallLog.js +124 -1
- package/dist/hub/bindings.d.ts +8 -1
- package/dist/hub/bindings.js +17 -6
- package/dist/hub/capability.d.ts +90 -3
- package/dist/hub/fake.d.ts +2 -2
- package/dist/hub/fake.js +1 -1
- package/dist/hub/questionGenerator.d.ts +5 -1
- package/dist/hub/questionGenerator.js +8 -6
- package/dist/hub/sanitize.js +3 -2
- package/dist/index.d.ts +3 -1
- package/dist/index.js +4 -1
- package/dist/mailbox/dispatch.d.ts +1 -1
- package/dist/mailbox/dispatch.js +22 -6
- package/dist/mailbox/envelope.d.ts +7 -1
- package/dist/mailbox/envelope.js +9 -2
- package/dist/mailbox/ipcServer.d.ts +10 -2
- package/dist/mailbox/ipcServer.js +163 -13
- package/dist/mailbox/store.d.ts +89 -3
- package/dist/mailbox/store.js +895 -41
- package/dist/ops/evolutionGraphProjection.d.ts +20 -0
- package/dist/ops/evolutionGraphProjection.js +315 -0
- package/dist/ops/index.d.ts +2 -1
- package/dist/ops/index.js +2 -1
- package/dist/ops/selfUpdate.d.ts +8 -0
- package/dist/ops/selfUpdate.js +24 -8
- package/dist/personality/schema.d.ts +16 -16
- package/dist/schema/evolutionGraph.d.ts +784 -0
- package/dist/schema/evolutionGraph.js +187 -0
- package/dist/schema/index.d.ts +1 -0
- package/dist/schema/index.js +1 -0
- package/dist/schema/signal.d.ts +3 -3
- package/dist/schema/signal.js +1 -1
- package/dist/signals/curriculum.d.ts +55 -0
- package/dist/signals/curriculum.js +202 -0
- package/dist/signals/cycleHistoryFromEvents.js +17 -8
- package/dist/signals/expand.d.ts +15 -1
- package/dist/signals/expand.js +169 -1
- package/dist/signals/extractor.d.ts +2 -2
- package/dist/signals/extractor.js +31 -6
- package/dist/signals/index.d.ts +3 -1
- package/dist/signals/index.js +3 -1
- package/dist/signals/metaSignals.d.ts +4 -0
- package/dist/signals/metaSignals.js +42 -0
- package/dist/signals/signalGate.js +1 -1
- package/dist/signals/taskDomain.d.ts +22 -0
- package/dist/signals/taskDomain.js +43 -0
- package/dist/strategy/constraintAblation.d.ts +64 -0
- package/dist/strategy/constraintAblation.js +2820 -0
- package/dist/strategy/constraintAblationPredicates.d.ts +31 -0
- package/dist/strategy/constraintAblationPredicates.js +339 -0
- package/dist/strategy/index.d.ts +2 -1
- package/dist/strategy/index.js +2 -1
- package/dist/trace/index.d.ts +3 -1
- package/dist/trace/index.js +3 -1
- package/dist/trace/learningTrace.d.ts +209 -0
- package/dist/trace/learningTrace.js +276 -0
- package/dist/trace/proxyTurns.d.ts +31 -0
- package/dist/trace/proxyTurns.js +137 -0
- package/dist/verify/sandboxRunner.d.ts +28 -0
- package/dist/verify/sandboxRunner.js +218 -19
- package/dist/verify/sandboxedValidation.d.ts +9 -0
- package/dist/verify/sandboxedValidation.js +113 -13
- package/dist/verify/validation.d.ts +11 -1
- package/dist/verify/validation.js +31 -0
- package/package.json +5 -2
|
@@ -0,0 +1,2820 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { classifyProvidedTargetPrefix } from './constraintAblationPredicates.js';
|
|
3
|
+
import { redactString, scanForLeaks } from '../hub/sanitize.js';
|
|
4
|
+
const KEY_VALUE_SECRET_RE = /\b(?:api[_-]?key|token|secret)\s*[:=]\s*(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;]+)/gi;
|
|
5
|
+
const OPENAI_SECRET_RE = /\bsk-[A-Za-z0-9_-]+\b/gi;
|
|
6
|
+
const EMAIL_RE = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;
|
|
7
|
+
const PHONE_RE = /\b1[3-9]\d{9}\b/g;
|
|
8
|
+
const IMPORTANT_TERMS = new Set([
|
|
9
|
+
'ai', 'ask', 'ci', 'log', 'run', 'test', 'tests', 'live', 'selection', 'secret', 'secrets', 'token', 'tokens',
|
|
10
|
+
]);
|
|
11
|
+
function sha256(text) {
|
|
12
|
+
return `sha256:${createHash('sha256').update(text).digest('hex')}`;
|
|
13
|
+
}
|
|
14
|
+
function normalize(text) {
|
|
15
|
+
return text
|
|
16
|
+
.toLowerCase()
|
|
17
|
+
.replace(/[\u201C\u201D]/g, '"')
|
|
18
|
+
.replace(/[\u2018\u2019]/g, "'")
|
|
19
|
+
.replace(/[`*_#>()[\].,;:!?]/g, ' ')
|
|
20
|
+
.replace(/\s+/g, ' ')
|
|
21
|
+
.trim();
|
|
22
|
+
}
|
|
23
|
+
export function redactConstraintText(text) {
|
|
24
|
+
return redactString(text
|
|
25
|
+
.replace(OPENAI_SECRET_RE, '[REDACTED_SECRET]')
|
|
26
|
+
.replace(KEY_VALUE_SECRET_RE, '[REDACTED_SECRET]')
|
|
27
|
+
.replace(EMAIL_RE, '[REDACTED_EMAIL]')
|
|
28
|
+
.replace(PHONE_RE, '[REDACTED_PHONE]'));
|
|
29
|
+
}
|
|
30
|
+
const CLAUSE_MARKER_RE = /\b(?:(?:(?:must|should|shall|do)\s+not|(?:mustn|shouldn|shalln|shan|don)(?:['\u2018\u2019])t)\s+only|(?:require(?:d|s)?|need(?:s)?)\s+to\s+not\s+only|must\s+(?:not(?!\s+only\b)|never)|must-not|should\s+(?:not(?!\s+only\b)|never)|shall\s+(?:not(?!\s+only\b)|never)|mustn(?:['\u2018\u2019])t(?!\s+only\b)|shouldn(?:['\u2018\u2019])t(?!\s+only\b)|shalln(?:['\u2018\u2019])t(?!\s+only\b)|shan(?:['\u2018\u2019])t(?!\s+only\b)|do\s+not(?!\s+only\b)|don(?:['\u2018\u2019])t(?!\s+only\b)|(?:require(?:d|s)?|need(?:s)?)\s+to\s+not(?!\s+only\b)|never|must|should|shall|require(?:d|s)?|need(?:s)?\s+to)\b/gi;
|
|
31
|
+
const NEGATIVE_CONSTRAINT_MARKER_RE = /^(?:must\s+(?:not(?!\s+only\b)|never)|must-not|should\s+(?:not(?!\s+only\b)|never)|shall\s+(?:not(?!\s+only\b)|never)|mustn(?:['\u2018\u2019])t(?!\s+only\b)|shouldn(?:['\u2018\u2019])t(?!\s+only\b)|shalln(?:['\u2018\u2019])t(?!\s+only\b)|shan(?:['\u2018\u2019])t(?!\s+only\b)|do\s+not(?!\s+only\b)|don(?:['\u2018\u2019])t(?!\s+only\b)|(?:require(?:d|s)?|need(?:s)?)\s+to\s+not(?!\s+only\b)|never)$/i;
|
|
32
|
+
const REDACTION_MARKER_RE = /\[REDACTED(?:_[A-Z]+)?\]/gi;
|
|
33
|
+
const EMAIL_VALUE_RE = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i;
|
|
34
|
+
const NEGATED_ACTION_RE = /(?:^not\s+(?!only\b)|\b(?:(?:am|are|did|do|does|had|has|have|is|need|needs|ought|was|were)\s+not(?!\s+only\b)|(?:aren't|couldn't|didn't|doesn't|don't|hadn't|hasn't|haven't|isn't|mightn't|needn't|oughtn't|wasn't|weren't|won't|wouldn't)(?!\s+only\b)|never|must\s+not(?!\s+only\b)|mustn't(?!\s+only\b)|should\s+not(?!\s+only\b)|shouldn't(?!\s+only\b)|shall\s+not(?!\s+only\b)|shalln't(?!\s+only\b)|shan't(?!\s+only\b)|cannot(?!\s+only\b)|can't(?!\s+only\b)|can\s+not(?!\s+only\b)|(?:will|would|can|could|may|might)\s+not(?!\s+only\b))\b)/i;
|
|
35
|
+
const WITHOUT_ACTION_RE = /\bwithout\b/i;
|
|
36
|
+
const POTENTIAL_CONDITIONAL_SCOPE_RE = /(?:^|,)\s*(?:(?:eventually|finally|initially|later|subsequently)\s+)*(?:assuming|if|once|provided|supposing|unless|when|while)\b|\b(?:and|or|then)\s+(?:(?:eventually|finally|initially|later|subsequently)\s+)*(?:assuming|if|once|provided|supposing|unless|when|while)\b|\bprovided\s+that\b/i;
|
|
37
|
+
const CLOSED_IF_ANYTHING_RE = /(^|,)\s*if\s+anything\s*,/gi;
|
|
38
|
+
const COORDINATED_PROVIDED_RE = /\b(and|or|then)\s+(?:(?:eventually|finally|initially|later|subsequently)\s+)*provided\b/gi;
|
|
39
|
+
const LEADING_PROVIDED_RE = /(^|,)\s*(?:(?:eventually|finally|initially|later|subsequently)\s+)*provided\b/gi;
|
|
40
|
+
const PROVIDED_BY_ADJUNCT_RE = /\bprovided\s+by\b/gi;
|
|
41
|
+
const LEADING_DISCOURSE_TEMPORAL_RE = /^\s*(?:eventually|finally|initially|later|subsequently)\s*,\s*(?:once|when|while)\b[^,;.!?\r\n]*,\s*/i;
|
|
42
|
+
const COORDINATED_TEMPORAL_RE = /\b(and|or|then)\s+(?:(?:eventually|finally|initially|later|subsequently)\s+)*(?:once|when|while)\b/gi;
|
|
43
|
+
const NEGATIVE_SUBJECT_MARKER_RE = /\b(?:neither|no|nobody|none|nothing|zero)\b/i;
|
|
44
|
+
const COMMA_DELIMITED_WITHOUT_RE = /\bwithout\b[^,;.!?\r\n]*,/gi;
|
|
45
|
+
const COMMA_DELIMITED_TEMPORAL_RE = /,\s*(?:once|when|while)\b[^,;.!?\r\n]*,/gi;
|
|
46
|
+
const AFFIRMATIVE_ACTION_LEMMAS = [
|
|
47
|
+
'access', 'allow', 'call', 'display', 'emit', 'enable', 'expose', 'export', 'feed', 'include', 'leak',
|
|
48
|
+
'log', 'output', 'print', 'provide', 'publish', 'reveal', 'run', 'send', 'share', 'show', 'store',
|
|
49
|
+
'transmit', 'upload', 'use', 'write',
|
|
50
|
+
];
|
|
51
|
+
const INDEPENDENT_AFFIRMATIVE_AUXILIARIES = new Set([
|
|
52
|
+
'am', 'are', 'be', 'been', 'being', 'can', 'could', 'did', 'do', 'does', 'had', 'has', 'have', 'is',
|
|
53
|
+
'may', 'might', 'must', 'shall', 'should', 'was', 'were', 'will', 'would',
|
|
54
|
+
]);
|
|
55
|
+
const CREDENTIAL_LEAK_TYPES = new Set([
|
|
56
|
+
'api_key',
|
|
57
|
+
'azure_client_secret',
|
|
58
|
+
'azure_instrumentation_key',
|
|
59
|
+
'azure_key',
|
|
60
|
+
'basic_auth',
|
|
61
|
+
'bearer_token',
|
|
62
|
+
'db_url',
|
|
63
|
+
'discord_token',
|
|
64
|
+
'env_value_leak',
|
|
65
|
+
'github_token',
|
|
66
|
+
'jwt',
|
|
67
|
+
'npm_token',
|
|
68
|
+
'password',
|
|
69
|
+
'private_key',
|
|
70
|
+
'proxy_token',
|
|
71
|
+
'secret',
|
|
72
|
+
'slack_token',
|
|
73
|
+
]);
|
|
74
|
+
function splitConstraintCandidates(text) {
|
|
75
|
+
const spans = [];
|
|
76
|
+
const boundary = /\r?\n|(?<=[.!?])\s+/g;
|
|
77
|
+
let start = 0;
|
|
78
|
+
const append = (end) => {
|
|
79
|
+
const raw = text.slice(start, end);
|
|
80
|
+
const leadingWhitespace = raw.match(/^\s*/)?.[0].length ?? 0;
|
|
81
|
+
let textStart = start + leadingWhitespace;
|
|
82
|
+
const listMarker = text.slice(textStart, end).match(/^(?:[-*]\s+|\d+[.)]\s+|\[[ xX]\]\s*)/);
|
|
83
|
+
if (listMarker)
|
|
84
|
+
textStart += listMarker[0].length;
|
|
85
|
+
const trailingWhitespace = text.slice(textStart, end).match(/\s*$/)?.[0].length ?? 0;
|
|
86
|
+
const textEnd = end - trailingWhitespace;
|
|
87
|
+
const candidate = text.slice(textStart, textEnd);
|
|
88
|
+
if (candidate)
|
|
89
|
+
spans.push({ text: candidate, start, end, textStart });
|
|
90
|
+
};
|
|
91
|
+
for (const match of text.matchAll(boundary)) {
|
|
92
|
+
const boundaryStart = match.index;
|
|
93
|
+
if (!match[0].includes('\n')) {
|
|
94
|
+
const punctuationIndex = boundaryStart - 1;
|
|
95
|
+
const candidatePrefix = text.slice(start, punctuationIndex + 1);
|
|
96
|
+
if (/^\s*\d+[.)]$/.test(candidatePrefix))
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
append(boundaryStart);
|
|
100
|
+
start = boundaryStart + match[0].length;
|
|
101
|
+
}
|
|
102
|
+
append(text.length);
|
|
103
|
+
return spans;
|
|
104
|
+
}
|
|
105
|
+
function kindForMarker(marker) {
|
|
106
|
+
return NEGATIVE_CONSTRAINT_MARKER_RE.test(marker) ? 'must_not' : 'must';
|
|
107
|
+
}
|
|
108
|
+
function splitConstraintClauses(text) {
|
|
109
|
+
const matches = [...text.matchAll(CLAUSE_MARKER_RE)];
|
|
110
|
+
return matches.flatMap((match, index) => {
|
|
111
|
+
const start = match.index;
|
|
112
|
+
const nextStart = matches[index + 1]?.index ?? text.length;
|
|
113
|
+
const segment = text.slice(start, nextStart);
|
|
114
|
+
const connector = segment.match(/\b(?:and|but|however|although|though)\s*$/i);
|
|
115
|
+
const contentEnd = start + (connector?.index ?? segment.length);
|
|
116
|
+
const trailingWhitespace = text.slice(start, contentEnd).match(/\s*$/)?.[0].length ?? 0;
|
|
117
|
+
const end = contentEnd - trailingWhitespace;
|
|
118
|
+
if (end <= start)
|
|
119
|
+
return [];
|
|
120
|
+
return [{ kind: kindForMarker(match[0]), text: text.slice(start, end), start, end }];
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
function safeSource(source) {
|
|
124
|
+
return source === 'plan' || source === 'task' || source === 'trace' ? source : 'trace';
|
|
125
|
+
}
|
|
126
|
+
function safeTraceId(traceId) {
|
|
127
|
+
if (typeof traceId !== 'string' || traceId.length === 0)
|
|
128
|
+
return undefined;
|
|
129
|
+
return `trace:${sha256(traceId).slice('sha256:'.length, 'sha256:'.length + 16)}`;
|
|
130
|
+
}
|
|
131
|
+
function sensitiveClassesForConstraint(text) {
|
|
132
|
+
const normalized = normalize(text);
|
|
133
|
+
const classes = new Set();
|
|
134
|
+
if (/\b(?:secrets?|tokens?|api[_ -]?keys?|credentials?|passwords?|bearer|private[_ -]?keys?)\b/.test(normalized)) {
|
|
135
|
+
classes.add('credential');
|
|
136
|
+
}
|
|
137
|
+
if (/\b(?:email|e-mail|mail)\b/.test(normalized))
|
|
138
|
+
classes.add('email');
|
|
139
|
+
if (/\b(?:paths?|directories?|filesystem|home\s+directory|user\s+profile)\b/.test(normalized)) {
|
|
140
|
+
classes.add('filesystem_path');
|
|
141
|
+
}
|
|
142
|
+
return [...classes];
|
|
143
|
+
}
|
|
144
|
+
const TARGET_STOPWORDS = new Set([
|
|
145
|
+
'a', 'also', 'although', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'for', 'from', 'however',
|
|
146
|
+
'in', 'into', 'is', 'of', 'on', 'only', 'or', 'redacted', 'that', 'the', 'this', 'though', 'to', 'until',
|
|
147
|
+
'when', 'with', 'without',
|
|
148
|
+
]);
|
|
149
|
+
const GENERIC_ACTION_TERMS = new Set(['add', 'call', 'include', 'print', 'use']);
|
|
150
|
+
const AMBIGUOUS_DEFERRED_VERB_TERMS = new Set(['live']);
|
|
151
|
+
function targetTerms(text, kind) {
|
|
152
|
+
const stripped = normalize(text
|
|
153
|
+
.replace(REDACTION_MARKER_RE, ' ')
|
|
154
|
+
.replace(CLAUSE_MARKER_RE, ' '))
|
|
155
|
+
.replace(/\b(?:must\s+not|must-not|mustn't|do\s+not|don't|never|must|required|requires|require|needs\s+to|need\s+to|should)\b/g, ' ')
|
|
156
|
+
.replace(/\s+/g, ' ')
|
|
157
|
+
.trim();
|
|
158
|
+
const seen = new Set();
|
|
159
|
+
const terms = [];
|
|
160
|
+
let expectVerb = true;
|
|
161
|
+
let skippedExpectedVerb = false;
|
|
162
|
+
const contrastive = isContrastiveConstraint(text);
|
|
163
|
+
let group = 0;
|
|
164
|
+
const groupCounts = new Map();
|
|
165
|
+
const groupLimit = kind === 'must_not' ? 6 : 4;
|
|
166
|
+
for (const term of stripped.split(' ')) {
|
|
167
|
+
if (!term)
|
|
168
|
+
continue;
|
|
169
|
+
if (TARGET_STOPWORDS.has(term)) {
|
|
170
|
+
if (contrastive && term === 'also') {
|
|
171
|
+
group += 1;
|
|
172
|
+
expectVerb = true;
|
|
173
|
+
skippedExpectedVerb = false;
|
|
174
|
+
}
|
|
175
|
+
else if (term === 'and' || term === 'or') {
|
|
176
|
+
expectVerb = true;
|
|
177
|
+
skippedExpectedVerb = false;
|
|
178
|
+
}
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
const verb = expectVerb;
|
|
182
|
+
const key = `${group}:${term}`;
|
|
183
|
+
if (GENERIC_ACTION_TERMS.has(term)
|
|
184
|
+
|| (term.length < 4 && !IMPORTANT_TERMS.has(term))
|
|
185
|
+
|| seen.has(key)
|
|
186
|
+
|| (groupCounts.get(group) ?? 0) >= groupLimit) {
|
|
187
|
+
if (expectVerb)
|
|
188
|
+
skippedExpectedVerb = true;
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
expectVerb = false;
|
|
192
|
+
seen.add(key);
|
|
193
|
+
groupCounts.set(group, (groupCounts.get(group) ?? 0) + 1);
|
|
194
|
+
terms.push({ text: term, verb, verbAfterSkippedTerm: verb && skippedExpectedVerb, group });
|
|
195
|
+
skippedExpectedVerb = false;
|
|
196
|
+
}
|
|
197
|
+
return terms;
|
|
198
|
+
}
|
|
199
|
+
const IRREGULAR_VERB_FORMS = new Map([
|
|
200
|
+
['be', ['be', 'am', 'is', 'are', 'was', 'were', 'been', 'being']],
|
|
201
|
+
['begin', ['begin', 'begins', 'began', 'begun', 'beginning']],
|
|
202
|
+
['bleed', ['bleed', 'bleeds', 'bled', 'bleeding']],
|
|
203
|
+
['break', ['break', 'breaks', 'broke', 'broken', 'breaking']],
|
|
204
|
+
['breed', ['breed', 'breeds', 'bred', 'breeding']],
|
|
205
|
+
['bring', ['bring', 'brings', 'brought', 'bringing']],
|
|
206
|
+
['build', ['build', 'builds', 'built', 'building']],
|
|
207
|
+
['buy', ['buy', 'buys', 'bought', 'buying']],
|
|
208
|
+
['catch', ['catch', 'catches', 'caught', 'catching']],
|
|
209
|
+
['choose', ['choose', 'chooses', 'chose', 'chosen', 'choosing']],
|
|
210
|
+
['do', ['do', 'does', 'did', 'done', 'doing']],
|
|
211
|
+
['feed', ['feed', 'feeds', 'fed', 'feeding']],
|
|
212
|
+
['find', ['find', 'finds', 'found', 'finding']],
|
|
213
|
+
['get', ['get', 'gets', 'got', 'gotten', 'getting']],
|
|
214
|
+
['give', ['give', 'gives', 'gave', 'given', 'giving']],
|
|
215
|
+
['go', ['go', 'goes', 'went', 'gone', 'going']],
|
|
216
|
+
['keep', ['keep', 'keeps', 'kept', 'keeping']],
|
|
217
|
+
['leave', ['leave', 'leaves', 'left', 'leaving']],
|
|
218
|
+
['make', ['make', 'makes', 'made', 'making']],
|
|
219
|
+
['read', ['read', 'reads', 'reading']],
|
|
220
|
+
['ring', ['ring', 'rings', 'rang', 'rung', 'ringing']],
|
|
221
|
+
['run', ['run', 'runs', 'ran', 'running']],
|
|
222
|
+
['send', ['send', 'sends', 'sent', 'sending']],
|
|
223
|
+
['show', ['show', 'shows', 'showed', 'shown', 'showing']],
|
|
224
|
+
['sing', ['sing', 'sings', 'sang', 'sung', 'singing']],
|
|
225
|
+
['speed', ['speed', 'speeds', 'sped', 'speeded', 'speeding']],
|
|
226
|
+
['take', ['take', 'takes', 'took', 'taken', 'taking']],
|
|
227
|
+
['teach', ['teach', 'teaches', 'taught', 'teaching']],
|
|
228
|
+
['tell', ['tell', 'tells', 'told', 'telling']],
|
|
229
|
+
['think', ['think', 'thinks', 'thought', 'thinking']],
|
|
230
|
+
['write', ['write', 'writes', 'wrote', 'written', 'writing']],
|
|
231
|
+
]);
|
|
232
|
+
const IRREGULAR_SIMPLE_PAST_FORMS = new Set(['read', 'was', 'were', ...[...IRREGULAR_VERB_FORMS.entries()]
|
|
233
|
+
.filter(([lemma]) => lemma !== 'be' && lemma !== 'read')
|
|
234
|
+
.map(([, forms]) => forms[2])]
|
|
235
|
+
.filter((form) => form !== undefined && !form.endsWith('ing')));
|
|
236
|
+
const IRREGULAR_PAST_PARTICIPLE_FORMS = new Set([
|
|
237
|
+
'been', 'read', 'showed', 'sped',
|
|
238
|
+
...[...IRREGULAR_VERB_FORMS.entries()]
|
|
239
|
+
.filter(([lemma]) => lemma !== 'be' && lemma !== 'read')
|
|
240
|
+
.map(([, forms]) => forms.at(-2))
|
|
241
|
+
.filter((form) => form !== undefined && !form.endsWith('ing')),
|
|
242
|
+
]);
|
|
243
|
+
const IRREGULAR_PROGRESSIVE_VERB_LEMMAS = new Map([...IRREGULAR_VERB_FORMS].flatMap(([lemma, forms]) => forms.filter((form) => form.endsWith('ing')).map((form) => [form, lemma])));
|
|
244
|
+
const IRREGULAR_NOUN_PLURALS = new Map([
|
|
245
|
+
['analysis', 'analyses'],
|
|
246
|
+
['basis', 'bases'],
|
|
247
|
+
['bus', 'buses'],
|
|
248
|
+
['crisis', 'crises'],
|
|
249
|
+
['leaf', 'leaves'],
|
|
250
|
+
['life', 'lives'],
|
|
251
|
+
['status', 'statuses'],
|
|
252
|
+
]);
|
|
253
|
+
const UNAMBIGUOUS_PLURAL_SINGULARS = new Map([
|
|
254
|
+
['analyses', 'analysis'],
|
|
255
|
+
['buses', 'bus'],
|
|
256
|
+
['crises', 'crisis'],
|
|
257
|
+
['statuses', 'status'],
|
|
258
|
+
]);
|
|
259
|
+
const DOUBLED_INFLECTION_BASES = new Set([
|
|
260
|
+
'admit', 'commit', 'control', 'debug', 'defer', 'embed', 'format', 'occur', 'permit', 'prefer', 'refer', 'submit',
|
|
261
|
+
'shred', 'transmit',
|
|
262
|
+
]);
|
|
263
|
+
const NON_DOUBLED_SHORT_CVC_BASES = new Set(['edit', 'open']);
|
|
264
|
+
const ED_SUFFIX_BASE_VERBS = new Set(['exceed', 'heed', 'need', 'proceed', 'seed', 'succeed']);
|
|
265
|
+
const EXACT_ONLY_PROGRESSIVE_LEMMAS = new Set(['be', 'do', 'go']);
|
|
266
|
+
const SHORT_INFLECTED_VERB_LEMMAS = new Map([
|
|
267
|
+
['died', 'die'],
|
|
268
|
+
['dying', 'die'],
|
|
269
|
+
['lied', 'lie'],
|
|
270
|
+
['lying', 'lie'],
|
|
271
|
+
['skied', 'ski'],
|
|
272
|
+
['skiing', 'ski'],
|
|
273
|
+
['taxied', 'taxi'],
|
|
274
|
+
['tied', 'tie'],
|
|
275
|
+
['tying', 'tie'],
|
|
276
|
+
['vied', 'vie'],
|
|
277
|
+
['vying', 'vie'],
|
|
278
|
+
]);
|
|
279
|
+
const SILENT_E_VERB_LEMMAS = new Map([
|
|
280
|
+
['ac', 'ace'],
|
|
281
|
+
['ag', 'age'],
|
|
282
|
+
['ap', 'ape'],
|
|
283
|
+
['creat', 'create'],
|
|
284
|
+
['delet', 'delete'],
|
|
285
|
+
['enabl', 'enable'],
|
|
286
|
+
['expos', 'expose'],
|
|
287
|
+
['includ', 'include'],
|
|
288
|
+
['ic', 'ice'],
|
|
289
|
+
['leav', 'leave'],
|
|
290
|
+
['liv', 'live'],
|
|
291
|
+
['mak', 'make'],
|
|
292
|
+
['mov', 'move'],
|
|
293
|
+
['ow', 'owe'],
|
|
294
|
+
['preserv', 'preserve'],
|
|
295
|
+
['provid', 'provide'],
|
|
296
|
+
['requir', 'require'],
|
|
297
|
+
['sav', 'save'],
|
|
298
|
+
['shar', 'share'],
|
|
299
|
+
['stor', 'store'],
|
|
300
|
+
['su', 'sue'],
|
|
301
|
+
['tak', 'take'],
|
|
302
|
+
['us', 'use'],
|
|
303
|
+
['validat', 'validate'],
|
|
304
|
+
['writ', 'write'],
|
|
305
|
+
]);
|
|
306
|
+
function addPluralForm(base, forms) {
|
|
307
|
+
const irregular = IRREGULAR_NOUN_PLURALS.get(base);
|
|
308
|
+
if (irregular)
|
|
309
|
+
forms.add(irregular);
|
|
310
|
+
else if (/[^aeiou]y$/u.test(base))
|
|
311
|
+
forms.add(`${base.slice(0, -1)}ies`);
|
|
312
|
+
else if (/(?:s|x|z|ch|sh)$/u.test(base))
|
|
313
|
+
forms.add(`${base}es`);
|
|
314
|
+
else
|
|
315
|
+
forms.add(`${base}s`);
|
|
316
|
+
}
|
|
317
|
+
function addSingularCandidates(term, bases) {
|
|
318
|
+
const irregular = UNAMBIGUOUS_PLURAL_SINGULARS.get(term);
|
|
319
|
+
if (irregular) {
|
|
320
|
+
bases.add(irregular);
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
if (/[^aeiou]ies$/u.test(term) && term.length > 3)
|
|
324
|
+
bases.add(`${term.slice(0, -3)}y`);
|
|
325
|
+
if (term.endsWith('ves') && term.length > 4) {
|
|
326
|
+
const stem = term.slice(0, -3);
|
|
327
|
+
bases.add(`${stem}f`);
|
|
328
|
+
bases.add(`${stem}fe`);
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
if (/(?:sses|xes|zzes|ches|shes)$/u.test(term) && term.length > 4)
|
|
332
|
+
bases.add(term.slice(0, -2));
|
|
333
|
+
else if (term.endsWith('s') && term.length > 3 && !/(?:ss|us|is)$/u.test(term))
|
|
334
|
+
bases.add(term.slice(0, -1));
|
|
335
|
+
}
|
|
336
|
+
function doublesFinalConsonant(base) {
|
|
337
|
+
if (DOUBLED_INFLECTION_BASES.has(base))
|
|
338
|
+
return true;
|
|
339
|
+
if (base.length > 4 || NON_DOUBLED_SHORT_CVC_BASES.has(base))
|
|
340
|
+
return false;
|
|
341
|
+
return /[bcdfghjklmnpqrstvwxyz][aeiou][bcdfghjklmnpqrstvz]$/u.test(base);
|
|
342
|
+
}
|
|
343
|
+
function addRegularVerbForms(base, forms) {
|
|
344
|
+
forms.add(base);
|
|
345
|
+
addPluralForm(base, forms);
|
|
346
|
+
if (base.endsWith('ie')) {
|
|
347
|
+
forms.add(`${base}d`);
|
|
348
|
+
forms.add(`${base.slice(0, -2)}ying`);
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
if (base.endsWith('e')) {
|
|
352
|
+
forms.add(`${base}d`);
|
|
353
|
+
forms.add(`${base.slice(0, -1)}ing`);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
if (/[^aeiou]y$/u.test(base))
|
|
357
|
+
forms.add(`${base.slice(0, -1)}ied`);
|
|
358
|
+
if (doublesFinalConsonant(base)) {
|
|
359
|
+
const last = base.at(-1);
|
|
360
|
+
forms.add(`${base}${last}ed`);
|
|
361
|
+
forms.add(`${base}${last}ing`);
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
if (!/[^aeiou]y$/u.test(base))
|
|
365
|
+
forms.add(`${base}ed`);
|
|
366
|
+
forms.add(`${base}ing`);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
function nominalForms(term) {
|
|
370
|
+
const forms = new Set([term]);
|
|
371
|
+
const bases = new Set();
|
|
372
|
+
addSingularCandidates(term, bases);
|
|
373
|
+
if (bases.size === 0)
|
|
374
|
+
addPluralForm(term, forms);
|
|
375
|
+
for (const base of bases) {
|
|
376
|
+
forms.add(base);
|
|
377
|
+
addPluralForm(base, forms);
|
|
378
|
+
}
|
|
379
|
+
return forms;
|
|
380
|
+
}
|
|
381
|
+
function verbForms(term, kind) {
|
|
382
|
+
const forms = new Set([term]);
|
|
383
|
+
const lemma = verbLemma(term);
|
|
384
|
+
if (!lemma)
|
|
385
|
+
return forms;
|
|
386
|
+
if (kind === 'must_not'
|
|
387
|
+
&& term.endsWith('ing')
|
|
388
|
+
&& EXACT_ONLY_PROGRESSIVE_LEMMAS.has(lemma))
|
|
389
|
+
return forms;
|
|
390
|
+
const irregular = IRREGULAR_VERB_FORMS.get(lemma);
|
|
391
|
+
if (irregular) {
|
|
392
|
+
for (const form of irregular)
|
|
393
|
+
forms.add(form);
|
|
394
|
+
}
|
|
395
|
+
else {
|
|
396
|
+
addRegularVerbForms(lemma, forms);
|
|
397
|
+
}
|
|
398
|
+
return forms;
|
|
399
|
+
}
|
|
400
|
+
function targetTermForms(term, kind) {
|
|
401
|
+
const nominal = nominalForms(term.text);
|
|
402
|
+
if (!term.verb || (term.verbAfterSkippedTerm && AMBIGUOUS_DEFERRED_VERB_TERMS.has(term.text))) {
|
|
403
|
+
return nominal;
|
|
404
|
+
}
|
|
405
|
+
const forms = verbForms(term.text, kind);
|
|
406
|
+
if (term.verbAfterSkippedTerm) {
|
|
407
|
+
for (const form of nominal)
|
|
408
|
+
forms.add(form);
|
|
409
|
+
}
|
|
410
|
+
return forms;
|
|
411
|
+
}
|
|
412
|
+
const AFFIRMATIVE_ACTION_FORMS = new Set([
|
|
413
|
+
...AFFIRMATIVE_ACTION_LEMMAS.flatMap((lemma) => [...verbForms(lemma, 'must_not')]),
|
|
414
|
+
'outputted',
|
|
415
|
+
'outputting',
|
|
416
|
+
]);
|
|
417
|
+
function verbLemma(term) {
|
|
418
|
+
if (IRREGULAR_VERB_FORMS.has(term)
|
|
419
|
+
|| DOUBLED_INFLECTION_BASES.has(term)
|
|
420
|
+
|| ED_SUFFIX_BASE_VERBS.has(term))
|
|
421
|
+
return term;
|
|
422
|
+
const shortInflectedLemma = SHORT_INFLECTED_VERB_LEMMAS.get(term);
|
|
423
|
+
if (shortInflectedLemma)
|
|
424
|
+
return shortInflectedLemma;
|
|
425
|
+
if (term.endsWith('ied') && term.length > 4) {
|
|
426
|
+
return `${term.slice(0, -3)}y`;
|
|
427
|
+
}
|
|
428
|
+
const suffix = term.endsWith('ing') ? 'ing' : term.endsWith('ed') ? 'ed' : undefined;
|
|
429
|
+
if (!suffix)
|
|
430
|
+
return term;
|
|
431
|
+
const irregularProgressiveLemma = IRREGULAR_PROGRESSIVE_VERB_LEMMAS.get(term);
|
|
432
|
+
if (irregularProgressiveLemma)
|
|
433
|
+
return irregularProgressiveLemma;
|
|
434
|
+
const stem = term.slice(0, -suffix.length);
|
|
435
|
+
const silentELemma = SILENT_E_VERB_LEMMAS.get(stem);
|
|
436
|
+
if (silentELemma)
|
|
437
|
+
return silentELemma;
|
|
438
|
+
if (stem.length <= 2)
|
|
439
|
+
return term;
|
|
440
|
+
const last = stem.at(-1);
|
|
441
|
+
if (last && last === stem.at(-2)) {
|
|
442
|
+
const undoubled = stem.slice(0, -1);
|
|
443
|
+
if (doublesFinalConsonant(undoubled))
|
|
444
|
+
return undoubled;
|
|
445
|
+
}
|
|
446
|
+
return stem;
|
|
447
|
+
}
|
|
448
|
+
function termMatchIndexes(output, term, kind) {
|
|
449
|
+
const outputTerms = output.match(/[a-z0-9_]+/gu) ?? [];
|
|
450
|
+
const forms = targetTermForms(term, kind);
|
|
451
|
+
return outputTerms.flatMap((outputTerm, index) => forms.has(outputTerm) ? [index] : []);
|
|
452
|
+
}
|
|
453
|
+
function termMatches(output, term, kind) {
|
|
454
|
+
return termMatchIndexes(output, term, kind).length > 0;
|
|
455
|
+
}
|
|
456
|
+
function isViolated(kind, terms, matchedTerms, contrastive) {
|
|
457
|
+
if (kind === 'must' && contrastive) {
|
|
458
|
+
const matched = new Set(matchedTerms);
|
|
459
|
+
const groups = new Set(terms.map((term) => term.group));
|
|
460
|
+
return [...groups].some((group) => {
|
|
461
|
+
const groupTerms = terms.filter((term) => term.group === group);
|
|
462
|
+
return groupTerms.filter((term) => matched.has(term.text)).length < Math.min(2, groupTerms.length);
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
if (kind === 'must')
|
|
466
|
+
return matchedTerms.length < Math.min(2, terms.length);
|
|
467
|
+
return matchedTerms.length >= Math.min(2, terms.length);
|
|
468
|
+
}
|
|
469
|
+
function isContrastiveConstraint(text) {
|
|
470
|
+
return /(?:\bnot|n't)\s+only\b[\s\S]*\bbut\s+also\b/i.test(normalize(text));
|
|
471
|
+
}
|
|
472
|
+
function severityFor(kind) {
|
|
473
|
+
return kind === 'must_not' ? 'high' : 'medium';
|
|
474
|
+
}
|
|
475
|
+
export function extractConstraints(traces) {
|
|
476
|
+
const seen = new Set();
|
|
477
|
+
const out = [];
|
|
478
|
+
for (const trace of traces) {
|
|
479
|
+
for (const candidate of splitConstraintCandidates(trace.text)) {
|
|
480
|
+
for (const clause of splitConstraintClauses(candidate.text)) {
|
|
481
|
+
const redactedText = redactConstraintText(clause.text);
|
|
482
|
+
const textHash = sha256(normalize(redactedText));
|
|
483
|
+
const source = safeSource(trace.source);
|
|
484
|
+
const traceId = trace.traceId ? safeTraceId(trace.traceId) : undefined;
|
|
485
|
+
const key = `${clause.kind}:${textHash}`;
|
|
486
|
+
if (seen.has(key))
|
|
487
|
+
continue;
|
|
488
|
+
seen.add(key);
|
|
489
|
+
const id = `constraint:${createHash('sha256').update(key).digest('hex').slice(0, 16)}`;
|
|
490
|
+
out.push({
|
|
491
|
+
id,
|
|
492
|
+
kind: clause.kind,
|
|
493
|
+
textHash,
|
|
494
|
+
redactedText,
|
|
495
|
+
source,
|
|
496
|
+
...(traceId ? { traceId } : {}),
|
|
497
|
+
sensitiveClasses: sensitiveClassesForConstraint(clause.text),
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
return out;
|
|
503
|
+
}
|
|
504
|
+
export function buildConstraintAblatedPrompts(prompt, constraints, opts = {}) {
|
|
505
|
+
const candidates = splitConstraintCandidates(prompt);
|
|
506
|
+
const originalPromptHash = sha256(prompt);
|
|
507
|
+
return constraints.flatMap((constraint) => {
|
|
508
|
+
const matches = candidates.flatMap((candidate) => {
|
|
509
|
+
return splitConstraintClauses(candidate.text).flatMap((clause) => {
|
|
510
|
+
if (sha256(normalize(redactConstraintText(clause.text))) !== constraint.textHash)
|
|
511
|
+
return [];
|
|
512
|
+
let start = candidate.textStart + clause.start;
|
|
513
|
+
let end = start + clause.text.length;
|
|
514
|
+
const before = prompt.slice(candidate.start, start);
|
|
515
|
+
const trailingConnector = before.match(/(?:,\s*|\s+)(?:and|but|however|although|though)\s*$/i);
|
|
516
|
+
if (trailingConnector)
|
|
517
|
+
start -= trailingConnector[0].length;
|
|
518
|
+
else if (/^\s*(?:[-*]\s+|\d+[.)]\s+|\[[ xX]\]\s*)$/.test(before))
|
|
519
|
+
start = candidate.start;
|
|
520
|
+
const after = prompt.slice(end, candidate.end);
|
|
521
|
+
const leadingConnector = after.match(/^\s+but\s+also\s+/i)
|
|
522
|
+
?? after.match(/^\s+(?:and|but|however|although|though)\s+/i);
|
|
523
|
+
if (leadingConnector && !trailingConnector)
|
|
524
|
+
end += leadingConnector[0].length;
|
|
525
|
+
return [{ start, end }];
|
|
526
|
+
});
|
|
527
|
+
});
|
|
528
|
+
if (matches.length !== 1)
|
|
529
|
+
return [];
|
|
530
|
+
const match = matches[0];
|
|
531
|
+
if (!match)
|
|
532
|
+
return [];
|
|
533
|
+
const ablatedPrompt = prompt.slice(0, match.start) + prompt.slice(match.end);
|
|
534
|
+
if (ablatedPrompt === prompt)
|
|
535
|
+
return [];
|
|
536
|
+
return [{
|
|
537
|
+
originalPromptHash,
|
|
538
|
+
ablatedPromptHash: sha256(ablatedPrompt),
|
|
539
|
+
removedConstraintIds: [constraint.id],
|
|
540
|
+
...(opts.includeRedactedPreview ? { redactedPreview: redactConstraintText(ablatedPrompt) } : {}),
|
|
541
|
+
}];
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
function sensitiveClassesForValue(value) {
|
|
545
|
+
const classes = new Set();
|
|
546
|
+
if (EMAIL_VALUE_RE.test(value))
|
|
547
|
+
classes.add('email');
|
|
548
|
+
if (new RegExp(KEY_VALUE_SECRET_RE.source, KEY_VALUE_SECRET_RE.flags).test(value)
|
|
549
|
+
|| new RegExp(OPENAI_SECRET_RE.source, OPENAI_SECRET_RE.flags).test(value)) {
|
|
550
|
+
classes.add('credential');
|
|
551
|
+
}
|
|
552
|
+
for (const leak of scanForLeaks(value).leaks) {
|
|
553
|
+
const type = String(leak.type);
|
|
554
|
+
if (type === 'email')
|
|
555
|
+
classes.add('email');
|
|
556
|
+
else if (type === 'local_path')
|
|
557
|
+
classes.add('filesystem_path');
|
|
558
|
+
else if (CREDENTIAL_LEAK_TYPES.has(type))
|
|
559
|
+
classes.add('credential');
|
|
560
|
+
}
|
|
561
|
+
return [...classes];
|
|
562
|
+
}
|
|
563
|
+
function hasIndependentAffirmativeAction(segment, actionForms, inheritsSubject = false) {
|
|
564
|
+
const stream = lexRequiredEvidence(segment);
|
|
565
|
+
for (let actionIndex = inheritsSubject ? 0 : 1; actionIndex < stream.tokens.length; actionIndex += 1) {
|
|
566
|
+
if (!actionForms.has(stream.tokens[actionIndex]?.value ?? ''))
|
|
567
|
+
continue;
|
|
568
|
+
if (stream.conditional[actionIndex] === true)
|
|
569
|
+
continue;
|
|
570
|
+
if (actionIndex === 0)
|
|
571
|
+
return true;
|
|
572
|
+
let prefixStart = 0;
|
|
573
|
+
while (prefixStart < actionIndex) {
|
|
574
|
+
const term = stream.tokens[prefixStart]?.value ?? '';
|
|
575
|
+
if (!isIgnoredRequiredToken(stream, prefixStart)
|
|
576
|
+
&& !LEADING_CLAUSE_MODIFIER_TERMS.has(term)
|
|
577
|
+
&& !/ly$/u.test(term))
|
|
578
|
+
break;
|
|
579
|
+
prefixStart += 1;
|
|
580
|
+
}
|
|
581
|
+
const first = stream.tokens[prefixStart]?.value ?? '';
|
|
582
|
+
if (inheritsSubject && prefixStart === actionIndex)
|
|
583
|
+
return true;
|
|
584
|
+
if (INDEPENDENT_AFFIRMATIVE_AUXILIARIES.has(first))
|
|
585
|
+
return true;
|
|
586
|
+
if (leadingSubjectPolarity(stream, prefixStart, actionIndex) !== 'positive')
|
|
587
|
+
continue;
|
|
588
|
+
if (SUBJECT_ARTICLE_TERMS.has(first) && prefixStart + 1 >= actionIndex)
|
|
589
|
+
continue;
|
|
590
|
+
return true;
|
|
591
|
+
}
|
|
592
|
+
return false;
|
|
593
|
+
}
|
|
594
|
+
const NEGATED_FINITE_AUXILIARIES = new Set([
|
|
595
|
+
'am', 'are', 'can', 'could', 'did', 'do', 'does', 'had', 'has', 'have', 'is', 'may', 'might', 'must',
|
|
596
|
+
'need', 'needs', 'ought', 'shall', 'should', 'was', 'were', 'will', 'would',
|
|
597
|
+
]);
|
|
598
|
+
const NEGATED_FINITE_CONTRACTIONS = new Set([
|
|
599
|
+
"aren't", "can't", 'cannot', "couldn't", "didn't", "doesn't", "don't", "hadn't", "hasn't", "haven't",
|
|
600
|
+
"isn't", "mightn't", "mustn't", "needn't", "oughtn't", "shan't", "shouldn't", "wasn't", "weren't",
|
|
601
|
+
"won't", "wouldn't",
|
|
602
|
+
]);
|
|
603
|
+
function hasExplicitSubjectBeforeNegation(segment) {
|
|
604
|
+
const stream = lexRequiredEvidence(segment);
|
|
605
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
606
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
607
|
+
const negatedAuxiliary = NEGATED_FINITE_CONTRACTIONS.has(term)
|
|
608
|
+
|| (NEGATED_FINITE_AUXILIARIES.has(term) && stream.tokens[index + 1]?.value === 'not');
|
|
609
|
+
if (!negatedAuxiliary)
|
|
610
|
+
continue;
|
|
611
|
+
return index > 0 && leadingSubjectPolarity(stream, 0, index) === 'positive';
|
|
612
|
+
}
|
|
613
|
+
return false;
|
|
614
|
+
}
|
|
615
|
+
function startsWithSubjectElidedNegation(segment) {
|
|
616
|
+
const stream = lexRequiredEvidence(segment);
|
|
617
|
+
const first = stream.tokens.findIndex((_, index) => !isIgnoredRequiredToken(stream, index));
|
|
618
|
+
if (first < 0)
|
|
619
|
+
return false;
|
|
620
|
+
const term = stream.tokens[first]?.value ?? '';
|
|
621
|
+
if (NEGATED_FINITE_CONTRACTIONS.has(term))
|
|
622
|
+
return true;
|
|
623
|
+
return NEGATED_FINITE_AUXILIARIES.has(term)
|
|
624
|
+
&& stream.tokens[first + 1]?.value === 'not';
|
|
625
|
+
}
|
|
626
|
+
function startsWithSubjectElidedFiniteAction(segment, actionForms) {
|
|
627
|
+
const stream = lexRequiredEvidence(segment);
|
|
628
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
629
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
630
|
+
continue;
|
|
631
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
632
|
+
if (LEADING_CLAUSE_MODIFIER_TERMS.has(term) || /ly$/u.test(term))
|
|
633
|
+
continue;
|
|
634
|
+
return stream.conditional[index] !== true
|
|
635
|
+
&& actionForms.has(term)
|
|
636
|
+
&& (term.endsWith('ed') || term.endsWith('s') || IRREGULAR_SIMPLE_PAST_FORMS.has(term));
|
|
637
|
+
}
|
|
638
|
+
return false;
|
|
639
|
+
}
|
|
640
|
+
const BASE_AGREEMENT_SUBJECTS = new Set(['i', 'they', 'we', 'you']);
|
|
641
|
+
const THIRD_PERSON_SUBJECTS = new Set(['he', 'it', 'she']);
|
|
642
|
+
const BASE_AGREEMENT_AUXILIARIES = new Set(['am', 'are', "aren't", 'do', "don't", 'have', "haven't"]);
|
|
643
|
+
const THIRD_PERSON_AUXILIARIES = new Set(['does', "doesn't", 'has', "hasn't", 'is', "isn't", 'needs']);
|
|
644
|
+
const NEGATED_AUXILIARY_TERMS = new Set([
|
|
645
|
+
'am', 'are', 'can', 'cannot', 'could', 'did', 'do', 'does', 'had', 'has', 'have', 'is', 'may', 'might',
|
|
646
|
+
'must', 'need', 'needs', 'ought', 'shall', 'should', 'was', 'were', 'will', 'would', "aren't", "can't", "couldn't", "didn't",
|
|
647
|
+
"doesn't", "don't", "hadn't", "hasn't", "haven't", "isn't", "mightn't", "mustn't", "shan't",
|
|
648
|
+
"needn't", "oughtn't", "shouldn't", "wasn't", "weren't", "won't", "wouldn't",
|
|
649
|
+
]);
|
|
650
|
+
function subjectAgreementBeforeNegation(segment) {
|
|
651
|
+
const stream = lexRequiredEvidence(segment);
|
|
652
|
+
let auxiliaryIndex = -1;
|
|
653
|
+
let auxiliary = '';
|
|
654
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
655
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
656
|
+
if (NEGATED_AUXILIARY_TERMS.has(term)
|
|
657
|
+
&& (term === 'cannot'
|
|
658
|
+
|| term.endsWith("n't")
|
|
659
|
+
|| ['not', 'never'].includes(stream.tokens[index + 1]?.value ?? ''))) {
|
|
660
|
+
auxiliaryIndex = index;
|
|
661
|
+
auxiliary = term;
|
|
662
|
+
break;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
if (auxiliaryIndex < 0)
|
|
666
|
+
return 'unknown';
|
|
667
|
+
if (BASE_AGREEMENT_AUXILIARIES.has(auxiliary))
|
|
668
|
+
return 'base';
|
|
669
|
+
if (THIRD_PERSON_AUXILIARIES.has(auxiliary))
|
|
670
|
+
return 'third-person';
|
|
671
|
+
let noun = '';
|
|
672
|
+
for (let index = 0; index < auxiliaryIndex; index += 1) {
|
|
673
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
674
|
+
continue;
|
|
675
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
676
|
+
if (BASE_AGREEMENT_SUBJECTS.has(term))
|
|
677
|
+
return 'base';
|
|
678
|
+
if (THIRD_PERSON_SUBJECTS.has(term))
|
|
679
|
+
return 'third-person';
|
|
680
|
+
if (SUBJECT_ARTICLE_TERMS.has(term)
|
|
681
|
+
|| NON_SUBJECT_PREFIX_TERMS.has(term)
|
|
682
|
+
|| LEADING_CLAUSE_MODIFIER_TERMS.has(term)
|
|
683
|
+
|| /ly$/u.test(term))
|
|
684
|
+
continue;
|
|
685
|
+
noun = term;
|
|
686
|
+
}
|
|
687
|
+
if (!noun)
|
|
688
|
+
return 'unknown';
|
|
689
|
+
return noun.endsWith('s') && !/(?:is|ss|us)$/u.test(noun) ? 'base' : 'third-person';
|
|
690
|
+
}
|
|
691
|
+
function negatedAuxiliaryScope(segment) {
|
|
692
|
+
if (/\b(?:(?:had|has|have)\s+(?:not|never)|hadn't|hasn't|haven't)\b/i.test(segment))
|
|
693
|
+
return 'perfect';
|
|
694
|
+
if (/\b(?:did\s+(?:not|never)|didn't)\b/i.test(segment))
|
|
695
|
+
return 'past-bare';
|
|
696
|
+
const progressive = /\b(?:(?:am|are|is|was|were)\s+(?:not|never)|aren't|isn't|wasn't|weren't)\b/i.exec(segment);
|
|
697
|
+
if (progressive) {
|
|
698
|
+
const suffix = segment.slice(progressive.index + progressive[0].length);
|
|
699
|
+
const localComplement = suffix.split(/[,;]|\b(?:after|because|before|once|so|when|while|without)\b/i, 1)[0] ?? '';
|
|
700
|
+
if (/\b[a-z]+ing\b/i.test(localComplement))
|
|
701
|
+
return 'progressive';
|
|
702
|
+
}
|
|
703
|
+
return /\b(?:(?:can|could|did|do|does|may|might|must|need|needs|ought|shall|should|will|would)\s+(?:not|never)|cannot|can't|couldn't|didn't|doesn't|don't|mightn't|mustn't|needn't|oughtn't|shan't|shouldn't|won't|wouldn't)\b/i.test(segment)
|
|
704
|
+
? 'bare'
|
|
705
|
+
: 'other';
|
|
706
|
+
}
|
|
707
|
+
function isThirdPersonActionForm(term, actionForms) {
|
|
708
|
+
if ([...IRREGULAR_VERB_FORMS.values()].some((forms) => forms[1] === term))
|
|
709
|
+
return true;
|
|
710
|
+
const bases = new Set();
|
|
711
|
+
addSingularCandidates(term, bases);
|
|
712
|
+
return [...bases].some((base) => actionForms.has(base));
|
|
713
|
+
}
|
|
714
|
+
function subjectElidedActionMorphology(segment, actionForms) {
|
|
715
|
+
const stream = lexRequiredEvidence(segment);
|
|
716
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
717
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
718
|
+
continue;
|
|
719
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
720
|
+
if (LEADING_CLAUSE_MODIFIER_TERMS.has(term) || /ly$/u.test(term))
|
|
721
|
+
continue;
|
|
722
|
+
if (!actionForms.has(term) || stream.conditional[index] === true)
|
|
723
|
+
return undefined;
|
|
724
|
+
if (term.endsWith('ing'))
|
|
725
|
+
return 'progressive';
|
|
726
|
+
if (IRREGULAR_SIMPLE_PAST_FORMS.has(term) && !IRREGULAR_PAST_PARTICIPLE_FORMS.has(term)) {
|
|
727
|
+
return 'simple-past';
|
|
728
|
+
}
|
|
729
|
+
if (term.endsWith('ed')
|
|
730
|
+
|| IRREGULAR_SIMPLE_PAST_FORMS.has(term)
|
|
731
|
+
|| IRREGULAR_PAST_PARTICIPLE_FORMS.has(term))
|
|
732
|
+
return 'participle';
|
|
733
|
+
return isThirdPersonActionForm(term, actionForms) ? 'third-person' : 'base';
|
|
734
|
+
}
|
|
735
|
+
return undefined;
|
|
736
|
+
}
|
|
737
|
+
function startsWithSubjectElidedGerund(segment) {
|
|
738
|
+
const stream = lexRequiredEvidence(segment);
|
|
739
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
740
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
741
|
+
continue;
|
|
742
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
743
|
+
if (LEADING_CLAUSE_MODIFIER_TERMS.has(term) || /ly$/u.test(term))
|
|
744
|
+
continue;
|
|
745
|
+
return term.endsWith('ing') && stream.conditional[index] !== true;
|
|
746
|
+
}
|
|
747
|
+
return false;
|
|
748
|
+
}
|
|
749
|
+
function finiteClauseSubjectPolarity(segment) {
|
|
750
|
+
const stream = lexRequiredEvidence(segment);
|
|
751
|
+
if (!hasFinitePredicatePrefix(stream, 0, stream.tokens.length))
|
|
752
|
+
return 'absent';
|
|
753
|
+
return leadingSubjectPolarity(stream, 0, stream.tokens.length);
|
|
754
|
+
}
|
|
755
|
+
function matchingActionSubjectPolarity(segment, actionForms) {
|
|
756
|
+
const stream = lexRequiredEvidence(segment);
|
|
757
|
+
for (let actionIndex = 0; actionIndex < stream.tokens.length; actionIndex += 1) {
|
|
758
|
+
if (!actionForms.has(stream.tokens[actionIndex]?.value ?? ''))
|
|
759
|
+
continue;
|
|
760
|
+
return leadingSubjectPolarity(stream, 0, actionIndex);
|
|
761
|
+
}
|
|
762
|
+
return 'absent';
|
|
763
|
+
}
|
|
764
|
+
function hasIndependentFiniteAction(segment, actionForms) {
|
|
765
|
+
const stream = lexRequiredEvidence(segment);
|
|
766
|
+
for (let actionIndex = 1; actionIndex < stream.tokens.length; actionIndex += 1) {
|
|
767
|
+
const term = stream.tokens[actionIndex]?.value ?? '';
|
|
768
|
+
if (!actionForms.has(term) || stream.conditional[actionIndex] === true)
|
|
769
|
+
continue;
|
|
770
|
+
const finite = term.endsWith('ed') || term.endsWith('s') || IRREGULAR_SIMPLE_PAST_FORMS.has(term);
|
|
771
|
+
if (finite && leadingSubjectPolarity(stream, 0, actionIndex) === 'positive')
|
|
772
|
+
return true;
|
|
773
|
+
}
|
|
774
|
+
return false;
|
|
775
|
+
}
|
|
776
|
+
function stripCommaDelimitedWithoutAdjuncts(clause, actionForms) {
|
|
777
|
+
return clause.replace(COMMA_DELIMITED_WITHOUT_RE, (match) => {
|
|
778
|
+
const coordinated = match.replace(/^\s*without\b/i, '').replace(/,\s*$/u, '').split(/\band\b/i).slice(1);
|
|
779
|
+
const hasFiniteCoordination = coordinated.some((segment) => (startsWithSubjectElidedFiniteAction(segment, actionForms)
|
|
780
|
+
|| hasIndependentFiniteAction(segment, actionForms)));
|
|
781
|
+
return hasFiniteCoordination ? match : ' ';
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
function affirmativePrefixBeforeNegation(segment, actionForms, inheritsSubject = true) {
|
|
785
|
+
const match = NEGATED_ACTION_RE.exec(segment);
|
|
786
|
+
if (!match || match.index === 0)
|
|
787
|
+
return undefined;
|
|
788
|
+
const prefix = normalize(segment.slice(0, match.index));
|
|
789
|
+
return prefix && hasIndependentAffirmativeAction(prefix, actionForms, inheritsSubject) ? prefix : undefined;
|
|
790
|
+
}
|
|
791
|
+
function hasIndependentSimplePastPredicate(segment) {
|
|
792
|
+
const stream = lexRequiredEvidence(segment);
|
|
793
|
+
for (let predicateIndex = 1; predicateIndex < stream.tokens.length; predicateIndex += 1) {
|
|
794
|
+
const term = stream.tokens[predicateIndex]?.value ?? '';
|
|
795
|
+
const simplePast = ['did', 'had', 'was', 'were'].includes(term)
|
|
796
|
+
|| term.endsWith('ed')
|
|
797
|
+
|| IRREGULAR_SIMPLE_PAST_FORMS.has(term);
|
|
798
|
+
if (simplePast && leadingSubjectPolarity(stream, 0, predicateIndex) === 'positive')
|
|
799
|
+
return true;
|
|
800
|
+
}
|
|
801
|
+
return false;
|
|
802
|
+
}
|
|
803
|
+
function hasIndependentSimplePastAction(segment, actionForms, inheritsSubject = false) {
|
|
804
|
+
const stream = lexRequiredEvidence(segment.slice(0, MAX_FACTUAL_TAIL_CHARS));
|
|
805
|
+
actionLoop: for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
806
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
807
|
+
if (!actionForms.has(term) || stream.conditional[index] === true)
|
|
808
|
+
continue;
|
|
809
|
+
for (let prior = index - 1; prior >= 0; prior -= 1) {
|
|
810
|
+
const priorTerm = stream.tokens[prior]?.value ?? '';
|
|
811
|
+
if (PREDICATE_COORDINATOR_TERMS.has(priorTerm))
|
|
812
|
+
break;
|
|
813
|
+
if (MODAL_AUXILIARY_TERMS.has(priorTerm) || SEMI_MODAL_AUXILIARY_TERMS.has(priorTerm)) {
|
|
814
|
+
continue actionLoop;
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
const subjectPolarity = leadingSubjectPolarity(stream, 0, index);
|
|
818
|
+
if ((term.endsWith('ed') || IRREGULAR_SIMPLE_PAST_FORMS.has(term))
|
|
819
|
+
&& (subjectPolarity === 'positive'
|
|
820
|
+
|| (inheritsSubject && subjectPolarity !== 'negative')))
|
|
821
|
+
return true;
|
|
822
|
+
}
|
|
823
|
+
return false;
|
|
824
|
+
}
|
|
825
|
+
function providedTargetPrefixKind(source, context) {
|
|
826
|
+
const objectTerms = context.terms.filter((term) => !term.verb);
|
|
827
|
+
return classifyProvidedTargetPrefix({
|
|
828
|
+
actionForms: context.actionForms,
|
|
829
|
+
lexicon: {
|
|
830
|
+
commaDelimitedPrepositions: COMMA_DELIMITED_PREPOSITION_TERMS,
|
|
831
|
+
conditionalMarkers: CONDITIONAL_MARKER_TERMS,
|
|
832
|
+
irregularPastParticiples: IRREGULAR_PAST_PARTICIPLE_FORMS,
|
|
833
|
+
irregularSimplePast: IRREGULAR_SIMPLE_PAST_FORMS,
|
|
834
|
+
leadingClauseModifiers: LEADING_CLAUSE_MODIFIER_TERMS,
|
|
835
|
+
passiveAuxiliaries: PASSIVE_AUXILIARY_TERMS,
|
|
836
|
+
subjectArticles: SUBJECT_ARTICLE_TERMS,
|
|
837
|
+
},
|
|
838
|
+
objectForms: objectTerms.map((term) => targetTermForms(term, 'must_not')),
|
|
839
|
+
predicates: {
|
|
840
|
+
finiteClauseSubjectPolarity,
|
|
841
|
+
hasIndependentAffirmativeAction,
|
|
842
|
+
isAttributiveTargetModifier,
|
|
843
|
+
isFinitePredicateTerm,
|
|
844
|
+
tokenize: lexRequiredEvidence,
|
|
845
|
+
},
|
|
846
|
+
source: normalize(source),
|
|
847
|
+
});
|
|
848
|
+
}
|
|
849
|
+
function hasTargetActionPrefix(source, targetActionForms) {
|
|
850
|
+
const stream = lexRequiredEvidence(source);
|
|
851
|
+
for (let actionIndex = 0; actionIndex < stream.tokens.length; actionIndex += 1) {
|
|
852
|
+
if (!targetActionForms.has(stream.tokens[actionIndex]?.value ?? ''))
|
|
853
|
+
continue;
|
|
854
|
+
if (actionIndex === 0 || leadingSubjectPolarity(stream, 0, actionIndex) === 'positive')
|
|
855
|
+
return true;
|
|
856
|
+
}
|
|
857
|
+
return false;
|
|
858
|
+
}
|
|
859
|
+
function hasProvidedMatrixAction(source, context) {
|
|
860
|
+
const commaIndex = source.indexOf(',');
|
|
861
|
+
if (commaIndex >= 0) {
|
|
862
|
+
const prefixKind = providedTargetPrefixKind(source.slice(0, commaIndex), context);
|
|
863
|
+
if (prefixKind === 'factual')
|
|
864
|
+
return false;
|
|
865
|
+
if (prefixKind === 'conditional')
|
|
866
|
+
return true;
|
|
867
|
+
}
|
|
868
|
+
let matrixScope = commaIndex >= 0
|
|
869
|
+
? source.slice(commaIndex + 1)
|
|
870
|
+
: source.replace(/^\s*that\b/iu, ' ');
|
|
871
|
+
const subordinateIndex = matrixScope.search(/\b(?:after|assuming|because|before|if|once|supposing|that|unless|when|while)\b/iu);
|
|
872
|
+
if (subordinateIndex >= 0)
|
|
873
|
+
matrixScope = matrixScope.slice(0, subordinateIndex);
|
|
874
|
+
const candidates = matrixScope.split(/\b(?:and|but|or|then)\b/iu);
|
|
875
|
+
return (commaIndex >= 0 ? candidates : candidates.slice(0, 1))
|
|
876
|
+
.some((candidate) => hasIndependentAffirmativeAction(candidate, context.targetActionForms));
|
|
877
|
+
}
|
|
878
|
+
function hasPotentialMarker(source, context) {
|
|
879
|
+
const { targetActionForms } = context;
|
|
880
|
+
const normalizedSource = source
|
|
881
|
+
.replace(CLOSED_IF_ANYTHING_RE, '$1 ')
|
|
882
|
+
.replace(COORDINATED_PROVIDED_RE, (match, connector, offset, whole) => (!targetActionForms.has('provided')
|
|
883
|
+
|| hasTargetActionPrefix(whole.slice(0, offset), targetActionForms)
|
|
884
|
+
|| hasProvidedMatrixAction(whole.slice(offset + match.length), context)
|
|
885
|
+
? match
|
|
886
|
+
: `${connector} `))
|
|
887
|
+
.replace(LEADING_PROVIDED_RE, (match, boundary, offset, whole) => (!targetActionForms.has('provided')
|
|
888
|
+
|| hasTargetActionPrefix(whole.slice(0, offset), targetActionForms)
|
|
889
|
+
|| hasProvidedMatrixAction(whole.slice(offset + match.length), context)
|
|
890
|
+
? match
|
|
891
|
+
: `${boundary} `));
|
|
892
|
+
return POTENTIAL_CONDITIONAL_SCOPE_RE.test(normalizedSource)
|
|
893
|
+
|| hasTargetActionBeforePostposedConditionalMarker(normalizedSource, targetActionForms);
|
|
894
|
+
}
|
|
895
|
+
function hasFactualProvidedAction(source, context) {
|
|
896
|
+
const stream = lexRequiredEvidence(source);
|
|
897
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
898
|
+
const token = stream.tokens[index];
|
|
899
|
+
if (token?.value !== 'provided' || isAdjectivalProvided(stream.tokens, index))
|
|
900
|
+
continue;
|
|
901
|
+
let clauseStart = index;
|
|
902
|
+
while (clauseStart > 0 && stream.tokens[clauseStart - 1]?.clause === token.clause)
|
|
903
|
+
clauseStart -= 1;
|
|
904
|
+
if (leadingSubjectPolarity(stream, clauseStart, index) !== 'positive')
|
|
905
|
+
continue;
|
|
906
|
+
const tail = stream.tokens.slice(index + 1)
|
|
907
|
+
.filter((candidate) => candidate.clause === token.clause)
|
|
908
|
+
.map((candidate) => candidate.value)
|
|
909
|
+
.join(' ');
|
|
910
|
+
if (!hasProvidedMatrixAction(tail, context))
|
|
911
|
+
return true;
|
|
912
|
+
}
|
|
913
|
+
return false;
|
|
914
|
+
}
|
|
915
|
+
function stripFactualTemporalMarkers(segment, context) {
|
|
916
|
+
let probe = segment.replace(PROVIDED_BY_ADJUNCT_RE, ' ');
|
|
917
|
+
probe = probe.replace(COORDINATED_TEMPORAL_RE, (match, connector, offset) => {
|
|
918
|
+
const prefix = probe.slice(0, offset);
|
|
919
|
+
const suffix = probe.slice(offset + match.length);
|
|
920
|
+
return hasIndependentSimplePastPredicate(prefix)
|
|
921
|
+
&& hasIndependentSimplePastAction(suffix, context.actionForms, true)
|
|
922
|
+
? `${connector} `
|
|
923
|
+
: match;
|
|
924
|
+
});
|
|
925
|
+
const leadingTemporal = LEADING_DISCOURSE_TEMPORAL_RE.exec(probe);
|
|
926
|
+
if (!leadingTemporal)
|
|
927
|
+
return probe;
|
|
928
|
+
const suffix = probe.slice(leadingTemporal[0].length);
|
|
929
|
+
return hasIndependentSimplePastAction(suffix, context.actionForms)
|
|
930
|
+
|| hasFactualProvidedAction(suffix, context)
|
|
931
|
+
? suffix
|
|
932
|
+
: probe;
|
|
933
|
+
}
|
|
934
|
+
function hasConditionalEvidence(probe, hadProvidedBy, context) {
|
|
935
|
+
return hasPotentialMarker(probe, context)
|
|
936
|
+
|| hasTargetActionBeforePostposedConditionalMarker(probe, context.targetActionForms)
|
|
937
|
+
|| (hadProvidedBy
|
|
938
|
+
&& !hasFactualProvidedAction(probe, context)
|
|
939
|
+
&& lexRequiredEvidence(probe).conditional.some(Boolean));
|
|
940
|
+
}
|
|
941
|
+
function hasPotentialConditionalScope(segment, actionForms, targetActionForms, terms) {
|
|
942
|
+
const context = { actionForms, targetActionForms, terms };
|
|
943
|
+
if (!hasPotentialMarker(segment, context))
|
|
944
|
+
return false;
|
|
945
|
+
const hadProvidedBy = segment.search(PROVIDED_BY_ADJUNCT_RE) >= 0;
|
|
946
|
+
const probe = stripFactualTemporalMarkers(segment, context);
|
|
947
|
+
return hasConditionalEvidence(probe, hadProvidedBy, context);
|
|
948
|
+
}
|
|
949
|
+
const NON_FACTUAL_MATRIX_AUXILIARIES = new Set([
|
|
950
|
+
'can', 'cannot', "can't", 'could', "couldn't", 'may', 'might', "mightn't", 'must', "mustn't",
|
|
951
|
+
'need', "needn't", 'ought', "oughtn't", 'shall', "shan't", 'should', "shouldn't", 'will', "won't",
|
|
952
|
+
'would', "wouldn't",
|
|
953
|
+
]);
|
|
954
|
+
function hasFactualMatrixPredicate(segment) {
|
|
955
|
+
const stream = lexRequiredEvidence(segment);
|
|
956
|
+
if (stream.tokens.some((token, index) => (!isIgnoredRequiredToken(stream, index) && NON_FACTUAL_MATRIX_AUXILIARIES.has(token.value))))
|
|
957
|
+
return false;
|
|
958
|
+
return finiteClauseSubjectPolarity(segment) !== 'absent';
|
|
959
|
+
}
|
|
960
|
+
const MAX_FACTUAL_TAIL_PROBES = 256;
|
|
961
|
+
const MAX_FACTUAL_TAIL_CHARS = 4_096;
|
|
962
|
+
function boundedFactualTail(segment, start) {
|
|
963
|
+
const window = segment.slice(start, start + MAX_FACTUAL_TAIL_CHARS);
|
|
964
|
+
const hardBoundary = window.search(/[.!?;\r\n]/u);
|
|
965
|
+
return normalize(hardBoundary >= 0 ? window.slice(0, hardBoundary) : window);
|
|
966
|
+
}
|
|
967
|
+
function factualTailsAfterBoundary(segment) {
|
|
968
|
+
const tails = [];
|
|
969
|
+
const seen = new Set();
|
|
970
|
+
const addTail = (start, temporal, matrixEnd, allowsInheritedGerund = false) => {
|
|
971
|
+
const candidate = boundedFactualTail(segment, start);
|
|
972
|
+
if (!candidate || seen.has(candidate))
|
|
973
|
+
return;
|
|
974
|
+
seen.add(candidate);
|
|
975
|
+
if (temporal) {
|
|
976
|
+
const prefixEnd = matrixEnd ?? start;
|
|
977
|
+
const matrixPrefix = normalize(segment.slice(Math.max(0, prefixEnd - MAX_FACTUAL_TAIL_CHARS), prefixEnd));
|
|
978
|
+
const inheritedGerund = allowsInheritedGerund && startsWithSubjectElidedGerund(candidate);
|
|
979
|
+
if (!hasFactualMatrixPredicate(matrixPrefix)
|
|
980
|
+
|| (!hasIndependentSimplePastPredicate(candidate) && !inheritedGerund))
|
|
981
|
+
return;
|
|
982
|
+
}
|
|
983
|
+
tails.push(candidate);
|
|
984
|
+
};
|
|
985
|
+
const boundaryRe = /\b(after|because|before|once|so|when|while)\b/gi;
|
|
986
|
+
let boundaryProbes = 0;
|
|
987
|
+
for (const match of segment.matchAll(boundaryRe)) {
|
|
988
|
+
if (boundaryProbes >= MAX_FACTUAL_TAIL_PROBES)
|
|
989
|
+
break;
|
|
990
|
+
boundaryProbes += 1;
|
|
991
|
+
const marker = match[1]?.toLowerCase() ?? '';
|
|
992
|
+
const temporal = marker === 'once' || marker === 'when' || marker === 'while';
|
|
993
|
+
addTail(match.index + match[0].length, temporal, match.index, marker === 'when' || marker === 'while');
|
|
994
|
+
}
|
|
995
|
+
const commaRe = /,/g;
|
|
996
|
+
let commaProbes = 0;
|
|
997
|
+
for (const match of segment.matchAll(commaRe)) {
|
|
998
|
+
if (commaProbes >= MAX_FACTUAL_TAIL_PROBES)
|
|
999
|
+
break;
|
|
1000
|
+
commaProbes += 1;
|
|
1001
|
+
addTail(match.index + 1, false);
|
|
1002
|
+
}
|
|
1003
|
+
return tails;
|
|
1004
|
+
}
|
|
1005
|
+
function affirmativeFragmentFromTail(candidate, actionForms, inheritsSubject) {
|
|
1006
|
+
if (NEGATED_ACTION_RE.test(candidate)) {
|
|
1007
|
+
return affirmativePrefixBeforeNegation(candidate, actionForms, inheritsSubject);
|
|
1008
|
+
}
|
|
1009
|
+
return hasIndependentAffirmativeAction(candidate, actionForms, inheritsSubject) ? candidate : undefined;
|
|
1010
|
+
}
|
|
1011
|
+
function factualTailSubjectPolarity(segment) {
|
|
1012
|
+
const stream = lexRequiredEvidence(segment);
|
|
1013
|
+
for (let predicateIndex = 1; predicateIndex < stream.tokens.length; predicateIndex += 1) {
|
|
1014
|
+
if (stream.conditional[predicateIndex] === true)
|
|
1015
|
+
continue;
|
|
1016
|
+
const term = stream.tokens[predicateIndex]?.value ?? '';
|
|
1017
|
+
const finite = isFinitePredicateTerm(term)
|
|
1018
|
+
|| IRREGULAR_SIMPLE_PAST_FORMS.has(term)
|
|
1019
|
+
|| term.endsWith('ed')
|
|
1020
|
+
|| (KNOWN_EVIDENCE_ACTION_TERMS.has(term) && term.endsWith('s'));
|
|
1021
|
+
if (finite)
|
|
1022
|
+
return leadingSubjectPolarity(stream, 0, predicateIndex);
|
|
1023
|
+
}
|
|
1024
|
+
return 'absent';
|
|
1025
|
+
}
|
|
1026
|
+
function factualTailEvidence(tails, actionForms, inheritsSubject) {
|
|
1027
|
+
let subjectMode = 'none';
|
|
1028
|
+
let subjectAgreement = 'unknown';
|
|
1029
|
+
for (const tail of tails) {
|
|
1030
|
+
const affirmative = affirmativeFragmentFromTail(tail, actionForms, inheritsSubject);
|
|
1031
|
+
if (affirmative)
|
|
1032
|
+
return { affirmative, subjectAgreement: 'unknown', subjectMode: 'affirmative' };
|
|
1033
|
+
if (NEGATED_ACTION_RE.test(tail)) {
|
|
1034
|
+
if (hasExplicitSubjectBeforeNegation(tail)) {
|
|
1035
|
+
const auxiliaryScope = negatedAuxiliaryScope(tail);
|
|
1036
|
+
subjectMode = auxiliaryScope === 'bare'
|
|
1037
|
+
? 'negated-bare'
|
|
1038
|
+
: auxiliaryScope === 'past-bare'
|
|
1039
|
+
? 'negated-past-bare'
|
|
1040
|
+
: auxiliaryScope === 'perfect'
|
|
1041
|
+
? 'negated-perfect'
|
|
1042
|
+
: auxiliaryScope === 'progressive'
|
|
1043
|
+
? 'negated-progressive'
|
|
1044
|
+
: 'finite';
|
|
1045
|
+
subjectAgreement = subjectAgreementBeforeNegation(tail);
|
|
1046
|
+
}
|
|
1047
|
+
continue;
|
|
1048
|
+
}
|
|
1049
|
+
if (inheritsSubject && startsWithSubjectElidedGerund(tail)) {
|
|
1050
|
+
subjectMode = 'affirmative';
|
|
1051
|
+
subjectAgreement = 'unknown';
|
|
1052
|
+
continue;
|
|
1053
|
+
}
|
|
1054
|
+
const polarity = factualTailSubjectPolarity(tail);
|
|
1055
|
+
if (polarity === 'positive') {
|
|
1056
|
+
subjectMode = 'affirmative';
|
|
1057
|
+
subjectAgreement = 'unknown';
|
|
1058
|
+
}
|
|
1059
|
+
else if (polarity === 'negative' && subjectMode === 'none') {
|
|
1060
|
+
subjectMode = 'negative';
|
|
1061
|
+
subjectAgreement = 'unknown';
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
return { subjectAgreement, subjectMode };
|
|
1065
|
+
}
|
|
1066
|
+
function contextualNoAdjunctAllowsImperative(segment) {
|
|
1067
|
+
const commaIndex = segment.indexOf(',');
|
|
1068
|
+
if (commaIndex < 0)
|
|
1069
|
+
return false;
|
|
1070
|
+
const terms = normalize(segment.slice(0, commaIndex)).split(' ').filter(Boolean);
|
|
1071
|
+
return terms[0] === 'no' && (terms[1] === 'later' || terms[1] === 'matter');
|
|
1072
|
+
}
|
|
1073
|
+
function predicateConditionality(segment, actionForms) {
|
|
1074
|
+
const stream = lexRequiredEvidence(segment);
|
|
1075
|
+
let conditional = false;
|
|
1076
|
+
let unconditional = false;
|
|
1077
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
1078
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
1079
|
+
continue;
|
|
1080
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
1081
|
+
const predicate = actionForms.has(term)
|
|
1082
|
+
|| isFinitePredicateTerm(term)
|
|
1083
|
+
|| IRREGULAR_SIMPLE_PAST_FORMS.has(term)
|
|
1084
|
+
|| term.endsWith('ed')
|
|
1085
|
+
|| (KNOWN_EVIDENCE_ACTION_TERMS.has(term) && /(?:ing|s)$/u.test(term));
|
|
1086
|
+
if (!predicate)
|
|
1087
|
+
continue;
|
|
1088
|
+
if (stream.conditional[index] === true)
|
|
1089
|
+
conditional = true;
|
|
1090
|
+
else
|
|
1091
|
+
unconditional = true;
|
|
1092
|
+
}
|
|
1093
|
+
if (conditional && unconditional)
|
|
1094
|
+
return 'mixed';
|
|
1095
|
+
if (conditional)
|
|
1096
|
+
return 'conditional';
|
|
1097
|
+
if (unconditional)
|
|
1098
|
+
return 'unconditional';
|
|
1099
|
+
return 'none';
|
|
1100
|
+
}
|
|
1101
|
+
function hasMatrixPredicateBeforeConditionalMarker(segment, actionForms) {
|
|
1102
|
+
const stream = lexRequiredEvidence(segment);
|
|
1103
|
+
const markerIndex = stream.tokens.findIndex((token, index) => (!isIgnoredRequiredToken(stream, index)
|
|
1104
|
+
&& CONDITIONAL_MARKER_TERMS.has(token.value)
|
|
1105
|
+
&& stream.conditional[index] === true));
|
|
1106
|
+
if (markerIndex < 0)
|
|
1107
|
+
return false;
|
|
1108
|
+
for (let index = 0; index < markerIndex; index += 1) {
|
|
1109
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
1110
|
+
continue;
|
|
1111
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
1112
|
+
if (actionForms.has(term)
|
|
1113
|
+
|| isFinitePredicateTerm(term)
|
|
1114
|
+
|| IRREGULAR_SIMPLE_PAST_FORMS.has(term)
|
|
1115
|
+
|| term.endsWith('ed')
|
|
1116
|
+
|| (KNOWN_EVIDENCE_ACTION_TERMS.has(term) && /(?:ing|s)$/u.test(term)))
|
|
1117
|
+
return true;
|
|
1118
|
+
}
|
|
1119
|
+
return false;
|
|
1120
|
+
}
|
|
1121
|
+
function hasTargetActionBeforePostposedConditionalMarker(segment, targetActionForms) {
|
|
1122
|
+
const stream = lexRequiredEvidence(segment);
|
|
1123
|
+
let targetActionIndex;
|
|
1124
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
1125
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
1126
|
+
const embeddedGovernorIndex = embeddedIfGovernorIndex(stream.tokens, index);
|
|
1127
|
+
const targetActionIsEmbeddedGovernor = embeddedGovernorIndex !== undefined
|
|
1128
|
+
&& embeddedGovernorIndex === targetActionIndex;
|
|
1129
|
+
const postposedMarker = isConditionalMarkerUse(stream.tokens, index)
|
|
1130
|
+
|| targetActionIsEmbeddedGovernor
|
|
1131
|
+
|| (TEMPORAL_CONDITIONAL_MARKER_TERMS.has(term)
|
|
1132
|
+
&& !isMentionedConditionalMarker(stream.tokens, index)
|
|
1133
|
+
&& !isFactualTemporalMarkerUse(stream.tokens, index));
|
|
1134
|
+
if (targetActionIndex !== undefined
|
|
1135
|
+
&& postposedMarker) {
|
|
1136
|
+
if (!TEMPORAL_CONDITIONAL_MARKER_TERMS.has(term))
|
|
1137
|
+
return true;
|
|
1138
|
+
const prefix = segment.slice(0, stream.tokens[index]?.start ?? 0);
|
|
1139
|
+
if (!hasIndependentSimplePastAction(prefix, targetActionForms))
|
|
1140
|
+
return true;
|
|
1141
|
+
}
|
|
1142
|
+
if (targetActionIndex === undefined
|
|
1143
|
+
&& targetActionForms.has(term)
|
|
1144
|
+
&& (index === 0 || leadingSubjectPolarity(stream, 0, index) === 'positive')) {
|
|
1145
|
+
targetActionIndex = index;
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
return false;
|
|
1149
|
+
}
|
|
1150
|
+
function affirmativeOutputClauses(redactedOutput, terms) {
|
|
1151
|
+
const actionForms = new Set(AFFIRMATIVE_ACTION_FORMS);
|
|
1152
|
+
const targetActionForms = new Set();
|
|
1153
|
+
for (const term of terms) {
|
|
1154
|
+
if (!term.verb)
|
|
1155
|
+
continue;
|
|
1156
|
+
for (const form of targetTermForms(term, 'must_not')) {
|
|
1157
|
+
actionForms.add(form);
|
|
1158
|
+
targetActionForms.add(form);
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
return redactedOutput
|
|
1162
|
+
.split(/(?<=[.!?;])|\r?\n|\b(?:but|however|although|though)\b/gi)
|
|
1163
|
+
.map((clause) => clause.replace(REDACTION_MARKER_RE, ' ').trim())
|
|
1164
|
+
.filter((clause) => clause.length > 0)
|
|
1165
|
+
.flatMap((rawClause) => {
|
|
1166
|
+
const scopedClause = stripCommaDelimitedWithoutAdjuncts(rawClause, actionForms);
|
|
1167
|
+
const clause = normalize(scopedClause);
|
|
1168
|
+
if (!NEGATED_ACTION_RE.test(clause)
|
|
1169
|
+
&& !WITHOUT_ACTION_RE.test(clause)
|
|
1170
|
+
&& !NEGATIVE_SUBJECT_MARKER_RE.test(clause)
|
|
1171
|
+
&& !hasPotentialConditionalScope(scopedClause, actionForms, targetActionForms, terms))
|
|
1172
|
+
return [clause];
|
|
1173
|
+
const parts = scopedClause.split(/\b(and|but|however|although|though|or|then|without)\b/i);
|
|
1174
|
+
const segments = [];
|
|
1175
|
+
let connector;
|
|
1176
|
+
for (let index = 0; index < parts.length; index += 1) {
|
|
1177
|
+
const part = parts[index] ?? '';
|
|
1178
|
+
if (index % 2 === 1) {
|
|
1179
|
+
connector = part.trim().toLowerCase();
|
|
1180
|
+
continue;
|
|
1181
|
+
}
|
|
1182
|
+
const rawText = part.trim();
|
|
1183
|
+
const text = normalize(rawText);
|
|
1184
|
+
if (!text)
|
|
1185
|
+
continue;
|
|
1186
|
+
segments.push({ ...(connector ? { connector } : {}), rawText, text });
|
|
1187
|
+
connector = undefined;
|
|
1188
|
+
}
|
|
1189
|
+
let requiresIndependentAction = NEGATED_ACTION_RE.test(segments[0]?.text ?? '');
|
|
1190
|
+
let inheritedSubjectMode = 'none';
|
|
1191
|
+
let inheritedSubjectAgreement = 'unknown';
|
|
1192
|
+
let conditionalCoordination = false;
|
|
1193
|
+
let conditionalCoordinationAllowsIndependentReset = false;
|
|
1194
|
+
return segments.flatMap((segment, index) => {
|
|
1195
|
+
if (segment.connector === 'without') {
|
|
1196
|
+
requiresIndependentAction = true;
|
|
1197
|
+
if (/,\s*$/u.test(segment.rawText)) {
|
|
1198
|
+
conditionalCoordination = false;
|
|
1199
|
+
conditionalCoordinationAllowsIndependentReset = false;
|
|
1200
|
+
}
|
|
1201
|
+
return [];
|
|
1202
|
+
}
|
|
1203
|
+
const postposedTargetConditional = hasTargetActionBeforePostposedConditionalMarker(segment.rawText, targetActionForms);
|
|
1204
|
+
const ownConditional = postposedTargetConditional
|
|
1205
|
+
|| (predicateConditionality(segment.text, actionForms) === 'conditional'
|
|
1206
|
+
&& !hasMatrixPredicateBeforeConditionalMarker(segment.text, actionForms));
|
|
1207
|
+
const inheritedConditional = conditionalCoordination
|
|
1208
|
+
&& (segment.connector === 'and' || segment.connector === 'or' || segment.connector === 'then')
|
|
1209
|
+
&& !(conditionalCoordinationAllowsIndependentReset
|
|
1210
|
+
&& hasIndependentAffirmativeAction(segment.text, actionForms));
|
|
1211
|
+
const conditionalSegment = ownConditional || inheritedConditional;
|
|
1212
|
+
const conditionalAllowsIndependentReset = ownConditional
|
|
1213
|
+
? postposedTargetConditional
|
|
1214
|
+
: conditionalCoordinationAllowsIndependentReset;
|
|
1215
|
+
const carriesConditionalCoordination = !/,\s*$/u.test(segment.rawText);
|
|
1216
|
+
if (NEGATED_ACTION_RE.test(segment.text)) {
|
|
1217
|
+
const affirmativePrefix = affirmativePrefixBeforeNegation(segment.text, actionForms);
|
|
1218
|
+
const negatedSubjectPolarity = finiteClauseSubjectPolarity(segment.text);
|
|
1219
|
+
const factualTails = factualTailsAfterBoundary(segment.rawText);
|
|
1220
|
+
const tailEvidence = factualTailEvidence(factualTails, actionForms, negatedSubjectPolarity !== 'negative');
|
|
1221
|
+
if (conditionalSegment) {
|
|
1222
|
+
conditionalCoordination = carriesConditionalCoordination;
|
|
1223
|
+
conditionalCoordinationAllowsIndependentReset = carriesConditionalCoordination
|
|
1224
|
+
&& conditionalAllowsIndependentReset;
|
|
1225
|
+
requiresIndependentAction = true;
|
|
1226
|
+
return [];
|
|
1227
|
+
}
|
|
1228
|
+
conditionalCoordination = false;
|
|
1229
|
+
conditionalCoordinationAllowsIndependentReset = false;
|
|
1230
|
+
requiresIndependentAction = true;
|
|
1231
|
+
const explicitSubject = hasExplicitSubjectBeforeNegation(segment.text);
|
|
1232
|
+
const preservesInheritedSubject = segment.connector === 'and'
|
|
1233
|
+
&& inheritedSubjectMode !== 'none'
|
|
1234
|
+
&& startsWithSubjectElidedNegation(segment.text);
|
|
1235
|
+
if (negatedSubjectPolarity === 'negative') {
|
|
1236
|
+
inheritedSubjectMode = 'negative';
|
|
1237
|
+
inheritedSubjectAgreement = 'unknown';
|
|
1238
|
+
}
|
|
1239
|
+
else if (explicitSubject) {
|
|
1240
|
+
const auxiliaryScope = negatedAuxiliaryScope(segment.text);
|
|
1241
|
+
inheritedSubjectMode = auxiliaryScope === 'bare'
|
|
1242
|
+
? 'negated-bare'
|
|
1243
|
+
: auxiliaryScope === 'past-bare'
|
|
1244
|
+
? 'negated-past-bare'
|
|
1245
|
+
: auxiliaryScope === 'perfect'
|
|
1246
|
+
? 'negated-perfect'
|
|
1247
|
+
: auxiliaryScope === 'progressive'
|
|
1248
|
+
? 'negated-progressive'
|
|
1249
|
+
: 'finite';
|
|
1250
|
+
inheritedSubjectAgreement = subjectAgreementBeforeNegation(segment.text);
|
|
1251
|
+
}
|
|
1252
|
+
else if (!preservesInheritedSubject) {
|
|
1253
|
+
inheritedSubjectMode = 'none';
|
|
1254
|
+
inheritedSubjectAgreement = 'unknown';
|
|
1255
|
+
}
|
|
1256
|
+
if (tailEvidence.subjectMode !== 'none') {
|
|
1257
|
+
inheritedSubjectMode = tailEvidence.subjectMode;
|
|
1258
|
+
inheritedSubjectAgreement = tailEvidence.subjectAgreement;
|
|
1259
|
+
}
|
|
1260
|
+
return [affirmativePrefix, tailEvidence.affirmative]
|
|
1261
|
+
.filter((fragment) => fragment !== undefined);
|
|
1262
|
+
}
|
|
1263
|
+
const finiteSubjectPolarity = finiteClauseSubjectPolarity(segment.text);
|
|
1264
|
+
const subjectPolarity = finiteSubjectPolarity === 'absent'
|
|
1265
|
+
? matchingActionSubjectPolarity(segment.text, actionForms)
|
|
1266
|
+
: finiteSubjectPolarity;
|
|
1267
|
+
if (subjectPolarity === 'negative') {
|
|
1268
|
+
inheritedSubjectMode = 'negative';
|
|
1269
|
+
inheritedSubjectAgreement = 'unknown';
|
|
1270
|
+
requiresIndependentAction = true;
|
|
1271
|
+
const factualTails = factualTailsAfterBoundary(segment.rawText);
|
|
1272
|
+
const tailEvidence = factualTailEvidence(factualTails, actionForms, contextualNoAdjunctAllowsImperative(segment.rawText));
|
|
1273
|
+
if (conditionalSegment) {
|
|
1274
|
+
conditionalCoordination = carriesConditionalCoordination;
|
|
1275
|
+
conditionalCoordinationAllowsIndependentReset = carriesConditionalCoordination
|
|
1276
|
+
&& conditionalAllowsIndependentReset;
|
|
1277
|
+
return [];
|
|
1278
|
+
}
|
|
1279
|
+
conditionalCoordination = false;
|
|
1280
|
+
conditionalCoordinationAllowsIndependentReset = false;
|
|
1281
|
+
if (tailEvidence.subjectMode !== 'none') {
|
|
1282
|
+
inheritedSubjectMode = tailEvidence.subjectMode;
|
|
1283
|
+
inheritedSubjectAgreement = tailEvidence.subjectAgreement;
|
|
1284
|
+
}
|
|
1285
|
+
return tailEvidence.affirmative ? [tailEvidence.affirmative] : [];
|
|
1286
|
+
}
|
|
1287
|
+
if (conditionalSegment) {
|
|
1288
|
+
conditionalCoordination = carriesConditionalCoordination;
|
|
1289
|
+
conditionalCoordinationAllowsIndependentReset = carriesConditionalCoordination
|
|
1290
|
+
&& conditionalAllowsIndependentReset;
|
|
1291
|
+
requiresIndependentAction = true;
|
|
1292
|
+
return [];
|
|
1293
|
+
}
|
|
1294
|
+
conditionalCoordination = false;
|
|
1295
|
+
conditionalCoordinationAllowsIndependentReset = false;
|
|
1296
|
+
if (subjectPolarity === 'positive')
|
|
1297
|
+
inheritedSubjectMode = 'affirmative';
|
|
1298
|
+
if (index === 0 || !requiresIndependentAction)
|
|
1299
|
+
return [segment.text];
|
|
1300
|
+
const followsComma = /,\s*$/u.test(segments[index - 1]?.rawText ?? '');
|
|
1301
|
+
if (segment.connector === 'and'
|
|
1302
|
+
&& followsComma
|
|
1303
|
+
&& inheritedSubjectMode !== 'negative'
|
|
1304
|
+
&& hasIndependentAffirmativeAction(segment.text, actionForms, true))
|
|
1305
|
+
return [segment.text];
|
|
1306
|
+
if (segment.connector === 'and' && inheritedSubjectMode === 'affirmative'
|
|
1307
|
+
&& hasIndependentAffirmativeAction(segment.text, actionForms, true))
|
|
1308
|
+
return [segment.text];
|
|
1309
|
+
const morphology = subjectElidedActionMorphology(segment.text, actionForms);
|
|
1310
|
+
if (segment.connector === 'and' && inheritedSubjectMode === 'negated-perfect'
|
|
1311
|
+
&& ((morphology === 'base' && inheritedSubjectAgreement === 'base')
|
|
1312
|
+
|| (morphology === 'third-person' && inheritedSubjectAgreement === 'third-person'))) {
|
|
1313
|
+
return [segment.text];
|
|
1314
|
+
}
|
|
1315
|
+
if (segment.connector === 'and' && inheritedSubjectMode === 'negated-progressive'
|
|
1316
|
+
&& (morphology === 'participle'
|
|
1317
|
+
|| morphology === 'simple-past'
|
|
1318
|
+
|| (morphology === 'base' && inheritedSubjectAgreement === 'base')
|
|
1319
|
+
|| (morphology === 'third-person' && inheritedSubjectAgreement === 'third-person'))) {
|
|
1320
|
+
return [segment.text];
|
|
1321
|
+
}
|
|
1322
|
+
if (segment.connector === 'and' && inheritedSubjectMode === 'negated-bare'
|
|
1323
|
+
&& (morphology === 'simple-past'
|
|
1324
|
+
|| (morphology === 'third-person' && inheritedSubjectAgreement === 'third-person'))) {
|
|
1325
|
+
return [segment.text];
|
|
1326
|
+
}
|
|
1327
|
+
if (segment.connector === 'and' && inheritedSubjectMode === 'negated-past-bare'
|
|
1328
|
+
&& (morphology === 'participle'
|
|
1329
|
+
|| morphology === 'simple-past'
|
|
1330
|
+
|| (morphology === 'third-person' && inheritedSubjectAgreement === 'third-person'))) {
|
|
1331
|
+
return [segment.text];
|
|
1332
|
+
}
|
|
1333
|
+
if (segment.connector === 'and' && inheritedSubjectMode === 'finite'
|
|
1334
|
+
&& startsWithSubjectElidedFiniteAction(segment.text, actionForms))
|
|
1335
|
+
return [segment.text];
|
|
1336
|
+
const thenInheritsSubject = segment.connector === 'then' && inheritedSubjectMode !== 'negative';
|
|
1337
|
+
return hasIndependentAffirmativeAction(segment.text, actionForms, thenInheritsSubject)
|
|
1338
|
+
? [segment.text]
|
|
1339
|
+
: [];
|
|
1340
|
+
});
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
function mustNotMatchedTerms(redactedOutput, terms) {
|
|
1344
|
+
return affirmativeOutputClauses(redactedOutput, terms)
|
|
1345
|
+
.map((clause) => {
|
|
1346
|
+
const matched = terms.filter((term) => termMatches(clause, term, 'must_not'));
|
|
1347
|
+
if (terms.some((term) => term.verb) && !matched.some((term) => term.verb))
|
|
1348
|
+
return [];
|
|
1349
|
+
return matched.map((term) => term.text);
|
|
1350
|
+
})
|
|
1351
|
+
.sort((left, right) => right.length - left.length)[0] ?? [];
|
|
1352
|
+
}
|
|
1353
|
+
function isBroadProgressiveAuxiliary(term) {
|
|
1354
|
+
if (!term.verb || !term.text.endsWith('ing'))
|
|
1355
|
+
return false;
|
|
1356
|
+
const lemma = verbLemma(term.text);
|
|
1357
|
+
return lemma !== undefined && EXACT_ONLY_PROGRESSIVE_LEMMAS.has(lemma);
|
|
1358
|
+
}
|
|
1359
|
+
const BROAD_EVIDENCE_LINK_TERMS = new Set([
|
|
1360
|
+
'a', 'an', 'the', 'all', 'any', 'both', 'each', 'every', 'her', 'his', 'its', 'my', 'not', 'only', 'our',
|
|
1361
|
+
'some', 'that', 'their', 'these', 'this', 'those', 'your',
|
|
1362
|
+
]);
|
|
1363
|
+
const PASSIVE_AUXILIARY_TERMS = new Set([
|
|
1364
|
+
'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'has', 'have', 'had',
|
|
1365
|
+
]);
|
|
1366
|
+
const PASSIVE_BE_AUXILIARY_TERMS = new Set([
|
|
1367
|
+
'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
1368
|
+
]);
|
|
1369
|
+
const PERFECT_AUXILIARY_TERMS = new Set(['had', 'has', 'have']);
|
|
1370
|
+
const PERFECT_BROAD_FORM_TERMS = new Set(['been', 'done', 'gone']);
|
|
1371
|
+
const MODAL_AUXILIARY_TERMS = new Set([
|
|
1372
|
+
'can', 'could', 'may', 'might', 'must', 'shall', 'should', 'will', 'would',
|
|
1373
|
+
]);
|
|
1374
|
+
const SEMI_MODAL_AUXILIARY_TERMS = new Set(['need', 'needed', 'needs', 'ought']);
|
|
1375
|
+
const INDEPENDENT_BROAD_FORM_TERMS = new Set([
|
|
1376
|
+
'am', 'are', 'is', 'was', 'were', 'does', 'did', 'goes', 'went', 'has', 'have', 'had',
|
|
1377
|
+
]);
|
|
1378
|
+
const NEGATIVE_EVIDENCE_PRONOUNS = new Set(['neither', 'nobody', 'none', 'nothing']);
|
|
1379
|
+
const NEGATIVE_EVIDENCE_DETERMINERS = new Set(['neither', 'no', 'zero']);
|
|
1380
|
+
const SUBJECT_PRONOUN_TERMS = new Set([
|
|
1381
|
+
'he', 'i', 'it', 'she', 'that', 'they', 'this', 'we', 'who', 'you',
|
|
1382
|
+
]);
|
|
1383
|
+
const SUBJECT_ARTICLE_TERMS = new Set(['a', 'an', 'the']);
|
|
1384
|
+
const AUXILIARY_CHAIN_MODIFIER_TERMS = new Set([
|
|
1385
|
+
'already', 'also', 'always', 'ever', 'just', 'still', 'yet',
|
|
1386
|
+
]);
|
|
1387
|
+
const LEADING_CLAUSE_MODIFIER_TERMS = new Set([
|
|
1388
|
+
'eventually', 'finally', 'initially', 'later', 'subsequently',
|
|
1389
|
+
]);
|
|
1390
|
+
// One capped token stream; each predicate probes only bounded nearby targets.
|
|
1391
|
+
// This keeps replay evaluation O(tokens * target terms * window) and memory linear.
|
|
1392
|
+
const MAX_REQUIRED_EVIDENCE_TERMS = 1_024;
|
|
1393
|
+
const MAX_REQUIRED_EVIDENCE_CANDIDATES = 256;
|
|
1394
|
+
const MAX_REQUIRED_EVIDENCE_PAIR_DISTANCE = 64;
|
|
1395
|
+
const MAX_REQUIRED_SUBJECT_LOOKBACK = 128;
|
|
1396
|
+
const PREDICATE_COORDINATOR_TERMS = new Set(['and', 'but', 'then']);
|
|
1397
|
+
const CONDITIONAL_MARKER_TERMS = new Set([
|
|
1398
|
+
'assuming', 'if', 'once', 'provided', 'supposing', 'unless', 'when', 'while',
|
|
1399
|
+
]);
|
|
1400
|
+
const TEMPORAL_CONDITIONAL_MARKER_TERMS = new Set(['once', 'when', 'while']);
|
|
1401
|
+
const CONDITIONAL_MARKER_MENTION_HEAD_TERMS = new Set(['keyword', 'term', 'word']);
|
|
1402
|
+
const EMBEDDED_IF_GOVERNOR_TERMS = new Set([
|
|
1403
|
+
'ask', 'asked', 'asking', 'asks',
|
|
1404
|
+
'check', 'checked', 'checking', 'checks',
|
|
1405
|
+
'confirm', 'confirmed', 'confirming', 'confirms',
|
|
1406
|
+
'decide', 'decided', 'decides', 'deciding',
|
|
1407
|
+
'determine', 'determined', 'determines', 'determining',
|
|
1408
|
+
'evaluate', 'evaluated', 'evaluates', 'evaluating',
|
|
1409
|
+
'learn', 'learned', 'learning', 'learns',
|
|
1410
|
+
'see', 'seeing', 'seen', 'sees', 'saw',
|
|
1411
|
+
'test', 'tested', 'testing', 'tests',
|
|
1412
|
+
'verify', 'verified', 'verifies', 'verifying',
|
|
1413
|
+
'wonder', 'wondered', 'wondering', 'wonders',
|
|
1414
|
+
]);
|
|
1415
|
+
const PREDICATE_SCOPE_RESET_TERMS = new Set([
|
|
1416
|
+
...CONDITIONAL_MARKER_TERMS,
|
|
1417
|
+
'although', 'as', 'because', 'however', 'once', 'though', 'when', 'whereas', 'while', 'without', 'yet',
|
|
1418
|
+
]);
|
|
1419
|
+
const COMMA_DELIMITED_SUBORDINATOR_TERMS = new Set([
|
|
1420
|
+
...CONDITIONAL_MARKER_TERMS,
|
|
1421
|
+
'although', 'as', 'because', 'once', 'though', 'when', 'whereas', 'while',
|
|
1422
|
+
]);
|
|
1423
|
+
const TARGET_SCOPE_RESET_TERMS_LOCAL = new Set([
|
|
1424
|
+
...PREDICATE_SCOPE_RESET_TERMS,
|
|
1425
|
+
...PREDICATE_COORDINATOR_TERMS,
|
|
1426
|
+
'after', 'before', 'since', 'than', 'that',
|
|
1427
|
+
]);
|
|
1428
|
+
const PASSIVE_SUBJECT_POSTMODIFIER_TERMS = new Set([
|
|
1429
|
+
'among', 'around', 'at', 'by', 'for', 'from', 'in', 'of', 'on', 'through', 'to', 'with', 'without',
|
|
1430
|
+
]);
|
|
1431
|
+
const COMMA_DELIMITED_PREPOSITION_TERMS = new Set([
|
|
1432
|
+
...PASSIVE_SUBJECT_POSTMODIFIER_TERMS,
|
|
1433
|
+
'according', 'despite', 'during', 'inside', 'outside', 'regarding', 'within',
|
|
1434
|
+
]);
|
|
1435
|
+
const TOTALITY_TARGET_MODIFIER_TERMS = new Set([
|
|
1436
|
+
'complete', 'entire', 'full', 'partial', 'total', 'whole',
|
|
1437
|
+
]);
|
|
1438
|
+
const KNOWN_EVIDENCE_ACTION_TERMS = new Set([
|
|
1439
|
+
...[...IRREGULAR_VERB_FORMS.values()].flat(),
|
|
1440
|
+
...GENERIC_ACTION_TERMS,
|
|
1441
|
+
]);
|
|
1442
|
+
const KNOWN_FINITE_PREDICATE_TERMS = new Set([
|
|
1443
|
+
...INDEPENDENT_BROAD_FORM_TERMS,
|
|
1444
|
+
...MODAL_AUXILIARY_TERMS,
|
|
1445
|
+
'said', 'told',
|
|
1446
|
+
]);
|
|
1447
|
+
const CONTEXTUAL_ZERO_METRIC_TERMS = new Set([
|
|
1448
|
+
'cost', 'downtime', 'latency', 'loss', 'overhead', 'variance',
|
|
1449
|
+
]);
|
|
1450
|
+
const ROOT_PREDICATE_BLOCKING_MODIFIER_TERMS = new Set([
|
|
1451
|
+
'allegedly', 'almost', 'apparently', 'maybe', 'nearly', 'perhaps', 'possibly', 'reportedly', 'supposedly',
|
|
1452
|
+
]);
|
|
1453
|
+
const CLAUSE_INITIAL_EVIDENTIAL_HEDGE_TERMS = new Set([
|
|
1454
|
+
...ROOT_PREDICATE_BLOCKING_MODIFIER_TERMS,
|
|
1455
|
+
'likely', 'presumably', 'probably', 'purportedly',
|
|
1456
|
+
]);
|
|
1457
|
+
const EMBEDDED_CLAIM_GOVERNOR_TERMS = new Set([
|
|
1458
|
+
'allege', 'alleged', 'alleges', 'alleging',
|
|
1459
|
+
'assert', 'asserted', 'asserting', 'asserts',
|
|
1460
|
+
'believe', 'believed', 'believes', 'believing',
|
|
1461
|
+
'claim', 'claimed', 'claims', 'claiming',
|
|
1462
|
+
'deny', 'denied', 'denies', 'denying',
|
|
1463
|
+
'hear', 'heard', 'hearing', 'hears',
|
|
1464
|
+
'report', 'reported', 'reporting', 'reports',
|
|
1465
|
+
'say', 'said', 'saying', 'says',
|
|
1466
|
+
'suppose', 'supposed', 'supposes', 'supposing',
|
|
1467
|
+
'tell', 'telling', 'tells', 'told',
|
|
1468
|
+
'think', 'thinking', 'thinks', 'thought',
|
|
1469
|
+
]);
|
|
1470
|
+
const UNAMBIGUOUS_EMBEDDED_CLAIM_GOVERNOR_TERMS = new Set([
|
|
1471
|
+
'asserts', 'heard', 'said', 'thought', 'told',
|
|
1472
|
+
]);
|
|
1473
|
+
const NOMINAL_CLAIM_HEAD_TERMS = new Set(['claim', 'claims', 'report', 'reports']);
|
|
1474
|
+
const RELATIVE_SAFE_FINITE_COMPLEMENTS = new Map([
|
|
1475
|
+
['made', new Set(['progress'])],
|
|
1476
|
+
['makes', new Set(['progress'])],
|
|
1477
|
+
['pass', new Set(['review'])],
|
|
1478
|
+
['passed', new Set(['review'])],
|
|
1479
|
+
['passes', new Set(['review'])],
|
|
1480
|
+
['survive', new Set(['review'])],
|
|
1481
|
+
['survived', new Set(['review'])],
|
|
1482
|
+
['survives', new Set(['review'])],
|
|
1483
|
+
['takes', new Set(['effect'])],
|
|
1484
|
+
['took', new Set(['effect'])],
|
|
1485
|
+
]);
|
|
1486
|
+
const NON_ADVERBIAL_PARENTHETICAL_IF_NOT_TERMS = new Set(['already', 'complete']);
|
|
1487
|
+
const AFFIRMATIVE_PREDICATE_MODIFIER_TERMS = new Set([
|
|
1488
|
+
...AUXILIARY_CHAIN_MODIFIER_TERMS,
|
|
1489
|
+
'actually', 'currently', 'definitely', 'only', 'successfully',
|
|
1490
|
+
]);
|
|
1491
|
+
const INVERTED_CONDITIONAL_AUXILIARY_TERMS = new Set([
|
|
1492
|
+
...MODAL_AUXILIARY_TERMS,
|
|
1493
|
+
'had', 'was', 'were',
|
|
1494
|
+
]);
|
|
1495
|
+
const NON_SUBJECT_PREFIX_TERMS = new Set([
|
|
1496
|
+
...PASSIVE_AUXILIARY_TERMS,
|
|
1497
|
+
...MODAL_AUXILIARY_TERMS,
|
|
1498
|
+
...SEMI_MODAL_AUXILIARY_TERMS,
|
|
1499
|
+
...LEADING_CLAUSE_MODIFIER_TERMS,
|
|
1500
|
+
'after', 'before', 'not', 'only', 'to',
|
|
1501
|
+
]);
|
|
1502
|
+
function isBlockingRootPredicateModifierTerm(term) {
|
|
1503
|
+
return ROOT_PREDICATE_BLOCKING_MODIFIER_TERMS.has(term)
|
|
1504
|
+
|| (term.endsWith('ly')
|
|
1505
|
+
&& !AFFIRMATIVE_PREDICATE_MODIFIER_TERMS.has(term)
|
|
1506
|
+
&& !LEADING_CLAUSE_MODIFIER_TERMS.has(term));
|
|
1507
|
+
}
|
|
1508
|
+
function isClauseInitialEvidentialHedgeTerm(term) {
|
|
1509
|
+
return CLAUSE_INITIAL_EVIDENTIAL_HEDGE_TERMS.has(term);
|
|
1510
|
+
}
|
|
1511
|
+
function isAdjectivalProvided(tokens, index) {
|
|
1512
|
+
if (tokens[index]?.value !== 'provided')
|
|
1513
|
+
return false;
|
|
1514
|
+
if (tokens[index + 1]?.value === 'by')
|
|
1515
|
+
return true;
|
|
1516
|
+
const clauseId = tokens[index].clause;
|
|
1517
|
+
let previous = index - 1;
|
|
1518
|
+
while (previous >= 0
|
|
1519
|
+
&& tokens[previous]?.clause === clauseId
|
|
1520
|
+
&& /ly$/u.test(tokens[previous]?.value ?? ''))
|
|
1521
|
+
previous -= 1;
|
|
1522
|
+
const previousTerm = tokens[previous]?.clause === clauseId ? tokens[previous]?.value ?? '' : '';
|
|
1523
|
+
return previousTerm !== 'not'
|
|
1524
|
+
&& previousTerm !== 'only'
|
|
1525
|
+
&& BROAD_EVIDENCE_LINK_TERMS.has(previousTerm);
|
|
1526
|
+
}
|
|
1527
|
+
function hasSubjectBeforeRegularPast(tokens, predicateIndex, lowerBound) {
|
|
1528
|
+
const clauseId = tokens[predicateIndex]?.clause;
|
|
1529
|
+
for (let index = predicateIndex - 1; index >= lowerBound; index -= 1) {
|
|
1530
|
+
const token = tokens[index];
|
|
1531
|
+
if (!token || token.clause !== clauseId)
|
|
1532
|
+
return false;
|
|
1533
|
+
const term = token.value;
|
|
1534
|
+
if (PREDICATE_COORDINATOR_TERMS.has(term) || PREDICATE_SCOPE_RESET_TERMS.has(term))
|
|
1535
|
+
return false;
|
|
1536
|
+
if (SUBJECT_PRONOUN_TERMS.has(term) || NEGATIVE_EVIDENCE_PRONOUNS.has(term))
|
|
1537
|
+
return true;
|
|
1538
|
+
if (SUBJECT_ARTICLE_TERMS.has(term)
|
|
1539
|
+
|| BROAD_EVIDENCE_LINK_TERMS.has(term)
|
|
1540
|
+
|| NON_SUBJECT_PREFIX_TERMS.has(term)
|
|
1541
|
+
|| /ly$/u.test(term)
|
|
1542
|
+
|| isAttributiveTargetModifier(term))
|
|
1543
|
+
continue;
|
|
1544
|
+
return true;
|
|
1545
|
+
}
|
|
1546
|
+
return false;
|
|
1547
|
+
}
|
|
1548
|
+
function isFinitePredicateBeforeParenthetical(tokens, index, lowerBound) {
|
|
1549
|
+
const term = tokens[index]?.value ?? '';
|
|
1550
|
+
if (KNOWN_FINITE_PREDICATE_TERMS.has(term) || /n't$/u.test(term))
|
|
1551
|
+
return true;
|
|
1552
|
+
return term.endsWith('ed') && hasSubjectBeforeRegularPast(tokens, index, lowerBound);
|
|
1553
|
+
}
|
|
1554
|
+
function isParentheticalIfNotModifier(tokens, index) {
|
|
1555
|
+
if (tokens[index]?.value !== 'if' || tokens[index + 1]?.value !== 'not')
|
|
1556
|
+
return false;
|
|
1557
|
+
const modifier = tokens[index + 2];
|
|
1558
|
+
const next = tokens[index + 3];
|
|
1559
|
+
const closedModifier = modifier?.clause === tokens[index]?.clause
|
|
1560
|
+
&& (next === undefined || next.clause !== modifier.clause || next.commaBefore);
|
|
1561
|
+
if (!closedModifier)
|
|
1562
|
+
return false;
|
|
1563
|
+
if (/ly$/u.test(modifier.value))
|
|
1564
|
+
return true;
|
|
1565
|
+
if (!tokens[index]?.commaBefore
|
|
1566
|
+
|| !NON_ADVERBIAL_PARENTHETICAL_IF_NOT_TERMS.has(modifier.value))
|
|
1567
|
+
return false;
|
|
1568
|
+
const lowerBound = Math.max(0, index - MAX_REQUIRED_SUBJECT_LOOKBACK);
|
|
1569
|
+
for (let prior = index - 1; prior >= lowerBound && tokens[prior]?.clause === modifier.clause; prior -= 1) {
|
|
1570
|
+
const term = tokens[prior]?.value ?? '';
|
|
1571
|
+
if (PREDICATE_COORDINATOR_TERMS.has(term) || PREDICATE_SCOPE_RESET_TERMS.has(term))
|
|
1572
|
+
return false;
|
|
1573
|
+
if (isFinitePredicateBeforeParenthetical(tokens, prior, lowerBound))
|
|
1574
|
+
return true;
|
|
1575
|
+
}
|
|
1576
|
+
return false;
|
|
1577
|
+
}
|
|
1578
|
+
function isParentheticalIfAnythingModifier(tokens, index) {
|
|
1579
|
+
const marker = tokens[index];
|
|
1580
|
+
const modifier = tokens[index + 1];
|
|
1581
|
+
const next = tokens[index + 2];
|
|
1582
|
+
if (marker?.value !== 'if'
|
|
1583
|
+
|| modifier?.value !== 'anything'
|
|
1584
|
+
|| modifier.clause !== marker.clause)
|
|
1585
|
+
return false;
|
|
1586
|
+
const closesBeforePostmodifier = next !== undefined
|
|
1587
|
+
&& next.clause === modifier.clause
|
|
1588
|
+
&& COMMA_DELIMITED_PREPOSITION_TERMS.has(next.value);
|
|
1589
|
+
const closesBeforeClauseBoundary = next !== undefined
|
|
1590
|
+
&& next.clause === modifier.clause
|
|
1591
|
+
&& TARGET_SCOPE_RESET_TERMS_LOCAL.has(next.value);
|
|
1592
|
+
const closedModifier = next === undefined
|
|
1593
|
+
|| next.clause !== modifier.clause
|
|
1594
|
+
|| next.commaBefore
|
|
1595
|
+
|| closesBeforePostmodifier
|
|
1596
|
+
|| closesBeforeClauseBoundary;
|
|
1597
|
+
return closedModifier && (index === 0
|
|
1598
|
+
|| marker.commaBefore
|
|
1599
|
+
|| next === undefined
|
|
1600
|
+
|| closesBeforePostmodifier
|
|
1601
|
+
|| closesBeforeClauseBoundary);
|
|
1602
|
+
}
|
|
1603
|
+
function isMentionedConditionalMarker(tokens, index) {
|
|
1604
|
+
const token = tokens[index];
|
|
1605
|
+
const previous = tokens[index - 1];
|
|
1606
|
+
return token !== undefined
|
|
1607
|
+
&& previous?.clause === token.clause
|
|
1608
|
+
&& CONDITIONAL_MARKER_MENTION_HEAD_TERMS.has(previous.value);
|
|
1609
|
+
}
|
|
1610
|
+
function embeddedIfGovernorIndex(tokens, index) {
|
|
1611
|
+
const marker = tokens[index];
|
|
1612
|
+
if (marker?.value !== 'if' || marker.commaBefore)
|
|
1613
|
+
return undefined;
|
|
1614
|
+
for (let previous = index - 1; previous >= 0; previous -= 1) {
|
|
1615
|
+
const token = tokens[previous];
|
|
1616
|
+
if (!token || token.clause !== marker.clause)
|
|
1617
|
+
return undefined;
|
|
1618
|
+
const term = token.value;
|
|
1619
|
+
if (EMBEDDED_IF_GOVERNOR_TERMS.has(term))
|
|
1620
|
+
return previous;
|
|
1621
|
+
if (PREDICATE_COORDINATOR_TERMS.has(term)
|
|
1622
|
+
|| CONDITIONAL_MARKER_TERMS.has(term)
|
|
1623
|
+
|| AFFIRMATIVE_ACTION_FORMS.has(term)
|
|
1624
|
+
|| isFinitePredicateTerm(term))
|
|
1625
|
+
return undefined;
|
|
1626
|
+
if (tokens[previous + 1]?.commaBefore)
|
|
1627
|
+
return undefined;
|
|
1628
|
+
}
|
|
1629
|
+
return undefined;
|
|
1630
|
+
}
|
|
1631
|
+
function isEmbeddedIfComplement(tokens, index) {
|
|
1632
|
+
return embeddedIfGovernorIndex(tokens, index) !== undefined;
|
|
1633
|
+
}
|
|
1634
|
+
function isFactualProvidedPredicate(tokens, index) {
|
|
1635
|
+
const provided = tokens[index];
|
|
1636
|
+
if (provided?.value !== 'provided' || provided.commaBefore)
|
|
1637
|
+
return false;
|
|
1638
|
+
let segmentStart = index;
|
|
1639
|
+
for (let previous = index - 1; previous >= 0; previous -= 1) {
|
|
1640
|
+
const token = tokens[previous];
|
|
1641
|
+
if (!token || token.clause !== provided.clause)
|
|
1642
|
+
break;
|
|
1643
|
+
if (tokens[previous + 1]?.commaBefore || PREDICATE_COORDINATOR_TERMS.has(token.value))
|
|
1644
|
+
break;
|
|
1645
|
+
segmentStart = previous;
|
|
1646
|
+
}
|
|
1647
|
+
if (segmentStart === index)
|
|
1648
|
+
return false;
|
|
1649
|
+
for (let previous = segmentStart; previous < index; previous += 1) {
|
|
1650
|
+
const term = tokens[previous]?.value ?? '';
|
|
1651
|
+
if (AFFIRMATIVE_ACTION_FORMS.has(term)
|
|
1652
|
+
|| isFinitePredicateTerm(term)
|
|
1653
|
+
|| KNOWN_EVIDENCE_ACTION_TERMS.has(term))
|
|
1654
|
+
return false;
|
|
1655
|
+
}
|
|
1656
|
+
return hasSubjectBeforeRegularPast(tokens, index, segmentStart);
|
|
1657
|
+
}
|
|
1658
|
+
function isConditionalMarkerUse(tokens, index) {
|
|
1659
|
+
const term = tokens[index]?.value ?? '';
|
|
1660
|
+
if (!CONDITIONAL_MARKER_TERMS.has(term)
|
|
1661
|
+
|| isAdjectivalProvided(tokens, index)
|
|
1662
|
+
|| isFactualProvidedPredicate(tokens, index)
|
|
1663
|
+
|| isParentheticalIfNotModifier(tokens, index)
|
|
1664
|
+
|| isParentheticalIfAnythingModifier(tokens, index)
|
|
1665
|
+
|| isMentionedConditionalMarker(tokens, index)
|
|
1666
|
+
|| isEmbeddedIfComplement(tokens, index))
|
|
1667
|
+
return false;
|
|
1668
|
+
if (!TEMPORAL_CONDITIONAL_MARKER_TERMS.has(term))
|
|
1669
|
+
return true;
|
|
1670
|
+
if (isFactualTemporalMarkerUse(tokens, index))
|
|
1671
|
+
return false;
|
|
1672
|
+
const clauseId = tokens[index]?.clause;
|
|
1673
|
+
let hardClauseStart = index;
|
|
1674
|
+
while (hardClauseStart > 0 && tokens[hardClauseStart - 1]?.clause === clauseId)
|
|
1675
|
+
hardClauseStart -= 1;
|
|
1676
|
+
if (term === 'once') {
|
|
1677
|
+
let segmentStart = hardClauseStart;
|
|
1678
|
+
for (let prior = index - 1; prior >= hardClauseStart; prior -= 1) {
|
|
1679
|
+
if (PREDICATE_COORDINATOR_TERMS.has(tokens[prior]?.value ?? '')) {
|
|
1680
|
+
segmentStart = prior + 1;
|
|
1681
|
+
break;
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
let nextIndex = index + 1;
|
|
1685
|
+
while (tokens[nextIndex]?.clause === clauseId && /ly$/u.test(tokens[nextIndex]?.value ?? ''))
|
|
1686
|
+
nextIndex += 1;
|
|
1687
|
+
const next = tokens[nextIndex];
|
|
1688
|
+
const nextTerm = next?.value ?? '';
|
|
1689
|
+
const followedByPredicate = next !== undefined
|
|
1690
|
+
&& next.clause === clauseId
|
|
1691
|
+
&& !next.commaBefore
|
|
1692
|
+
&& (isFinitePredicateTerm(nextTerm)
|
|
1693
|
+
|| INDEPENDENT_AFFIRMATIVE_AUXILIARIES.has(nextTerm)
|
|
1694
|
+
|| KNOWN_EVIDENCE_ACTION_TERMS.has(nextTerm));
|
|
1695
|
+
if (followedByPredicate && hasSubjectBeforeRegularPast(tokens, index, segmentStart))
|
|
1696
|
+
return false;
|
|
1697
|
+
}
|
|
1698
|
+
return conditionalScopeStart(tokens, hardClauseStart, index) === index;
|
|
1699
|
+
}
|
|
1700
|
+
function isFactualTemporalMarkerUse(tokens, index) {
|
|
1701
|
+
const term = tokens[index]?.value ?? '';
|
|
1702
|
+
const previous = tokens[index - 1]?.value ?? '';
|
|
1703
|
+
const next = tokens[index + 1]?.value ?? '';
|
|
1704
|
+
return (term === 'once' && isFactualOnceUse(tokens, index))
|
|
1705
|
+
|| (term === 'while' && (['a', 'that', 'the', 'this'].includes(previous) || next === 'later'));
|
|
1706
|
+
}
|
|
1707
|
+
function isFactualOnceUse(tokens, index) {
|
|
1708
|
+
if (tokens[index]?.value !== 'once')
|
|
1709
|
+
return false;
|
|
1710
|
+
const previous = tokens[index - 1]?.value ?? '';
|
|
1711
|
+
const next = tokens[index + 1]?.value ?? '';
|
|
1712
|
+
return tokens[index]?.compound === true
|
|
1713
|
+
|| ['at', 'for', 'just'].includes(previous)
|
|
1714
|
+
|| ['again', 'more', 'upon'].includes(next);
|
|
1715
|
+
}
|
|
1716
|
+
function hasMatrixSubjectBeforeTemporalAdjunct(source, offset) {
|
|
1717
|
+
const hardPrefix = source.slice(Math.max(0, offset - 256), offset).split(/[.!?;\r\n]/u).pop() ?? '';
|
|
1718
|
+
const segment = hardPrefix.slice(hardPrefix.lastIndexOf(',') + 1);
|
|
1719
|
+
const terms = normalize(segment).split(' ').filter(Boolean);
|
|
1720
|
+
if (terms.length === 0)
|
|
1721
|
+
return false;
|
|
1722
|
+
if (terms.some((term) => SUBJECT_PRONOUN_TERMS.has(term)))
|
|
1723
|
+
return true;
|
|
1724
|
+
if (terms.some((term, index) => SUBJECT_ARTICLE_TERMS.has(term) && index + 1 < terms.length))
|
|
1725
|
+
return true;
|
|
1726
|
+
if (terms.some((term, index) => NEGATIVE_EVIDENCE_DETERMINERS.has(term) && index + 1 < terms.length))
|
|
1727
|
+
return true;
|
|
1728
|
+
const first = terms[0] ?? '';
|
|
1729
|
+
return !LEADING_CLAUSE_MODIFIER_TERMS.has(first)
|
|
1730
|
+
&& !COMMA_DELIMITED_PREPOSITION_TERMS.has(first)
|
|
1731
|
+
&& !NON_SUBJECT_PREFIX_TERMS.has(first)
|
|
1732
|
+
&& !/ly$/u.test(first);
|
|
1733
|
+
}
|
|
1734
|
+
function stripFactualTemporalParentheticals(source) {
|
|
1735
|
+
return source.replace(COMMA_DELIMITED_TEMPORAL_RE, (match, offset, whole) => {
|
|
1736
|
+
if (!hasMatrixSubjectBeforeTemporalAdjunct(whole, offset))
|
|
1737
|
+
return match;
|
|
1738
|
+
const suffixStart = offset + match.length;
|
|
1739
|
+
const suffixWindow = whole.slice(suffixStart, suffixStart + 256).split(/[.!?;\r\n]/u, 1)[0] ?? '';
|
|
1740
|
+
const suffixTerms = normalize(suffixWindow).split(' ').filter(Boolean);
|
|
1741
|
+
const keepsModality = suffixTerms.slice(0, 4).some((term) => (MODAL_AUXILIARY_TERMS.has(term) || SEMI_MODAL_AUXILIARY_TERMS.has(term)));
|
|
1742
|
+
return keepsModality ? match : ' ';
|
|
1743
|
+
});
|
|
1744
|
+
}
|
|
1745
|
+
function conditionalScopeStart(tokens, hardClauseStart, markerIndex) {
|
|
1746
|
+
let segmentStart = hardClauseStart;
|
|
1747
|
+
for (let index = markerIndex - 1; index >= hardClauseStart; index -= 1) {
|
|
1748
|
+
if (PREDICATE_COORDINATOR_TERMS.has(tokens[index]?.value ?? '')) {
|
|
1749
|
+
segmentStart = index + 1;
|
|
1750
|
+
break;
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
for (let index = segmentStart; index < markerIndex; index += 1) {
|
|
1754
|
+
const term = tokens[index]?.value ?? '';
|
|
1755
|
+
if (isFinitePredicateTerm(term)
|
|
1756
|
+
|| KNOWN_EVIDENCE_ACTION_TERMS.has(term)
|
|
1757
|
+
|| term.endsWith('ing'))
|
|
1758
|
+
return segmentStart;
|
|
1759
|
+
}
|
|
1760
|
+
return markerIndex;
|
|
1761
|
+
}
|
|
1762
|
+
function lexRequiredEvidence(redactedOutput) {
|
|
1763
|
+
const source = stripFactualTemporalParentheticals(redactedOutput.replace(REDACTION_MARKER_RE, ' '))
|
|
1764
|
+
.toLowerCase()
|
|
1765
|
+
.replace(/[\u2018\u2019]/g, "'");
|
|
1766
|
+
const tokens = [];
|
|
1767
|
+
const matches = source.matchAll(/[a-z0-9_]+(?:'[a-z]+)?/gu);
|
|
1768
|
+
let previousEnd = 0;
|
|
1769
|
+
let clause = 0;
|
|
1770
|
+
for (const match of matches) {
|
|
1771
|
+
if (tokens.length >= MAX_REQUIRED_EVIDENCE_TERMS)
|
|
1772
|
+
break;
|
|
1773
|
+
const start = match.index;
|
|
1774
|
+
const end = start + match[0].length;
|
|
1775
|
+
const separator = source.slice(previousEnd, start);
|
|
1776
|
+
if (tokens.length > 0 && /[.!?;\r\n]/u.test(separator))
|
|
1777
|
+
clause += 1;
|
|
1778
|
+
tokens.push({
|
|
1779
|
+
value: match[0],
|
|
1780
|
+
start,
|
|
1781
|
+
end,
|
|
1782
|
+
clause,
|
|
1783
|
+
commaBefore: separator.includes(','),
|
|
1784
|
+
compound: source[start - 1] === '-' || source[end] === '-',
|
|
1785
|
+
});
|
|
1786
|
+
previousEnd = end;
|
|
1787
|
+
}
|
|
1788
|
+
const ignored = Array.from({ length: tokens.length }, () => false);
|
|
1789
|
+
const transparentCommaBefore = Array.from({ length: tokens.length }, () => false);
|
|
1790
|
+
const hardClauseStarts = Array.from({ length: tokens.length }, () => 0);
|
|
1791
|
+
const hardClauseEnds = Array.from({ length: tokens.length }, () => tokens.length);
|
|
1792
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
1793
|
+
if (isFactualOnceUse(tokens, index))
|
|
1794
|
+
ignored[index] = true;
|
|
1795
|
+
}
|
|
1796
|
+
let hardClauseStart = 0;
|
|
1797
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
1798
|
+
if (index > 0 && tokens[index - 1]?.clause !== tokens[index]?.clause)
|
|
1799
|
+
hardClauseStart = index;
|
|
1800
|
+
hardClauseStarts[index] = hardClauseStart;
|
|
1801
|
+
}
|
|
1802
|
+
let hardClauseEnd = tokens.length;
|
|
1803
|
+
for (let index = tokens.length - 1; index >= 0; index -= 1) {
|
|
1804
|
+
const token = tokens[index];
|
|
1805
|
+
const next = tokens[index + 1];
|
|
1806
|
+
if (!token)
|
|
1807
|
+
continue;
|
|
1808
|
+
if (next === undefined || next.clause !== token.clause) {
|
|
1809
|
+
hardClauseEnd = index + 1;
|
|
1810
|
+
}
|
|
1811
|
+
hardClauseEnds[index] = hardClauseEnd;
|
|
1812
|
+
}
|
|
1813
|
+
const failClosedSubordinateRanges = [];
|
|
1814
|
+
for (let index = 0; index + 1 < tokens.length; index += 1) {
|
|
1815
|
+
if (!isParentheticalIfNotModifier(tokens, index))
|
|
1816
|
+
continue;
|
|
1817
|
+
const clauseId = tokens[index].clause;
|
|
1818
|
+
let close = index + 1;
|
|
1819
|
+
while (close < tokens.length
|
|
1820
|
+
&& tokens[close]?.clause === clauseId
|
|
1821
|
+
&& !tokens[close]?.commaBefore)
|
|
1822
|
+
close += 1;
|
|
1823
|
+
if (close >= tokens.length || tokens[close]?.clause !== clauseId)
|
|
1824
|
+
continue;
|
|
1825
|
+
const atScopeStart = index === 0
|
|
1826
|
+
|| tokens[index - 1]?.clause !== clauseId
|
|
1827
|
+
|| tokens[index]?.commaBefore;
|
|
1828
|
+
if (!atScopeStart)
|
|
1829
|
+
continue;
|
|
1830
|
+
const embedded = index > 0
|
|
1831
|
+
&& tokens[index - 1]?.clause === clauseId
|
|
1832
|
+
&& tokens[index]?.commaBefore;
|
|
1833
|
+
for (let ignoredIndex = index; ignoredIndex < close; ignoredIndex += 1)
|
|
1834
|
+
ignored[ignoredIndex] = true;
|
|
1835
|
+
if (embedded) {
|
|
1836
|
+
transparentCommaBefore[index] = true;
|
|
1837
|
+
transparentCommaBefore[close] = true;
|
|
1838
|
+
}
|
|
1839
|
+
index = close - 1;
|
|
1840
|
+
}
|
|
1841
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
1842
|
+
const startTerm = tokens[index]?.value ?? '';
|
|
1843
|
+
const structuralPrefix = COMMA_DELIMITED_SUBORDINATOR_TERMS.has(startTerm)
|
|
1844
|
+
|| COMMA_DELIMITED_PREPOSITION_TERMS.has(startTerm)
|
|
1845
|
+
|| isBlockingRootPredicateModifierTerm(startTerm)
|
|
1846
|
+
|| /ly$/u.test(startTerm);
|
|
1847
|
+
if (ignored[index] || !structuralPrefix)
|
|
1848
|
+
continue;
|
|
1849
|
+
const clauseId = tokens[index].clause;
|
|
1850
|
+
let close = index + 1;
|
|
1851
|
+
while (close < tokens.length
|
|
1852
|
+
&& tokens[close]?.clause === clauseId
|
|
1853
|
+
&& !tokens[close]?.commaBefore)
|
|
1854
|
+
close += 1;
|
|
1855
|
+
if (close >= tokens.length || tokens[close]?.clause !== clauseId)
|
|
1856
|
+
continue;
|
|
1857
|
+
const atScopeStart = index === 0
|
|
1858
|
+
|| tokens[index - 1]?.clause !== clauseId
|
|
1859
|
+
|| tokens[index]?.commaBefore;
|
|
1860
|
+
if (!atScopeStart)
|
|
1861
|
+
continue;
|
|
1862
|
+
const embedded = index > 0
|
|
1863
|
+
&& tokens[index - 1]?.clause === clauseId
|
|
1864
|
+
&& tokens[index]?.commaBefore;
|
|
1865
|
+
let containsFinitePredicate = false;
|
|
1866
|
+
for (let spanIndex = index; spanIndex < close; spanIndex += 1) {
|
|
1867
|
+
if (isFinitePredicateTerm(tokens[spanIndex]?.value ?? ''))
|
|
1868
|
+
containsFinitePredicate = true;
|
|
1869
|
+
}
|
|
1870
|
+
if (isConditionalMarkerUse(tokens, index)) {
|
|
1871
|
+
failClosedSubordinateRanges.push({
|
|
1872
|
+
start: conditionalScopeStart(tokens, hardClauseStarts[index] ?? 0, index),
|
|
1873
|
+
end: hardClauseEnds[index] ?? close,
|
|
1874
|
+
});
|
|
1875
|
+
}
|
|
1876
|
+
else if (isClauseInitialEvidentialHedgeTerm(startTerm)) {
|
|
1877
|
+
failClosedSubordinateRanges.push({ start: index, end: hardClauseEnds[index] ?? close });
|
|
1878
|
+
}
|
|
1879
|
+
else if (containsFinitePredicate) {
|
|
1880
|
+
failClosedSubordinateRanges.push({ start: index, end: close });
|
|
1881
|
+
}
|
|
1882
|
+
else {
|
|
1883
|
+
for (let ignoredIndex = index; ignoredIndex < close; ignoredIndex += 1)
|
|
1884
|
+
ignored[ignoredIndex] = true;
|
|
1885
|
+
}
|
|
1886
|
+
if (embedded) {
|
|
1887
|
+
transparentCommaBefore[index] = true;
|
|
1888
|
+
transparentCommaBefore[close] = true;
|
|
1889
|
+
}
|
|
1890
|
+
index = close - 1;
|
|
1891
|
+
}
|
|
1892
|
+
const conditional = Array.from({ length: tokens.length }, () => false);
|
|
1893
|
+
// Factual subordinate predicates are ambiguous replay evidence and remain fail-closed.
|
|
1894
|
+
for (const range of failClosedSubordinateRanges) {
|
|
1895
|
+
for (let index = range.start; index < range.end; index += 1)
|
|
1896
|
+
conditional[index] = true;
|
|
1897
|
+
}
|
|
1898
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
1899
|
+
if (ignored[index] || !isConditionalMarkerUse(tokens, index))
|
|
1900
|
+
continue;
|
|
1901
|
+
const clauseId = tokens[index].clause;
|
|
1902
|
+
const scopeStart = conditionalScopeStart(tokens, hardClauseStarts[index] ?? 0, index);
|
|
1903
|
+
const scopeEnd = hardClauseEnds[index] ?? tokens.length;
|
|
1904
|
+
for (let conditionalIndex = scopeStart; conditionalIndex < scopeEnd; conditionalIndex += 1) {
|
|
1905
|
+
const token = tokens[conditionalIndex];
|
|
1906
|
+
if (!token || token.clause !== clauseId)
|
|
1907
|
+
break;
|
|
1908
|
+
conditional[conditionalIndex] = true;
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
1912
|
+
const token = tokens[index];
|
|
1913
|
+
let clauseInitial = index === 0 || tokens[index - 1]?.clause !== token?.clause;
|
|
1914
|
+
if (!clauseInitial && token?.commaBefore) {
|
|
1915
|
+
clauseInitial = true;
|
|
1916
|
+
for (let prefix = index - 1; prefix >= 0; prefix -= 1) {
|
|
1917
|
+
const prefixToken = tokens[prefix];
|
|
1918
|
+
if (!prefixToken || prefixToken.clause !== token.clause)
|
|
1919
|
+
break;
|
|
1920
|
+
if (ignored[prefix]
|
|
1921
|
+
|| LEADING_CLAUSE_MODIFIER_TERMS.has(prefixToken.value)
|
|
1922
|
+
|| AFFIRMATIVE_PREDICATE_MODIFIER_TERMS.has(prefixToken.value))
|
|
1923
|
+
continue;
|
|
1924
|
+
clauseInitial = false;
|
|
1925
|
+
break;
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
if (!token || !clauseInitial || !INVERTED_CONDITIONAL_AUXILIARY_TERMS.has(token.value))
|
|
1929
|
+
continue;
|
|
1930
|
+
const scopeEnd = hardClauseEnds[index] ?? tokens.length;
|
|
1931
|
+
for (let conditionalIndex = index; conditionalIndex < scopeEnd; conditionalIndex += 1) {
|
|
1932
|
+
if (tokens[conditionalIndex]?.clause !== token.clause)
|
|
1933
|
+
break;
|
|
1934
|
+
conditional[conditionalIndex] = true;
|
|
1935
|
+
}
|
|
1936
|
+
}
|
|
1937
|
+
return { tokens, ignored, transparentCommaBefore, conditional };
|
|
1938
|
+
}
|
|
1939
|
+
function isIgnoredRequiredToken(stream, index) {
|
|
1940
|
+
return stream.ignored[index] === true;
|
|
1941
|
+
}
|
|
1942
|
+
function hasEffectiveCommaBefore(stream, index) {
|
|
1943
|
+
return stream.tokens[index]?.commaBefore === true && stream.transparentCommaBefore[index] !== true;
|
|
1944
|
+
}
|
|
1945
|
+
function isNegativeRequiredToken(stream, index) {
|
|
1946
|
+
const token = stream.tokens[index];
|
|
1947
|
+
const lexicalNoOne = token?.value === 'no'
|
|
1948
|
+
&& token.compound
|
|
1949
|
+
&& stream.tokens[index + 1]?.value === 'one'
|
|
1950
|
+
&& stream.tokens[index + 1]?.compound === true;
|
|
1951
|
+
const zeroMetric = token?.value === 'zero'
|
|
1952
|
+
&& isContextualZeroMetric(stream, index, stream.tokens.length)
|
|
1953
|
+
&& !(stream.tokens[index + 1]?.value === 'cost'
|
|
1954
|
+
&& ['center', 'centers'].includes(stream.tokens[index + 2]?.value ?? ''));
|
|
1955
|
+
return token !== undefined
|
|
1956
|
+
&& (lexicalNoOne || (!token.compound
|
|
1957
|
+
&& !zeroMetric
|
|
1958
|
+
&& (NEGATIVE_EVIDENCE_PRONOUNS.has(token.value) || NEGATIVE_EVIDENCE_DETERMINERS.has(token.value))));
|
|
1959
|
+
}
|
|
1960
|
+
function isContextualZeroMetric(stream, zeroIndex, end) {
|
|
1961
|
+
for (let index = zeroIndex + 1; index < end; index += 1) {
|
|
1962
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
1963
|
+
continue;
|
|
1964
|
+
return CONTEXTUAL_ZERO_METRIC_TERMS.has(stream.tokens[index]?.value ?? '');
|
|
1965
|
+
}
|
|
1966
|
+
return false;
|
|
1967
|
+
}
|
|
1968
|
+
function leadingSubjectPolarity(stream, start, end) {
|
|
1969
|
+
for (let index = start; index < end; index += 1) {
|
|
1970
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
1971
|
+
continue;
|
|
1972
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
1973
|
+
if (isNegativeRequiredToken(stream, index))
|
|
1974
|
+
return 'negative';
|
|
1975
|
+
if (SUBJECT_PRONOUN_TERMS.has(term) || SUBJECT_ARTICLE_TERMS.has(term))
|
|
1976
|
+
return 'positive';
|
|
1977
|
+
if (NON_SUBJECT_PREFIX_TERMS.has(term) || /ly$/u.test(term))
|
|
1978
|
+
continue;
|
|
1979
|
+
return 'positive';
|
|
1980
|
+
}
|
|
1981
|
+
return 'absent';
|
|
1982
|
+
}
|
|
1983
|
+
function isFinitePredicateTerm(term) {
|
|
1984
|
+
return KNOWN_FINITE_PREDICATE_TERMS.has(term) || /(?:ed|n't)$/u.test(term);
|
|
1985
|
+
}
|
|
1986
|
+
function hasFinitePredicatePrefix(stream, start, end) {
|
|
1987
|
+
const terms = [];
|
|
1988
|
+
for (let index = start; index < end; index += 1) {
|
|
1989
|
+
if (!isIgnoredRequiredToken(stream, index))
|
|
1990
|
+
terms.push(stream.tokens[index]?.value ?? '');
|
|
1991
|
+
}
|
|
1992
|
+
while (terms.length > 0 && (/ly$/u.test(terms[0] ?? '') || LEADING_CLAUSE_MODIFIER_TERMS.has(terms[0] ?? ''))) {
|
|
1993
|
+
terms.shift();
|
|
1994
|
+
}
|
|
1995
|
+
if (terms.length === 0)
|
|
1996
|
+
return false;
|
|
1997
|
+
const first = terms[0] ?? '';
|
|
1998
|
+
if (SUBJECT_PRONOUN_TERMS.has(first) || NEGATIVE_EVIDENCE_PRONOUNS.has(first)) {
|
|
1999
|
+
terms.shift();
|
|
2000
|
+
}
|
|
2001
|
+
else {
|
|
2002
|
+
if (SUBJECT_ARTICLE_TERMS.has(first) || NEGATIVE_EVIDENCE_DETERMINERS.has(first))
|
|
2003
|
+
terms.shift();
|
|
2004
|
+
while (terms.length > 1 && isAttributiveTargetModifier(terms[0] ?? ''))
|
|
2005
|
+
terms.shift();
|
|
2006
|
+
if (terms.length > 0)
|
|
2007
|
+
terms.shift();
|
|
2008
|
+
}
|
|
2009
|
+
return terms.some((term) => isFinitePredicateTerm(term));
|
|
2010
|
+
}
|
|
2011
|
+
function hasNegativePredicateMarker(stream, start, end) {
|
|
2012
|
+
for (let index = start; index < end; index += 1) {
|
|
2013
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2014
|
+
continue;
|
|
2015
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2016
|
+
if (term === 'not') {
|
|
2017
|
+
let next = index + 1;
|
|
2018
|
+
while (next < end && isIgnoredRequiredToken(stream, next))
|
|
2019
|
+
next += 1;
|
|
2020
|
+
if (stream.tokens[next]?.value === 'only')
|
|
2021
|
+
continue;
|
|
2022
|
+
}
|
|
2023
|
+
if (term === 'not' || term === 'never' || term === 'cannot' || /n't$/u.test(term))
|
|
2024
|
+
return true;
|
|
2025
|
+
}
|
|
2026
|
+
return false;
|
|
2027
|
+
}
|
|
2028
|
+
function hasFinitePredicateInSpan(stream, start, end) {
|
|
2029
|
+
for (let index = start; index < end; index += 1) {
|
|
2030
|
+
if (!isIgnoredRequiredToken(stream, index)
|
|
2031
|
+
&& isFinitePredicateTerm(stream.tokens[index]?.value ?? ''))
|
|
2032
|
+
return true;
|
|
2033
|
+
}
|
|
2034
|
+
return false;
|
|
2035
|
+
}
|
|
2036
|
+
function relativeSafeFinitePredicateEnd(stream, start, end) {
|
|
2037
|
+
for (let index = start; index < end; index += 1) {
|
|
2038
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2039
|
+
continue;
|
|
2040
|
+
const complements = RELATIVE_SAFE_FINITE_COMPLEMENTS.get(stream.tokens[index]?.value ?? '');
|
|
2041
|
+
if (!complements)
|
|
2042
|
+
return undefined;
|
|
2043
|
+
for (let complement = index + 1; complement < end; complement += 1) {
|
|
2044
|
+
if (isIgnoredRequiredToken(stream, complement))
|
|
2045
|
+
continue;
|
|
2046
|
+
return complements.has(stream.tokens[complement]?.value ?? '') ? complement + 1 : undefined;
|
|
2047
|
+
}
|
|
2048
|
+
return undefined;
|
|
2049
|
+
}
|
|
2050
|
+
return undefined;
|
|
2051
|
+
}
|
|
2052
|
+
function relativePredicateStart(stream, start, end) {
|
|
2053
|
+
let index = start;
|
|
2054
|
+
while (index < end) {
|
|
2055
|
+
if (isIgnoredRequiredToken(stream, index)) {
|
|
2056
|
+
index += 1;
|
|
2057
|
+
continue;
|
|
2058
|
+
}
|
|
2059
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2060
|
+
if (/ly$/u.test(term) || AFFIRMATIVE_PREDICATE_MODIFIER_TERMS.has(term)) {
|
|
2061
|
+
index += 1;
|
|
2062
|
+
continue;
|
|
2063
|
+
}
|
|
2064
|
+
break;
|
|
2065
|
+
}
|
|
2066
|
+
return index;
|
|
2067
|
+
}
|
|
2068
|
+
function hasBoundedRegularRelativePredicate(stream, predicateIndex, end) {
|
|
2069
|
+
const term = stream.tokens[predicateIndex]?.value ?? '';
|
|
2070
|
+
if (KNOWN_FINITE_PREDICATE_TERMS.has(term) || /n't$/u.test(term))
|
|
2071
|
+
return true;
|
|
2072
|
+
if (!term.endsWith('ed'))
|
|
2073
|
+
return false;
|
|
2074
|
+
let next = predicateIndex + 1;
|
|
2075
|
+
while (next < end && isIgnoredRequiredToken(stream, next))
|
|
2076
|
+
next += 1;
|
|
2077
|
+
if (next >= end)
|
|
2078
|
+
return true;
|
|
2079
|
+
const nextTerm = stream.tokens[next]?.value ?? '';
|
|
2080
|
+
if (BROAD_EVIDENCE_LINK_TERMS.has(nextTerm) && nextTerm !== 'that')
|
|
2081
|
+
return true;
|
|
2082
|
+
return false;
|
|
2083
|
+
}
|
|
2084
|
+
function hasBoundedRelativeFinitePredicate(stream, start, end) {
|
|
2085
|
+
if (hasFinitePredicatePrefix(stream, start, end))
|
|
2086
|
+
return true;
|
|
2087
|
+
const predicateIndex = relativePredicateStart(stream, start, end);
|
|
2088
|
+
if (predicateIndex >= end)
|
|
2089
|
+
return false;
|
|
2090
|
+
const term = stream.tokens[predicateIndex]?.value ?? '';
|
|
2091
|
+
if (EMBEDDED_CLAIM_GOVERNOR_TERMS.has(term))
|
|
2092
|
+
return false;
|
|
2093
|
+
if (isRequiredPassiveAuxiliary(term))
|
|
2094
|
+
return true;
|
|
2095
|
+
return relativeSafeFinitePredicateEnd(stream, predicateIndex, end) !== undefined
|
|
2096
|
+
|| hasBoundedRegularRelativePredicate(stream, predicateIndex, end);
|
|
2097
|
+
}
|
|
2098
|
+
function hasEmbeddedClaimGovernorInSpan(stream, start, end) {
|
|
2099
|
+
for (let index = start; index < end; index += 1) {
|
|
2100
|
+
if (!isIgnoredRequiredToken(stream, index)
|
|
2101
|
+
&& EMBEDDED_CLAIM_GOVERNOR_TERMS.has(stream.tokens[index]?.value ?? ''))
|
|
2102
|
+
return true;
|
|
2103
|
+
}
|
|
2104
|
+
return false;
|
|
2105
|
+
}
|
|
2106
|
+
function isExplicitNoDoubtContext(stream, start, end) {
|
|
2107
|
+
const values = [];
|
|
2108
|
+
for (let index = start; index < end; index += 1) {
|
|
2109
|
+
if (!isIgnoredRequiredToken(stream, index))
|
|
2110
|
+
values.push(stream.tokens[index]?.value ?? '');
|
|
2111
|
+
}
|
|
2112
|
+
if (values.shift() !== 'there' || !PASSIVE_BE_AUXILIARY_TERMS.has(values.shift() ?? ''))
|
|
2113
|
+
return false;
|
|
2114
|
+
while (values.length > 0 && /ly$/u.test(values[0] ?? ''))
|
|
2115
|
+
values.shift();
|
|
2116
|
+
return values.length === 2 && values[0] === 'no' && values[1] === 'doubt';
|
|
2117
|
+
}
|
|
2118
|
+
function complementSubjectScope(stream, initialStart, end) {
|
|
2119
|
+
let start = initialStart;
|
|
2120
|
+
let negativeGovernor = false;
|
|
2121
|
+
for (let index = start; index < end; index += 1) {
|
|
2122
|
+
if (isIgnoredRequiredToken(stream, index) || stream.tokens[index]?.value !== 'that')
|
|
2123
|
+
continue;
|
|
2124
|
+
if (!hasFinitePredicatePrefix(stream, start, index)) {
|
|
2125
|
+
// Ambiguous relative/factual subordinate evidence stays fail-closed.
|
|
2126
|
+
if (!hasBoundedRelativeFinitePredicate(stream, index + 1, end))
|
|
2127
|
+
negativeGovernor = true;
|
|
2128
|
+
continue;
|
|
2129
|
+
}
|
|
2130
|
+
if (!isExplicitNoDoubtContext(stream, start, index)
|
|
2131
|
+
|| leadingSubjectPolarity(stream, start, index) === 'negative'
|
|
2132
|
+
|| hasNegativePredicateMarker(stream, start, index))
|
|
2133
|
+
negativeGovernor = true;
|
|
2134
|
+
start = index + 1;
|
|
2135
|
+
}
|
|
2136
|
+
return { start, negativeGovernor };
|
|
2137
|
+
}
|
|
2138
|
+
function startsExplicitSubordinateClause(stream, start, verbIndex) {
|
|
2139
|
+
for (let index = start; index < verbIndex; index += 1) {
|
|
2140
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2141
|
+
continue;
|
|
2142
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2143
|
+
if (SUBJECT_PRONOUN_TERMS.has(term)
|
|
2144
|
+
|| SUBJECT_ARTICLE_TERMS.has(term)
|
|
2145
|
+
|| isNegativeRequiredToken(stream, index)
|
|
2146
|
+
|| isRequiredPassiveAuxiliary(term)
|
|
2147
|
+
|| INDEPENDENT_BROAD_FORM_TERMS.has(term))
|
|
2148
|
+
return true;
|
|
2149
|
+
}
|
|
2150
|
+
return false;
|
|
2151
|
+
}
|
|
2152
|
+
function predicateBoundaryBefore(stream, before) {
|
|
2153
|
+
const token = stream.tokens[before];
|
|
2154
|
+
if (!token)
|
|
2155
|
+
return { start: 0, inheritsSubject: false };
|
|
2156
|
+
const lowerBound = Math.max(0, before - MAX_REQUIRED_SUBJECT_LOOKBACK);
|
|
2157
|
+
for (let index = before - 1; index >= lowerBound; index -= 1) {
|
|
2158
|
+
const current = stream.tokens[index];
|
|
2159
|
+
if (!current || current.clause !== token.clause)
|
|
2160
|
+
return { start: index + 1, inheritsSubject: false };
|
|
2161
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2162
|
+
continue;
|
|
2163
|
+
if (hasEffectiveCommaBefore(stream, index + 1)) {
|
|
2164
|
+
return { start: index + 1, inheritsSubject: false };
|
|
2165
|
+
}
|
|
2166
|
+
if (PREDICATE_COORDINATOR_TERMS.has(current.value)) {
|
|
2167
|
+
return { start: index + 1, index, inheritsSubject: true };
|
|
2168
|
+
}
|
|
2169
|
+
if (current.value === 'after' || current.value === 'before') {
|
|
2170
|
+
let prefixStart = index;
|
|
2171
|
+
while (prefixStart > lowerBound
|
|
2172
|
+
&& stream.tokens[prefixStart - 1]?.clause === current.clause
|
|
2173
|
+
&& !hasEffectiveCommaBefore(stream, prefixStart))
|
|
2174
|
+
prefixStart -= 1;
|
|
2175
|
+
if (hasFinitePredicatePrefix(stream, prefixStart, index)
|
|
2176
|
+
&& startsExplicitSubordinateClause(stream, index + 1, before)) {
|
|
2177
|
+
return { start: index + 1, index, inheritsSubject: false };
|
|
2178
|
+
}
|
|
2179
|
+
}
|
|
2180
|
+
if (PREDICATE_SCOPE_RESET_TERMS.has(current.value)) {
|
|
2181
|
+
return {
|
|
2182
|
+
start: index + 1,
|
|
2183
|
+
index,
|
|
2184
|
+
inheritsSubject: false,
|
|
2185
|
+
negatesPredicate: current.value === 'without',
|
|
2186
|
+
};
|
|
2187
|
+
}
|
|
2188
|
+
}
|
|
2189
|
+
return { start: lowerBound, inheritsSubject: false };
|
|
2190
|
+
}
|
|
2191
|
+
function predicateSubjectPolarity(stream, boundary, verbIndex) {
|
|
2192
|
+
let cursorBoundary = boundary;
|
|
2193
|
+
let cursorEnd = verbIndex;
|
|
2194
|
+
while (true) {
|
|
2195
|
+
const subjectScope = complementSubjectScope(stream, cursorBoundary.start, cursorEnd);
|
|
2196
|
+
const polarity = subjectScope.negativeGovernor
|
|
2197
|
+
? 'negative'
|
|
2198
|
+
: leadingSubjectPolarity(stream, subjectScope.start, cursorEnd);
|
|
2199
|
+
if (polarity !== 'absent' || !cursorBoundary.inheritsSubject || cursorBoundary.index === undefined) {
|
|
2200
|
+
return polarity;
|
|
2201
|
+
}
|
|
2202
|
+
cursorEnd = cursorBoundary.index;
|
|
2203
|
+
cursorBoundary = predicateBoundaryBefore(stream, cursorEnd);
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
function predicateEndAfter(stream, verbIndex) {
|
|
2207
|
+
const token = stream.tokens[verbIndex];
|
|
2208
|
+
if (!token)
|
|
2209
|
+
return verbIndex + 1;
|
|
2210
|
+
const upperBound = Math.min(stream.tokens.length, verbIndex + MAX_REQUIRED_EVIDENCE_PAIR_DISTANCE + 1);
|
|
2211
|
+
for (let index = verbIndex + 1; index < upperBound; index += 1) {
|
|
2212
|
+
const current = stream.tokens[index];
|
|
2213
|
+
if (!current || current.clause !== token.clause || hasEffectiveCommaBefore(stream, index))
|
|
2214
|
+
return index;
|
|
2215
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2216
|
+
continue;
|
|
2217
|
+
if (PREDICATE_COORDINATOR_TERMS.has(current.value)
|
|
2218
|
+
|| PREDICATE_SCOPE_RESET_TERMS.has(current.value))
|
|
2219
|
+
return index;
|
|
2220
|
+
}
|
|
2221
|
+
return upperBound;
|
|
2222
|
+
}
|
|
2223
|
+
function predicateCandidates(stream, verbIndexes) {
|
|
2224
|
+
const candidates = [];
|
|
2225
|
+
const seenVerbIndexes = new Set();
|
|
2226
|
+
for (const verbIndex of verbIndexes) {
|
|
2227
|
+
if (candidates.length >= MAX_REQUIRED_EVIDENCE_CANDIDATES)
|
|
2228
|
+
break;
|
|
2229
|
+
if (seenVerbIndexes.has(verbIndex) || isIgnoredRequiredToken(stream, verbIndex))
|
|
2230
|
+
continue;
|
|
2231
|
+
seenVerbIndexes.add(verbIndex);
|
|
2232
|
+
const boundary = predicateBoundaryBefore(stream, verbIndex);
|
|
2233
|
+
const subjectScope = complementSubjectScope(stream, boundary.start, verbIndex);
|
|
2234
|
+
candidates.push({
|
|
2235
|
+
verbIndex,
|
|
2236
|
+
localStart: subjectScope.start,
|
|
2237
|
+
end: predicateEndAfter(stream, verbIndex),
|
|
2238
|
+
subjectNegative: predicateSubjectPolarity(stream, boundary, verbIndex) === 'negative',
|
|
2239
|
+
conditional: stream.conditional[verbIndex] === true,
|
|
2240
|
+
boundaryNegated: boundary.negatesPredicate === true,
|
|
2241
|
+
});
|
|
2242
|
+
}
|
|
2243
|
+
return candidates;
|
|
2244
|
+
}
|
|
2245
|
+
function nextRequiredToken(stream, index, end) {
|
|
2246
|
+
for (let next = index + 1; next <= end; next += 1) {
|
|
2247
|
+
if (!isIgnoredRequiredToken(stream, next))
|
|
2248
|
+
return stream.tokens[next]?.value;
|
|
2249
|
+
}
|
|
2250
|
+
return undefined;
|
|
2251
|
+
}
|
|
2252
|
+
function isLocalPredicateNegated(stream, candidate, through) {
|
|
2253
|
+
if (candidate.boundaryNegated)
|
|
2254
|
+
return true;
|
|
2255
|
+
for (let index = candidate.localStart; index <= through; index += 1) {
|
|
2256
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2257
|
+
continue;
|
|
2258
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2259
|
+
if (term === 'not' && nextRequiredToken(stream, index, through) === 'only')
|
|
2260
|
+
continue;
|
|
2261
|
+
if (term === 'not' || term === 'never' || term === 'without' || term === 'cannot' || /n't$/u.test(term)) {
|
|
2262
|
+
return true;
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
return false;
|
|
2266
|
+
}
|
|
2267
|
+
function tokenMatchIndexes(stream, term, kind) {
|
|
2268
|
+
const forms = targetTermForms(term, kind);
|
|
2269
|
+
const indexes = [];
|
|
2270
|
+
for (let index = 0; index < stream.tokens.length; index += 1) {
|
|
2271
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2272
|
+
continue;
|
|
2273
|
+
if (forms.has(stream.tokens[index]?.value ?? ''))
|
|
2274
|
+
indexes.push(index);
|
|
2275
|
+
}
|
|
2276
|
+
return indexes;
|
|
2277
|
+
}
|
|
2278
|
+
function isAttributiveTargetModifier(term) {
|
|
2279
|
+
return BROAD_EVIDENCE_LINK_TERMS.has(term)
|
|
2280
|
+
|| TOTALITY_TARGET_MODIFIER_TERMS.has(term)
|
|
2281
|
+
|| (term.length <= 3
|
|
2282
|
+
&& /^[a-z]+$/u.test(term)
|
|
2283
|
+
&& !KNOWN_EVIDENCE_ACTION_TERMS.has(term)
|
|
2284
|
+
&& !PASSIVE_SUBJECT_POSTMODIFIER_TERMS.has(term)
|
|
2285
|
+
&& !PREDICATE_COORDINATOR_TERMS.has(term)
|
|
2286
|
+
&& !PREDICATE_SCOPE_RESET_TERMS.has(term)
|
|
2287
|
+
&& !NEGATIVE_EVIDENCE_PRONOUNS.has(term)
|
|
2288
|
+
&& !NEGATIVE_EVIDENCE_DETERMINERS.has(term))
|
|
2289
|
+
|| /(?:ed|able|al|ary|ful|ible|ic|ive|less|ory|ous)$/u.test(term);
|
|
2290
|
+
}
|
|
2291
|
+
function hasNegativePrenominalTarget(stream, targetIndex, lowerBound) {
|
|
2292
|
+
let sawContextualZeroMetric = false;
|
|
2293
|
+
for (let index = targetIndex - 1; index >= lowerBound; index -= 1) {
|
|
2294
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2295
|
+
continue;
|
|
2296
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2297
|
+
if (hasEffectiveCommaBefore(stream, index + 1) || TARGET_SCOPE_RESET_TERMS_LOCAL.has(term))
|
|
2298
|
+
return false;
|
|
2299
|
+
if (isNegativeRequiredToken(stream, index))
|
|
2300
|
+
return term !== 'zero' || !sawContextualZeroMetric;
|
|
2301
|
+
if (CONTEXTUAL_ZERO_METRIC_TERMS.has(term)) {
|
|
2302
|
+
sawContextualZeroMetric = true;
|
|
2303
|
+
continue;
|
|
2304
|
+
}
|
|
2305
|
+
if (term === 'of' || /ly$/u.test(term) || isAttributiveTargetModifier(term))
|
|
2306
|
+
continue;
|
|
2307
|
+
if (KNOWN_EVIDENCE_ACTION_TERMS.has(term) || term.endsWith('ing'))
|
|
2308
|
+
return false;
|
|
2309
|
+
}
|
|
2310
|
+
return false;
|
|
2311
|
+
}
|
|
2312
|
+
function hasActiveTargetLinks(stream, verbIndex, targetIndex) {
|
|
2313
|
+
for (let index = verbIndex + 1; index < targetIndex; index += 1) {
|
|
2314
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2315
|
+
continue;
|
|
2316
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2317
|
+
if (hasEffectiveCommaBefore(stream, index)
|
|
2318
|
+
|| PREDICATE_COORDINATOR_TERMS.has(term)
|
|
2319
|
+
|| PREDICATE_SCOPE_RESET_TERMS.has(term)
|
|
2320
|
+
|| EMBEDDED_CLAIM_GOVERNOR_TERMS.has(term)
|
|
2321
|
+
|| !isAttributiveTargetModifier(term))
|
|
2322
|
+
return false;
|
|
2323
|
+
if (term === 'not' && nextRequiredToken(stream, index, targetIndex) !== 'only')
|
|
2324
|
+
return false;
|
|
2325
|
+
}
|
|
2326
|
+
return true;
|
|
2327
|
+
}
|
|
2328
|
+
function nearestPredicateChainTerm(stream, candidate, skipInfinitiveMarker) {
|
|
2329
|
+
for (let index = candidate.verbIndex - 1; index >= candidate.localStart; index -= 1) {
|
|
2330
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2331
|
+
continue;
|
|
2332
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2333
|
+
if (term === 'only') {
|
|
2334
|
+
let previous = index - 1;
|
|
2335
|
+
while (previous >= candidate.localStart && isIgnoredRequiredToken(stream, previous))
|
|
2336
|
+
previous -= 1;
|
|
2337
|
+
if (stream.tokens[previous]?.value === 'not') {
|
|
2338
|
+
index = previous;
|
|
2339
|
+
continue;
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
if (AFFIRMATIVE_PREDICATE_MODIFIER_TERMS.has(term))
|
|
2343
|
+
continue;
|
|
2344
|
+
if (skipInfinitiveMarker && term === 'to')
|
|
2345
|
+
continue;
|
|
2346
|
+
return term;
|
|
2347
|
+
}
|
|
2348
|
+
return undefined;
|
|
2349
|
+
}
|
|
2350
|
+
function hasBlockingRootPredicateModifier(stream, candidate) {
|
|
2351
|
+
for (let index = candidate.localStart; index < candidate.verbIndex; index += 1) {
|
|
2352
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2353
|
+
continue;
|
|
2354
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2355
|
+
if (isBlockingRootPredicateModifierTerm(term)
|
|
2356
|
+
&& !isSubjectParticipleAdverb(stream, candidate, index))
|
|
2357
|
+
return true;
|
|
2358
|
+
}
|
|
2359
|
+
return false;
|
|
2360
|
+
}
|
|
2361
|
+
function hasExplicitEmbeddedSubjectAfter(stream, start, end) {
|
|
2362
|
+
for (let index = start; index < end; index += 1) {
|
|
2363
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2364
|
+
continue;
|
|
2365
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2366
|
+
if (SUBJECT_PRONOUN_TERMS.has(term)
|
|
2367
|
+
|| SUBJECT_ARTICLE_TERMS.has(term)
|
|
2368
|
+
|| isNegativeRequiredToken(stream, index))
|
|
2369
|
+
return true;
|
|
2370
|
+
}
|
|
2371
|
+
return false;
|
|
2372
|
+
}
|
|
2373
|
+
function isCompletedRelativePredicate(stream, candidate, predicateIndex) {
|
|
2374
|
+
for (let index = predicateIndex - 1; index >= candidate.localStart; index -= 1) {
|
|
2375
|
+
if (isIgnoredRequiredToken(stream, index) || stream.tokens[index]?.value !== 'that')
|
|
2376
|
+
continue;
|
|
2377
|
+
return !hasFinitePredicatePrefix(stream, candidate.localStart, index);
|
|
2378
|
+
}
|
|
2379
|
+
return false;
|
|
2380
|
+
}
|
|
2381
|
+
function isParticipialSubjectModifier(stream, candidate, modifierIndex) {
|
|
2382
|
+
for (let index = candidate.localStart; index < modifierIndex; index += 1) {
|
|
2383
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2384
|
+
continue;
|
|
2385
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2386
|
+
const determiner = BROAD_EVIDENCE_LINK_TERMS.has(term) && term !== 'not' && term !== 'only';
|
|
2387
|
+
if (determiner
|
|
2388
|
+
|| /ly$/u.test(term)
|
|
2389
|
+
|| /(?:ed|ing)$/u.test(term)
|
|
2390
|
+
|| isAttributiveTargetModifier(term))
|
|
2391
|
+
continue;
|
|
2392
|
+
return false;
|
|
2393
|
+
}
|
|
2394
|
+
let sawHead = false;
|
|
2395
|
+
for (let index = modifierIndex + 1; index < candidate.verbIndex; index += 1) {
|
|
2396
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2397
|
+
continue;
|
|
2398
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2399
|
+
if (hasEffectiveCommaBefore(stream, index)
|
|
2400
|
+
|| PREDICATE_COORDINATOR_TERMS.has(term)
|
|
2401
|
+
|| PREDICATE_SCOPE_RESET_TERMS.has(term)
|
|
2402
|
+
|| BROAD_EVIDENCE_LINK_TERMS.has(term)
|
|
2403
|
+
|| SUBJECT_PRONOUN_TERMS.has(term)
|
|
2404
|
+
|| term === 'that'
|
|
2405
|
+
|| term === 'to')
|
|
2406
|
+
return false;
|
|
2407
|
+
if (isRequiredPassiveAuxiliary(term)) {
|
|
2408
|
+
if (!sawHead)
|
|
2409
|
+
return false;
|
|
2410
|
+
continue;
|
|
2411
|
+
}
|
|
2412
|
+
if (KNOWN_EVIDENCE_ACTION_TERMS.has(term)) {
|
|
2413
|
+
const baseActionNoun = !sawHead
|
|
2414
|
+
&& (IRREGULAR_VERB_FORMS.has(term) || GENERIC_ACTION_TERMS.has(term))
|
|
2415
|
+
&& !EXACT_ONLY_PROGRESSIVE_LEMMAS.has(term);
|
|
2416
|
+
if (!baseActionNoun)
|
|
2417
|
+
return false;
|
|
2418
|
+
sawHead = true;
|
|
2419
|
+
continue;
|
|
2420
|
+
}
|
|
2421
|
+
if (/ly$/u.test(term))
|
|
2422
|
+
continue;
|
|
2423
|
+
if (/(?:ed|ing)$/u.test(term)) {
|
|
2424
|
+
if (sawHead)
|
|
2425
|
+
return false;
|
|
2426
|
+
continue;
|
|
2427
|
+
}
|
|
2428
|
+
sawHead = true;
|
|
2429
|
+
}
|
|
2430
|
+
return sawHead;
|
|
2431
|
+
}
|
|
2432
|
+
function isRelativeNominalClaimHead(stream, candidate, claimIndex) {
|
|
2433
|
+
if (!NOMINAL_CLAIM_HEAD_TERMS.has(stream.tokens[claimIndex]?.value ?? ''))
|
|
2434
|
+
return false;
|
|
2435
|
+
for (let index = candidate.localStart; index < claimIndex; index += 1) {
|
|
2436
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2437
|
+
continue;
|
|
2438
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2439
|
+
const determiner = BROAD_EVIDENCE_LINK_TERMS.has(term) && term !== 'not' && term !== 'only';
|
|
2440
|
+
if (determiner || isNegativeRequiredToken(stream, index) || /ly$/u.test(term)
|
|
2441
|
+
|| isAttributiveTargetModifier(term))
|
|
2442
|
+
continue;
|
|
2443
|
+
return false;
|
|
2444
|
+
}
|
|
2445
|
+
let relativeStart = claimIndex + 1;
|
|
2446
|
+
while (relativeStart < candidate.verbIndex && isIgnoredRequiredToken(stream, relativeStart)) {
|
|
2447
|
+
relativeStart += 1;
|
|
2448
|
+
}
|
|
2449
|
+
if (stream.tokens[relativeStart]?.value !== 'that')
|
|
2450
|
+
return false;
|
|
2451
|
+
const relativePredicate = relativePredicateStart(stream, relativeStart + 1, candidate.verbIndex);
|
|
2452
|
+
if (relativePredicate >= candidate.verbIndex)
|
|
2453
|
+
return false;
|
|
2454
|
+
const firstRelativeValue = stream.tokens[relativePredicate]?.value ?? '';
|
|
2455
|
+
if (EMBEDDED_CLAIM_GOVERNOR_TERMS.has(firstRelativeValue))
|
|
2456
|
+
return false;
|
|
2457
|
+
const copularRelative = isRequiredPassiveAuxiliary(firstRelativeValue);
|
|
2458
|
+
if (!copularRelative) {
|
|
2459
|
+
const safeEnd = relativeSafeFinitePredicateEnd(stream, relativePredicate, candidate.verbIndex);
|
|
2460
|
+
if (safeEnd !== undefined) {
|
|
2461
|
+
return !hasEmbeddedClaimGovernorInSpan(stream, safeEnd, candidate.verbIndex);
|
|
2462
|
+
}
|
|
2463
|
+
return hasBoundedRegularRelativePredicate(stream, relativePredicate, candidate.verbIndex);
|
|
2464
|
+
}
|
|
2465
|
+
for (let index = relativePredicate; index < candidate.verbIndex; index += 1) {
|
|
2466
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2467
|
+
continue;
|
|
2468
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2469
|
+
if (isRequiredPassiveAuxiliary(term)
|
|
2470
|
+
|| AFFIRMATIVE_PREDICATE_MODIFIER_TERMS.has(term)
|
|
2471
|
+
|| term === 'not'
|
|
2472
|
+
|| term === 'never')
|
|
2473
|
+
continue;
|
|
2474
|
+
if (EMBEDDED_CLAIM_GOVERNOR_TERMS.has(term))
|
|
2475
|
+
return false;
|
|
2476
|
+
return true;
|
|
2477
|
+
}
|
|
2478
|
+
return false;
|
|
2479
|
+
}
|
|
2480
|
+
function isSubjectParticipleAdverb(stream, candidate, adverbIndex) {
|
|
2481
|
+
let modifierIndex = adverbIndex + 1;
|
|
2482
|
+
while (modifierIndex < candidate.verbIndex && isIgnoredRequiredToken(stream, modifierIndex)) {
|
|
2483
|
+
modifierIndex += 1;
|
|
2484
|
+
}
|
|
2485
|
+
return /(?:ed|ing)$/u.test(stream.tokens[modifierIndex]?.value ?? '')
|
|
2486
|
+
&& isParticipialSubjectModifier(stream, candidate, modifierIndex);
|
|
2487
|
+
}
|
|
2488
|
+
function hasEmbeddedClaimGovernor(stream, candidate) {
|
|
2489
|
+
for (let index = candidate.localStart; index < candidate.verbIndex; index += 1) {
|
|
2490
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2491
|
+
continue;
|
|
2492
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2493
|
+
if (!EMBEDDED_CLAIM_GOVERNOR_TERMS.has(term))
|
|
2494
|
+
continue;
|
|
2495
|
+
if (isRelativeNominalClaimHead(stream, candidate, index))
|
|
2496
|
+
continue;
|
|
2497
|
+
if (/(?:ed|ing)$/u.test(term) && isParticipialSubjectModifier(stream, candidate, index))
|
|
2498
|
+
continue;
|
|
2499
|
+
if (hasExplicitEmbeddedSubjectAfter(stream, index + 1, candidate.verbIndex)
|
|
2500
|
+
|| hasFinitePredicateInSpan(stream, index + 1, candidate.verbIndex))
|
|
2501
|
+
return true;
|
|
2502
|
+
}
|
|
2503
|
+
return false;
|
|
2504
|
+
}
|
|
2505
|
+
function hasPriorNonAuxiliaryGovernor(stream, candidate) {
|
|
2506
|
+
if (hasEmbeddedClaimGovernor(stream, candidate))
|
|
2507
|
+
return true;
|
|
2508
|
+
for (let index = candidate.localStart; index < candidate.verbIndex; index += 1) {
|
|
2509
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2510
|
+
continue;
|
|
2511
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2512
|
+
if (isRequiredPassiveAuxiliary(term)
|
|
2513
|
+
|| AFFIRMATIVE_PREDICATE_MODIFIER_TERMS.has(term)
|
|
2514
|
+
|| NON_SUBJECT_PREFIX_TERMS.has(term))
|
|
2515
|
+
continue;
|
|
2516
|
+
if (isRelativeNominalClaimHead(stream, candidate, index))
|
|
2517
|
+
continue;
|
|
2518
|
+
if (UNAMBIGUOUS_EMBEDDED_CLAIM_GOVERNOR_TERMS.has(term))
|
|
2519
|
+
return true;
|
|
2520
|
+
if (/(?:ed|ing)$/u.test(term)) {
|
|
2521
|
+
if (isParticipialSubjectModifier(stream, candidate, index))
|
|
2522
|
+
continue;
|
|
2523
|
+
if (isCompletedRelativePredicate(stream, candidate, index))
|
|
2524
|
+
continue;
|
|
2525
|
+
return true;
|
|
2526
|
+
}
|
|
2527
|
+
if (term.length > 3
|
|
2528
|
+
&& term.endsWith('s')
|
|
2529
|
+
&& hasExplicitEmbeddedSubjectAfter(stream, index + 1, candidate.verbIndex))
|
|
2530
|
+
return true;
|
|
2531
|
+
}
|
|
2532
|
+
return false;
|
|
2533
|
+
}
|
|
2534
|
+
function hasFiniteActivePredicate(stream, candidate) {
|
|
2535
|
+
const verb = stream.tokens[candidate.verbIndex]?.value ?? '';
|
|
2536
|
+
if (hasBlockingRootPredicateModifier(stream, candidate)
|
|
2537
|
+
|| hasPriorNonAuxiliaryGovernor(stream, candidate))
|
|
2538
|
+
return false;
|
|
2539
|
+
if (INDEPENDENT_BROAD_FORM_TERMS.has(verb))
|
|
2540
|
+
return true;
|
|
2541
|
+
if (verb.endsWith('ing')) {
|
|
2542
|
+
if (candidate.verbIndex === candidate.localStart)
|
|
2543
|
+
return true;
|
|
2544
|
+
const license = nearestPredicateChainTerm(stream, candidate, false);
|
|
2545
|
+
return license !== undefined
|
|
2546
|
+
&& (PASSIVE_BE_AUXILIARY_TERMS.has(license)
|
|
2547
|
+
|| MODAL_AUXILIARY_TERMS.has(license)
|
|
2548
|
+
|| SEMI_MODAL_AUXILIARY_TERMS.has(license));
|
|
2549
|
+
}
|
|
2550
|
+
const license = nearestPredicateChainTerm(stream, candidate, true);
|
|
2551
|
+
if (PERFECT_BROAD_FORM_TERMS.has(verb)) {
|
|
2552
|
+
return license !== undefined && PERFECT_AUXILIARY_TERMS.has(license);
|
|
2553
|
+
}
|
|
2554
|
+
return license !== undefined
|
|
2555
|
+
&& (MODAL_AUXILIARY_TERMS.has(license) || SEMI_MODAL_AUXILIARY_TERMS.has(license));
|
|
2556
|
+
}
|
|
2557
|
+
function isRequiredPassiveAuxiliary(term) {
|
|
2558
|
+
return PASSIVE_AUXILIARY_TERMS.has(term)
|
|
2559
|
+
|| MODAL_AUXILIARY_TERMS.has(term)
|
|
2560
|
+
|| SEMI_MODAL_AUXILIARY_TERMS.has(term);
|
|
2561
|
+
}
|
|
2562
|
+
function hasPassiveTargetLinks(stream, targetIndex, verbIndex) {
|
|
2563
|
+
const links = [];
|
|
2564
|
+
for (let index = targetIndex + 1; index < verbIndex; index += 1) {
|
|
2565
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2566
|
+
continue;
|
|
2567
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2568
|
+
if (hasEffectiveCommaBefore(stream, index)
|
|
2569
|
+
|| PREDICATE_COORDINATOR_TERMS.has(term)
|
|
2570
|
+
|| PREDICATE_SCOPE_RESET_TERMS.has(term))
|
|
2571
|
+
return false;
|
|
2572
|
+
links.push(term);
|
|
2573
|
+
}
|
|
2574
|
+
const firstAuxiliary = links.findIndex((term) => isRequiredPassiveAuxiliary(term));
|
|
2575
|
+
if (firstAuxiliary < 0)
|
|
2576
|
+
return false;
|
|
2577
|
+
const postmodifier = links.slice(0, firstAuxiliary);
|
|
2578
|
+
if (postmodifier.length > 0
|
|
2579
|
+
&& !PASSIVE_SUBJECT_POSTMODIFIER_TERMS.has(postmodifier[0] ?? ''))
|
|
2580
|
+
return false;
|
|
2581
|
+
if (postmodifier.some((term) => NEGATIVE_EVIDENCE_PRONOUNS.has(term)
|
|
2582
|
+
|| NEGATIVE_EVIDENCE_DETERMINERS.has(term)))
|
|
2583
|
+
return false;
|
|
2584
|
+
const auxiliaryChain = links.slice(firstAuxiliary);
|
|
2585
|
+
if (!auxiliaryChain.some((term) => PASSIVE_BE_AUXILIARY_TERMS.has(term)))
|
|
2586
|
+
return false;
|
|
2587
|
+
return auxiliaryChain.every((term, index) => ((term !== 'not' || auxiliaryChain[index + 1] === 'only')
|
|
2588
|
+
&& (isRequiredPassiveAuxiliary(term)
|
|
2589
|
+
|| term === 'to'
|
|
2590
|
+
|| BROAD_EVIDENCE_LINK_TERMS.has(term)
|
|
2591
|
+
|| AFFIRMATIVE_PREDICATE_MODIFIER_TERMS.has(term))));
|
|
2592
|
+
}
|
|
2593
|
+
function nearestTargetAfter(candidate, indexes) {
|
|
2594
|
+
return indexes.find((index) => index > candidate.verbIndex
|
|
2595
|
+
&& index < candidate.end
|
|
2596
|
+
&& index - candidate.verbIndex <= MAX_REQUIRED_EVIDENCE_PAIR_DISTANCE);
|
|
2597
|
+
}
|
|
2598
|
+
function nearestTargetBefore(candidate, indexes) {
|
|
2599
|
+
for (let offset = indexes.length - 1; offset >= 0; offset -= 1) {
|
|
2600
|
+
const index = indexes[offset];
|
|
2601
|
+
if (index < candidate.localStart)
|
|
2602
|
+
return undefined;
|
|
2603
|
+
if (index < candidate.verbIndex
|
|
2604
|
+
&& candidate.verbIndex - index <= MAX_REQUIRED_EVIDENCE_PAIR_DISTANCE)
|
|
2605
|
+
return index;
|
|
2606
|
+
}
|
|
2607
|
+
return undefined;
|
|
2608
|
+
}
|
|
2609
|
+
function hasNegativePredicateTail(stream, start, end) {
|
|
2610
|
+
for (let index = start; index < end; index += 1) {
|
|
2611
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2612
|
+
continue;
|
|
2613
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2614
|
+
if (isNegativeRequiredToken(stream, index))
|
|
2615
|
+
return true;
|
|
2616
|
+
if (term === 'not' && nextRequiredToken(stream, index, end - 1) === 'only')
|
|
2617
|
+
continue;
|
|
2618
|
+
if (term === 'not' || term === 'never' || term === 'cannot' || /n't$/u.test(term))
|
|
2619
|
+
return true;
|
|
2620
|
+
}
|
|
2621
|
+
return false;
|
|
2622
|
+
}
|
|
2623
|
+
function hasLaterFinitePredicate(stream, start, candidate) {
|
|
2624
|
+
return hasFinitePredicateInSpan(stream, start, candidate.end);
|
|
2625
|
+
}
|
|
2626
|
+
function hasNonIgnoredHardClauseTail(stream, start, clause) {
|
|
2627
|
+
for (let index = start; index < stream.tokens.length; index += 1) {
|
|
2628
|
+
const token = stream.tokens[index];
|
|
2629
|
+
if (!token || token.clause !== clause)
|
|
2630
|
+
break;
|
|
2631
|
+
if (!isIgnoredRequiredToken(stream, index))
|
|
2632
|
+
return true;
|
|
2633
|
+
}
|
|
2634
|
+
return false;
|
|
2635
|
+
}
|
|
2636
|
+
function standaloneGerundTailStart(stream, candidate) {
|
|
2637
|
+
let start = candidate.verbIndex + 1;
|
|
2638
|
+
while (start < stream.tokens.length && isIgnoredRequiredToken(stream, start))
|
|
2639
|
+
start += 1;
|
|
2640
|
+
if (stream.tokens[start]?.value === 'it')
|
|
2641
|
+
start += 1;
|
|
2642
|
+
return start;
|
|
2643
|
+
}
|
|
2644
|
+
function isPassiveTargetOwnedByPriorNominal(stream, targetIndex, lowerBound) {
|
|
2645
|
+
let index = targetIndex - 1;
|
|
2646
|
+
while (index >= lowerBound) {
|
|
2647
|
+
if (isIgnoredRequiredToken(stream, index)) {
|
|
2648
|
+
index -= 1;
|
|
2649
|
+
continue;
|
|
2650
|
+
}
|
|
2651
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2652
|
+
if (isAttributiveTargetModifier(term) || /ly$/u.test(term)) {
|
|
2653
|
+
index -= 1;
|
|
2654
|
+
continue;
|
|
2655
|
+
}
|
|
2656
|
+
const contextualMetric = CONTEXTUAL_ZERO_METRIC_TERMS.has(term);
|
|
2657
|
+
for (let ownerIndex = index - 1; ownerIndex >= lowerBound; ownerIndex -= 1) {
|
|
2658
|
+
if (isIgnoredRequiredToken(stream, ownerIndex))
|
|
2659
|
+
continue;
|
|
2660
|
+
const owner = stream.tokens[ownerIndex]?.value ?? '';
|
|
2661
|
+
if (contextualMetric && owner === 'zero')
|
|
2662
|
+
return false;
|
|
2663
|
+
if (SUBJECT_ARTICLE_TERMS.has(owner)
|
|
2664
|
+
|| isNegativeRequiredToken(stream, ownerIndex)
|
|
2665
|
+
|| isAttributiveTargetModifier(owner)
|
|
2666
|
+
|| /ly$/u.test(owner))
|
|
2667
|
+
continue;
|
|
2668
|
+
return true;
|
|
2669
|
+
}
|
|
2670
|
+
return false;
|
|
2671
|
+
}
|
|
2672
|
+
return false;
|
|
2673
|
+
}
|
|
2674
|
+
function hasContextualZeroMetricTarget(stream, targetIndex, lowerBound) {
|
|
2675
|
+
for (let index = targetIndex - 1; index >= lowerBound; index -= 1) {
|
|
2676
|
+
if (isIgnoredRequiredToken(stream, index))
|
|
2677
|
+
continue;
|
|
2678
|
+
const term = stream.tokens[index]?.value ?? '';
|
|
2679
|
+
if (hasEffectiveCommaBefore(stream, index + 1) || TARGET_SCOPE_RESET_TERMS_LOCAL.has(term))
|
|
2680
|
+
return false;
|
|
2681
|
+
if (term === 'zero' && isNegativeRequiredToken(stream, index)) {
|
|
2682
|
+
return isContextualZeroMetric(stream, index, targetIndex);
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
return false;
|
|
2686
|
+
}
|
|
2687
|
+
function provesStandaloneBroadEvidence(stream, candidate) {
|
|
2688
|
+
const verb = stream.tokens[candidate.verbIndex]?.value ?? '';
|
|
2689
|
+
const rootGerund = verb.endsWith('ing') && candidate.verbIndex === candidate.localStart;
|
|
2690
|
+
const invalidTail = rootGerund
|
|
2691
|
+
? hasNonIgnoredHardClauseTail(stream, standaloneGerundTailStart(stream, candidate), stream.tokens[candidate.verbIndex]?.clause ?? -1)
|
|
2692
|
+
: hasLaterFinitePredicate(stream, candidate.verbIndex + 1, candidate);
|
|
2693
|
+
return !candidate.conditional
|
|
2694
|
+
&& !candidate.subjectNegative
|
|
2695
|
+
&& hasFiniteActivePredicate(stream, candidate)
|
|
2696
|
+
&& !isLocalPredicateNegated(stream, candidate, candidate.verbIndex)
|
|
2697
|
+
&& !invalidTail
|
|
2698
|
+
&& !hasNegativePredicateTail(stream, candidate.verbIndex + 1, candidate.end);
|
|
2699
|
+
}
|
|
2700
|
+
function provesRoleAwareBroadEvidence(stream, lemma, verbIndexes, evidenceIndexes) {
|
|
2701
|
+
let pairCount = 0;
|
|
2702
|
+
for (const candidate of predicateCandidates(stream, verbIndexes)) {
|
|
2703
|
+
if (candidate.conditional)
|
|
2704
|
+
continue;
|
|
2705
|
+
for (const indexes of evidenceIndexes) {
|
|
2706
|
+
if (pairCount >= MAX_REQUIRED_EVIDENCE_CANDIDATES)
|
|
2707
|
+
return false;
|
|
2708
|
+
pairCount += 1;
|
|
2709
|
+
const activeTarget = nearestTargetAfter(candidate, indexes);
|
|
2710
|
+
const activeVerb = stream.tokens[candidate.verbIndex]?.value ?? '';
|
|
2711
|
+
const rootGerund = activeVerb.endsWith('ing') && candidate.verbIndex === candidate.localStart;
|
|
2712
|
+
if (activeTarget !== undefined
|
|
2713
|
+
&& !candidate.subjectNegative
|
|
2714
|
+
&& hasFiniteActivePredicate(stream, candidate)
|
|
2715
|
+
&& !isLocalPredicateNegated(stream, candidate, activeTarget)
|
|
2716
|
+
&& !hasNegativePrenominalTarget(stream, activeTarget, candidate.verbIndex + 1)
|
|
2717
|
+
&& hasActiveTargetLinks(stream, candidate.verbIndex, activeTarget)
|
|
2718
|
+
&& !(rootGerund && hasNonIgnoredHardClauseTail(stream, activeTarget + 1, stream.tokens[candidate.verbIndex]?.clause ?? -1))
|
|
2719
|
+
&& !(!rootGerund
|
|
2720
|
+
&& activeVerb.endsWith('ing')
|
|
2721
|
+
&& hasLaterFinitePredicate(stream, activeTarget + 1, candidate))
|
|
2722
|
+
&& !hasNegativePredicateTail(stream, activeTarget + 1, candidate.end))
|
|
2723
|
+
return true;
|
|
2724
|
+
if (lemma !== 'do' || stream.tokens[candidate.verbIndex]?.value !== 'done')
|
|
2725
|
+
continue;
|
|
2726
|
+
const passiveTarget = nearestTargetBefore(candidate, indexes);
|
|
2727
|
+
if (passiveTarget !== undefined
|
|
2728
|
+
&& (!candidate.subjectNegative
|
|
2729
|
+
|| hasContextualZeroMetricTarget(stream, passiveTarget, candidate.localStart))
|
|
2730
|
+
&& !hasEmbeddedClaimGovernor(stream, candidate)
|
|
2731
|
+
&& !isLocalPredicateNegated(stream, candidate, candidate.verbIndex)
|
|
2732
|
+
&& !hasNegativePrenominalTarget(stream, passiveTarget, candidate.localStart)
|
|
2733
|
+
&& !isPassiveTargetOwnedByPriorNominal(stream, passiveTarget, candidate.localStart)
|
|
2734
|
+
&& !hasNegativePredicateTail(stream, candidate.verbIndex + 1, candidate.end)
|
|
2735
|
+
&& hasPassiveTargetLinks(stream, passiveTarget, candidate.verbIndex))
|
|
2736
|
+
return true;
|
|
2737
|
+
}
|
|
2738
|
+
}
|
|
2739
|
+
return false;
|
|
2740
|
+
}
|
|
2741
|
+
function mustMatchedTerms(stream, terms) {
|
|
2742
|
+
const indexedTerms = terms.map((term) => ({
|
|
2743
|
+
term,
|
|
2744
|
+
indexes: tokenMatchIndexes(stream, term, 'must'),
|
|
2745
|
+
}));
|
|
2746
|
+
const matched = new Set(indexedTerms
|
|
2747
|
+
.filter((entry) => !isBroadProgressiveAuxiliary(entry.term) && entry.indexes.length > 0)
|
|
2748
|
+
.map((entry) => entry.term.text));
|
|
2749
|
+
for (const entry of indexedTerms) {
|
|
2750
|
+
if (!isBroadProgressiveAuxiliary(entry.term) || entry.indexes.length === 0)
|
|
2751
|
+
continue;
|
|
2752
|
+
if (terms.length === 1) {
|
|
2753
|
+
const exactIndexes = entry.indexes.filter((index) => stream.tokens[index]?.value === entry.term.text);
|
|
2754
|
+
if (predicateCandidates(stream, exactIndexes).some((candidate) => (provesStandaloneBroadEvidence(stream, candidate))))
|
|
2755
|
+
matched.add(entry.term.text);
|
|
2756
|
+
continue;
|
|
2757
|
+
}
|
|
2758
|
+
const lemma = verbLemma(entry.term.text);
|
|
2759
|
+
if (lemma && provesRoleAwareBroadEvidence(stream, lemma, entry.indexes, indexedTerms.filter((other) => other !== entry).map((other) => other.indexes)))
|
|
2760
|
+
matched.add(entry.term.text);
|
|
2761
|
+
}
|
|
2762
|
+
return terms.filter((term) => matched.has(term.text)).map((term) => term.text);
|
|
2763
|
+
}
|
|
2764
|
+
export function detectConstraintViolations(output, constraints) {
|
|
2765
|
+
const redactedOutput = redactConstraintText(output);
|
|
2766
|
+
const evidenceHash = sha256(redactedOutput);
|
|
2767
|
+
const outputSensitiveClasses = new Set(sensitiveClassesForValue(output));
|
|
2768
|
+
const violations = [];
|
|
2769
|
+
let requiredEvidence;
|
|
2770
|
+
for (const constraint of constraints) {
|
|
2771
|
+
const terms = targetTerms(constraint.redactedText, constraint.kind);
|
|
2772
|
+
const sensitiveMatch = constraint.kind === 'must_not'
|
|
2773
|
+
&& constraint.sensitiveClasses.some((sensitiveClass) => outputSensitiveClasses.has(sensitiveClass));
|
|
2774
|
+
if (terms.length === 0 && !sensitiveMatch)
|
|
2775
|
+
continue;
|
|
2776
|
+
const matchedTerms = constraint.kind === 'must_not'
|
|
2777
|
+
? mustNotMatchedTerms(redactedOutput, terms)
|
|
2778
|
+
: mustMatchedTerms(requiredEvidence ??= lexRequiredEvidence(redactedOutput), terms);
|
|
2779
|
+
if (!sensitiveMatch && !isViolated(constraint.kind, terms, matchedTerms, isContrastiveConstraint(constraint.redactedText)))
|
|
2780
|
+
continue;
|
|
2781
|
+
violations.push({
|
|
2782
|
+
constraintId: constraint.id,
|
|
2783
|
+
kind: constraint.kind,
|
|
2784
|
+
severity: severityFor(constraint.kind),
|
|
2785
|
+
evidenceHash,
|
|
2786
|
+
matchedTerms: sensitiveMatch && matchedTerms.length === 0 ? ['sensitive_value'] : matchedTerms,
|
|
2787
|
+
});
|
|
2788
|
+
}
|
|
2789
|
+
return violations;
|
|
2790
|
+
}
|
|
2791
|
+
export function computeConstraintAblationScore(input) {
|
|
2792
|
+
const ablationCount = Math.max(0, input.ablationCount);
|
|
2793
|
+
const baselineViolationCount = input.baselineViolations.length;
|
|
2794
|
+
const ablatedViolationCount = input.ablatedViolations.length;
|
|
2795
|
+
const removedConstraintIds = new Set(input.removedConstraintIds);
|
|
2796
|
+
const relevantBaselineViolations = input.baselineViolations.filter((violation) => removedConstraintIds.has(violation.constraintId));
|
|
2797
|
+
const relevantAblatedViolations = input.ablatedViolations.filter((violation) => removedConstraintIds.has(violation.constraintId));
|
|
2798
|
+
const denominator = Math.max(1, removedConstraintIds.size);
|
|
2799
|
+
const sensitivity = Math.max(0, Math.min(1, (relevantAblatedViolations.length - relevantBaselineViolations.length) / denominator));
|
|
2800
|
+
const threshold = input.sensitivityThreshold ?? 0.5;
|
|
2801
|
+
const sensitive = sensitivity >= threshold;
|
|
2802
|
+
return {
|
|
2803
|
+
source: 'constraint_ablation_replay',
|
|
2804
|
+
sensitivity,
|
|
2805
|
+
ablationCount,
|
|
2806
|
+
baselineViolationCount,
|
|
2807
|
+
ablatedViolationCount,
|
|
2808
|
+
mustViolationCount: relevantAblatedViolations.filter((v) => v.kind === 'must').length,
|
|
2809
|
+
mustNotViolationCount: relevantAblatedViolations.filter((v) => v.kind === 'must_not').length,
|
|
2810
|
+
taskSuccess: input.taskSuccess,
|
|
2811
|
+
comparison: compareSensitivityWithSuccess(input.taskSuccess.status, sensitive),
|
|
2812
|
+
};
|
|
2813
|
+
}
|
|
2814
|
+
function compareSensitivityWithSuccess(status, sensitive) {
|
|
2815
|
+
if (status === 'unknown')
|
|
2816
|
+
return 'unknown_success';
|
|
2817
|
+
if (status === 'success')
|
|
2818
|
+
return sensitive ? 'success_constraint_sensitive' : 'success_constraint_insensitive';
|
|
2819
|
+
return sensitive ? 'failure_constraint_sensitive' : 'failure_constraint_insensitive';
|
|
2820
|
+
}
|