@cejel/cejel 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -6
- package/dist/{chunk-SFEPABAX.js → chunk-DVVBWBAA.js} +1602 -205
- package/dist/index.js +3580 -23
- package/dist/mcp/index.js +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import {
|
|
3
3
|
WitanReportSchema,
|
|
4
4
|
createWitanAttestation,
|
|
5
|
+
external_exports,
|
|
5
6
|
formatExternalSourceLine,
|
|
6
7
|
renderWitanBadgeEndpoint,
|
|
7
8
|
renderWitanBadgeSvg,
|
|
@@ -9,14 +10,3476 @@ import {
|
|
|
9
10
|
runCejelScan,
|
|
10
11
|
serializeWitanReport,
|
|
11
12
|
verifyWitanAttestationBinding
|
|
12
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-DVVBWBAA.js";
|
|
13
14
|
|
|
14
15
|
// src/index.ts
|
|
15
|
-
import { createHash } from "crypto";
|
|
16
|
-
import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "fs";
|
|
17
|
-
import { join, resolve } from "path";
|
|
16
|
+
import { createHash as createHash3 } from "crypto";
|
|
17
|
+
import { mkdirSync, readFileSync as readFileSync2, realpathSync, writeFileSync } from "fs";
|
|
18
|
+
import { join as join2, resolve as resolve3 } from "path";
|
|
18
19
|
import { pathToFileURL } from "url";
|
|
19
20
|
|
|
21
|
+
// src/packs/llm/detector.ts
|
|
22
|
+
import { createHash } from "crypto";
|
|
23
|
+
import { lstatSync as lstatSync2, readFileSync } from "fs";
|
|
24
|
+
import { extname, resolve as resolve2, sep as sep2 } from "path";
|
|
25
|
+
|
|
26
|
+
// src/packs/llm/lexical.ts
|
|
27
|
+
function maskJavaScriptNonCode(contents) {
|
|
28
|
+
const chars = [...contents];
|
|
29
|
+
let quote = null;
|
|
30
|
+
let escaped2 = false;
|
|
31
|
+
let lineComment = false;
|
|
32
|
+
let blockComment = false;
|
|
33
|
+
for (let index = 0; index < chars.length; index += 1) {
|
|
34
|
+
const character = chars[index] ?? "";
|
|
35
|
+
const next = chars[index + 1] ?? "";
|
|
36
|
+
if (lineComment) {
|
|
37
|
+
if (character === "\n") lineComment = false;
|
|
38
|
+
else chars[index] = " ";
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (blockComment) {
|
|
42
|
+
if (character === "*" && next === "/") {
|
|
43
|
+
chars[index] = chars[index + 1] = " ";
|
|
44
|
+
blockComment = false;
|
|
45
|
+
index += 1;
|
|
46
|
+
} else if (character !== "\n") chars[index] = " ";
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
if (quote) {
|
|
50
|
+
if (character !== "\n") chars[index] = " ";
|
|
51
|
+
if (escaped2) escaped2 = false;
|
|
52
|
+
else if (character === "\\") escaped2 = true;
|
|
53
|
+
else if (character === quote) quote = null;
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
if (character === "/" && next === "/") {
|
|
57
|
+
chars[index] = chars[index + 1] = " ";
|
|
58
|
+
lineComment = true;
|
|
59
|
+
index += 1;
|
|
60
|
+
} else if (character === "/" && next === "*") {
|
|
61
|
+
chars[index] = chars[index + 1] = " ";
|
|
62
|
+
blockComment = true;
|
|
63
|
+
index += 1;
|
|
64
|
+
} else if (character === "'" || character === '"' || character === "`") {
|
|
65
|
+
chars[index] = " ";
|
|
66
|
+
quote = character;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return chars.join("");
|
|
70
|
+
}
|
|
71
|
+
function maskPythonNonCode(contents) {
|
|
72
|
+
const chars = [...contents];
|
|
73
|
+
let quote = null;
|
|
74
|
+
let triple = false;
|
|
75
|
+
let escaped2 = false;
|
|
76
|
+
let comment = false;
|
|
77
|
+
for (let index = 0; index < chars.length; index += 1) {
|
|
78
|
+
const character = chars[index] ?? "";
|
|
79
|
+
if (comment) {
|
|
80
|
+
if (character === "\n") comment = false;
|
|
81
|
+
else chars[index] = " ";
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
if (quote) {
|
|
85
|
+
if (character !== "\n") chars[index] = " ";
|
|
86
|
+
if (escaped2) escaped2 = false;
|
|
87
|
+
else if (character === "\\") escaped2 = true;
|
|
88
|
+
else if (triple && contents.slice(index, index + 3) === quote.repeat(3)) {
|
|
89
|
+
chars[index] = chars[index + 1] = chars[index + 2] = " ";
|
|
90
|
+
quote = null;
|
|
91
|
+
triple = false;
|
|
92
|
+
index += 2;
|
|
93
|
+
} else if (!triple && character === quote) quote = null;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (character === "#") {
|
|
97
|
+
chars[index] = " ";
|
|
98
|
+
comment = true;
|
|
99
|
+
} else if (character === "'" || character === '"') {
|
|
100
|
+
triple = contents.slice(index, index + 3) === character.repeat(3);
|
|
101
|
+
quote = character;
|
|
102
|
+
chars[index] = " ";
|
|
103
|
+
if (triple) {
|
|
104
|
+
chars[index + 1] = chars[index + 2] = " ";
|
|
105
|
+
index += 2;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return chars.join("");
|
|
110
|
+
}
|
|
111
|
+
function isExcludedLlmSourcePath(path) {
|
|
112
|
+
const normalized = path.replaceAll("\\", "/");
|
|
113
|
+
return /(?:^|\/)(?:__tests__|test|tests|fixtures?|examples?|docs?|vendor|generated)(?:\/|$)/i.test(
|
|
114
|
+
normalized
|
|
115
|
+
) || /\.(?:test|spec|fixture)\.[cm]?[jt]sx?$/i.test(normalized) || /(?:^|\/)(?:test_[^/]+|[^/]+_test)\.py$/i.test(normalized);
|
|
116
|
+
}
|
|
117
|
+
function hasUnmaskedJavaScriptMatch(contents, pattern) {
|
|
118
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
119
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
|
|
120
|
+
for (const match of contents.matchAll(new RegExp(pattern.source, flags))) {
|
|
121
|
+
if (match.index !== void 0 && (masked[match.index] ?? " ") !== " ") return true;
|
|
122
|
+
}
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// src/packs/llm/javascript-integrations.ts
|
|
127
|
+
function matchingBrace(masked, start) {
|
|
128
|
+
let depth = 0;
|
|
129
|
+
for (let index = start; index < masked.length; index += 1) {
|
|
130
|
+
if (masked[index] === "{") depth += 1;
|
|
131
|
+
if (masked[index] === "}") {
|
|
132
|
+
depth -= 1;
|
|
133
|
+
if (depth === 0) return index;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return masked.length;
|
|
137
|
+
}
|
|
138
|
+
function splitTopLevel(value, delimiter) {
|
|
139
|
+
const parts = [];
|
|
140
|
+
let start = 0;
|
|
141
|
+
let round = 0;
|
|
142
|
+
let square = 0;
|
|
143
|
+
let curly = 0;
|
|
144
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
145
|
+
const character = value[index];
|
|
146
|
+
if (character === "(") round += 1;
|
|
147
|
+
else if (character === ")") round -= 1;
|
|
148
|
+
else if (character === "[") square += 1;
|
|
149
|
+
else if (character === "]") square -= 1;
|
|
150
|
+
else if (character === "{") curly += 1;
|
|
151
|
+
else if (character === "}") curly -= 1;
|
|
152
|
+
else if (character === delimiter && round === 0 && square === 0 && curly === 0) {
|
|
153
|
+
parts.push(value.slice(start, index));
|
|
154
|
+
start = index + 1;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
parts.push(value.slice(start));
|
|
158
|
+
return parts;
|
|
159
|
+
}
|
|
160
|
+
function topLevelCharacter(value, sought) {
|
|
161
|
+
let round = 0;
|
|
162
|
+
let square = 0;
|
|
163
|
+
let curly = 0;
|
|
164
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
165
|
+
const character = value[index];
|
|
166
|
+
if (character === "(") round += 1;
|
|
167
|
+
else if (character === ")") round -= 1;
|
|
168
|
+
else if (character === "[") square += 1;
|
|
169
|
+
else if (character === "]") square -= 1;
|
|
170
|
+
else if (character === "{") curly += 1;
|
|
171
|
+
else if (character === "}") curly -= 1;
|
|
172
|
+
else if (character === sought && round === 0 && square === 0 && curly === 0) return index;
|
|
173
|
+
}
|
|
174
|
+
return -1;
|
|
175
|
+
}
|
|
176
|
+
function matchingOpeningDelimiter(value, closeIndex) {
|
|
177
|
+
const close = value[closeIndex];
|
|
178
|
+
const open = close === "}" ? "{" : close === "]" ? "[" : "(";
|
|
179
|
+
let depth = 0;
|
|
180
|
+
for (let index = closeIndex; index >= 0; index -= 1) {
|
|
181
|
+
if (value[index] === close) depth += 1;
|
|
182
|
+
if (value[index] === open) {
|
|
183
|
+
depth -= 1;
|
|
184
|
+
if (depth === 0) return index;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return -1;
|
|
188
|
+
}
|
|
189
|
+
function unwrapBindingParentheses(value) {
|
|
190
|
+
let normalized = value.trim();
|
|
191
|
+
while (normalized.startsWith("(") && normalized.endsWith(")")) {
|
|
192
|
+
if (matchingOpeningDelimiter(normalized, normalized.length - 1) !== 0) break;
|
|
193
|
+
normalized = normalized.slice(1, -1).trim();
|
|
194
|
+
}
|
|
195
|
+
return normalized;
|
|
196
|
+
}
|
|
197
|
+
function bindingNames(pattern) {
|
|
198
|
+
const names = /* @__PURE__ */ new Set();
|
|
199
|
+
const collect = (input) => {
|
|
200
|
+
let normalized = unwrapBindingParentheses(input).trim();
|
|
201
|
+
if (normalized.startsWith("...")) normalized = normalized.slice(3).trim();
|
|
202
|
+
const defaultIndex = topLevelCharacter(normalized, "=");
|
|
203
|
+
if (defaultIndex >= 0) normalized = normalized.slice(0, defaultIndex).trim();
|
|
204
|
+
if (normalized.startsWith("[") && normalized.endsWith("]")) {
|
|
205
|
+
for (const element of splitTopLevel(normalized.slice(1, -1), ",")) collect(element);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (normalized.startsWith("{") && normalized.endsWith("}")) {
|
|
209
|
+
for (const property of splitTopLevel(normalized.slice(1, -1), ",")) {
|
|
210
|
+
const colon = topLevelCharacter(property, ":");
|
|
211
|
+
if (colon >= 0) collect(property.slice(colon + 1));
|
|
212
|
+
else collect(property);
|
|
213
|
+
}
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
const identifier = normalized.match(/^[A-Za-z_$][\w$]*$/)?.[0];
|
|
217
|
+
if (identifier) names.add(identifier);
|
|
218
|
+
};
|
|
219
|
+
collect(pattern);
|
|
220
|
+
return names;
|
|
221
|
+
}
|
|
222
|
+
function parameterNames(parameters) {
|
|
223
|
+
const names = /* @__PURE__ */ new Set();
|
|
224
|
+
for (const parameter of splitTopLevel(parameters, ",")) {
|
|
225
|
+
for (const name of bindingNames(parameter)) names.add(name);
|
|
226
|
+
}
|
|
227
|
+
return names;
|
|
228
|
+
}
|
|
229
|
+
function destructuringAssignmentEvents(masked, beforeIndex, identifier) {
|
|
230
|
+
const events = [];
|
|
231
|
+
const prefix = masked.slice(0, beforeIndex);
|
|
232
|
+
for (let index = 0; index < prefix.length; index += 1) {
|
|
233
|
+
if (prefix[index] !== "=" || prefix[index - 1] === "=" || prefix[index + 1] === "=" || prefix[index + 1] === ">" || /[!<>+\-*/%&|^?]/.test(prefix[index - 1] ?? "")) continue;
|
|
234
|
+
let targetEnd = index - 1;
|
|
235
|
+
while (/\s/.test(prefix[targetEnd] ?? "")) targetEnd -= 1;
|
|
236
|
+
if (!["}", "]"].includes(prefix[targetEnd] ?? "")) continue;
|
|
237
|
+
const targetStart = matchingOpeningDelimiter(prefix, targetEnd);
|
|
238
|
+
if (targetStart < 0) continue;
|
|
239
|
+
const target = prefix.slice(targetStart, targetEnd + 1);
|
|
240
|
+
if (bindingNames(target).has(identifier)) {
|
|
241
|
+
events.push({ index: targetStart, expression: void 0 });
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return events;
|
|
245
|
+
}
|
|
246
|
+
function functionScopes(masked) {
|
|
247
|
+
const scopes = [];
|
|
248
|
+
const patterns = [
|
|
249
|
+
/\b(?:async\s+)?function(?:\s+[A-Za-z_$][\w$]*)?\s*\(([^)]*)\)\s*\{/g,
|
|
250
|
+
/(?:\(([^()]*)\)|([A-Za-z_$][\w$]*))\s*=>\s*\{/g,
|
|
251
|
+
/\b(?:async\s+)?(?!(?:if|for|while|switch|catch|with)\b)[A-Za-z_$][\w$]*\s*\(([^)]*)\)\s*\{/g
|
|
252
|
+
];
|
|
253
|
+
for (const pattern of patterns) {
|
|
254
|
+
for (const match of masked.matchAll(pattern)) {
|
|
255
|
+
const relativeBrace = match[0].lastIndexOf("{");
|
|
256
|
+
const bodyStart = match.index + relativeBrace;
|
|
257
|
+
scopes.push({
|
|
258
|
+
bodyStart,
|
|
259
|
+
bodyEnd: matchingBrace(masked, bodyStart),
|
|
260
|
+
parameters: parameterNames(match[1] ?? match[2] ?? "")
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
for (const match of masked.matchAll(
|
|
265
|
+
/(?:\(([^()]*)\)|([A-Za-z_$][\w$]*))\s*=>\s*(?!\{)/g
|
|
266
|
+
)) {
|
|
267
|
+
const bodyStart = match.index + match[0].length;
|
|
268
|
+
const tail = masked.slice(bodyStart);
|
|
269
|
+
const terminators = [tail.indexOf(";"), tail.indexOf("\n")].filter(
|
|
270
|
+
(index) => index >= 0
|
|
271
|
+
);
|
|
272
|
+
const bodyEnd = bodyStart + (terminators.length > 0 ? Math.min(...terminators) : tail.length);
|
|
273
|
+
scopes.push({
|
|
274
|
+
bodyStart,
|
|
275
|
+
bodyEnd,
|
|
276
|
+
parameters: parameterNames(match[1] ?? match[2] ?? "")
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
return scopes;
|
|
280
|
+
}
|
|
281
|
+
function importedBindings(contents) {
|
|
282
|
+
const openAiClasses = /* @__PURE__ */ new Set();
|
|
283
|
+
const anthropicClasses = /* @__PURE__ */ new Set();
|
|
284
|
+
const vercelCalls = /* @__PURE__ */ new Set();
|
|
285
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
286
|
+
for (const match of contents.matchAll(
|
|
287
|
+
/import\s+([A-Za-z_$][\w$]*)\s+from\s+['"](openai|@anthropic-ai\/sdk)['"]/g
|
|
288
|
+
)) {
|
|
289
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
290
|
+
if (match[2] === "openai" && match[1]) openAiClasses.add(match[1]);
|
|
291
|
+
if (match[2] === "@anthropic-ai/sdk" && match[1]) anthropicClasses.add(match[1]);
|
|
292
|
+
}
|
|
293
|
+
for (const match of contents.matchAll(
|
|
294
|
+
/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*require\(\s*['"](openai|@anthropic-ai\/sdk)['"]\s*\)/g
|
|
295
|
+
)) {
|
|
296
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
297
|
+
if (match[2] === "openai" && match[1]) openAiClasses.add(match[1]);
|
|
298
|
+
if (match[2] === "@anthropic-ai/sdk" && match[1]) anthropicClasses.add(match[1]);
|
|
299
|
+
}
|
|
300
|
+
for (const match of contents.matchAll(
|
|
301
|
+
/import\s*\{([^}]+)\}\s*from\s*['"]ai['"]/g
|
|
302
|
+
)) {
|
|
303
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
304
|
+
for (const member of (match[1] ?? "").split(",")) {
|
|
305
|
+
const binding = member.trim().match(/^(generateText|streamText)(?:\s+as\s+([A-Za-z_$][\w$]*))?$/);
|
|
306
|
+
if (binding?.[1]) vercelCalls.add(binding[2] ?? binding[1]);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return { openAiClasses, anthropicClasses, vercelCalls };
|
|
310
|
+
}
|
|
311
|
+
function supportedJavaScriptModelCallIndices(contents) {
|
|
312
|
+
const imports = importedBindings(contents);
|
|
313
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
314
|
+
const indices = /* @__PURE__ */ new Set();
|
|
315
|
+
const scopes = functionScopes(masked);
|
|
316
|
+
const innermostScopeAt = (index) => scopes.filter((scope) => scope.bodyStart < index && index < scope.bodyEnd).sort((left, right) => right.bodyStart - left.bodyStart)[0];
|
|
317
|
+
const bindingEvents = (identifier, beforeIndex) => {
|
|
318
|
+
const escaped2 = identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
319
|
+
const pattern = new RegExp(
|
|
320
|
+
`\\b(?:const|let|var)\\s+${escaped2}\\b(?:\\s*=\\s*(?![=>])([^;\\n]+))?|(?<![.$\\w])${escaped2}\\s*=\\s*(?![=>])([^;\\n]+)|\\b(?:function|class)\\s+${escaped2}\\b`,
|
|
321
|
+
"g"
|
|
322
|
+
);
|
|
323
|
+
const directEvents = [...masked.slice(0, beforeIndex).matchAll(pattern)].map((match) => ({
|
|
324
|
+
index: match.index,
|
|
325
|
+
expression: match[1] ?? match[2]
|
|
326
|
+
}));
|
|
327
|
+
const destructuringEvents = destructuringAssignmentEvents(masked, beforeIndex, identifier);
|
|
328
|
+
return [...directEvents, ...destructuringEvents].sort((left, right) => left.index - right.index);
|
|
329
|
+
};
|
|
330
|
+
const visibleScopeChain = (callIndex) => [
|
|
331
|
+
...scopes.filter((scope) => scope.bodyStart < callIndex && callIndex < scope.bodyEnd).sort((left, right) => right.bodyStart - left.bodyStart),
|
|
332
|
+
void 0
|
|
333
|
+
];
|
|
334
|
+
const latestEventInScope = (events, scope) => events.filter((event) => innermostScopeAt(event.index) === scope).at(-1);
|
|
335
|
+
const importedBindingIsVisible = (identifier, callIndex) => {
|
|
336
|
+
const events = bindingEvents(identifier, callIndex);
|
|
337
|
+
for (const scope of visibleScopeChain(callIndex)) {
|
|
338
|
+
if (scope?.parameters.has(identifier)) return false;
|
|
339
|
+
if (latestEventInScope(events, scope)) return false;
|
|
340
|
+
}
|
|
341
|
+
return true;
|
|
342
|
+
};
|
|
343
|
+
const supportedClientBinding = (receiver, classes, callIndex) => {
|
|
344
|
+
const events = bindingEvents(receiver, callIndex);
|
|
345
|
+
for (const scope of visibleScopeChain(callIndex)) {
|
|
346
|
+
if (scope?.parameters.has(receiver)) return false;
|
|
347
|
+
const event = latestEventInScope(events, scope);
|
|
348
|
+
if (!event) continue;
|
|
349
|
+
if (!event.expression) return false;
|
|
350
|
+
return [...classes].some((className) => {
|
|
351
|
+
const escapedClass = className.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
352
|
+
return new RegExp(`^\\s*new\\s+${escapedClass}\\s*\\(`).test(event.expression ?? "") && importedBindingIsVisible(className, event.index);
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
return false;
|
|
356
|
+
};
|
|
357
|
+
for (const match of masked.matchAll(
|
|
358
|
+
/(?:\.responses\.create|\.chat\.completions\.create|\.messages\.create|\b[A-Za-z_$][\w$]*\s*\()/g
|
|
359
|
+
)) {
|
|
360
|
+
const token = match[0];
|
|
361
|
+
if (!token) continue;
|
|
362
|
+
if (!token.startsWith(".")) {
|
|
363
|
+
const name = token.match(/^([A-Za-z_$][\w$]*)/)?.[1];
|
|
364
|
+
if (name && imports.vercelCalls.has(name) && importedBindingIsVisible(name, match.index)) {
|
|
365
|
+
indices.add(match.index);
|
|
366
|
+
}
|
|
367
|
+
continue;
|
|
368
|
+
}
|
|
369
|
+
const prefix = masked.slice(0, match.index);
|
|
370
|
+
const receiver = prefix.match(/([A-Za-z_$][\w$]*)\s*$/)?.[1];
|
|
371
|
+
const directClass = prefix.match(/new\s+([A-Za-z_$][\w$]*)\s*\(\s*\)\s*$/)?.[1];
|
|
372
|
+
if (token.startsWith(".messages.create") && (receiver && supportedClientBinding(receiver, imports.anthropicClasses, match.index) || directClass && imports.anthropicClasses.has(directClass) && importedBindingIsVisible(directClass, match.index)) || !token.startsWith(".messages.create") && (receiver && supportedClientBinding(receiver, imports.openAiClasses, match.index) || directClass && imports.openAiClasses.has(directClass) && importedBindingIsVisible(directClass, match.index))) {
|
|
373
|
+
indices.add(match.index);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return indices;
|
|
377
|
+
}
|
|
378
|
+
function hasSupportedJavaScriptModelCall(contents) {
|
|
379
|
+
return supportedJavaScriptModelCallIndices(contents).size > 0;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// src/packs/llm/action-rules.ts
|
|
383
|
+
var MODEL_OUTPUT_PATTERN = /(?:\.output_text\b|\.choices\s*\[[^\]]+\]\s*\.message\s*\.content\b|\.content\s*\[[^\]]+\]\s*\.text\b)/;
|
|
384
|
+
var IDENTIFIER_ASSIGNMENT_PATTERN = /^\s*(?:(?:const|let|var)\s+)?([A-Za-z_$][\w$]*)\s*=\s*(?!=)(.+)$/;
|
|
385
|
+
var STRUCTURED_PARSE_PATTERN = /\bJSON\.parse\s*\(([^)]*)\)/;
|
|
386
|
+
var RUNTIME_PARSE_PATTERN = /\b[A-Za-z_$][\w$]*(?:Schema|schema)\s*(?:\.strict\s*\(\s*\))?\.parse\s*\(/;
|
|
387
|
+
var CONSEQUENTIAL_DISPATCH_PATTERN = /\b(?:deploy|execute|executeAction|runCommand|transferFunds|sendPayment|grantAccess|revokeAccess|updateRole|deleteFile|deleteRecord|writeFile|sendEmail|sendMessage|publish)\s*\(/;
|
|
388
|
+
var TOOL_ASSIGNMENT_PATTERN = /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:tool|defineTool|createTool)\s*\(/g;
|
|
389
|
+
var MEMBER_TOOL_REGISTRATION_PATTERN = /\b([A-Za-z_$][\w$]*)\.registerTool\s*\(/g;
|
|
390
|
+
var APPROVAL_FAIL_CLOSED_PATTERN = /if\s*\(\s*!\s*\(?\s*(?:await\s+)?(?:requestApproval|requireApproval|humanApproval|approvalGate|waitForApproval)\s*\([^)]*\)\s*\)?\s*\)\s*(?:\{\s*)?(?:throw\b|return\b)/s;
|
|
391
|
+
var ALLOWLIST_FAIL_CLOSED_PATTERN = /if\s*\(\s*!\s*[A-Za-z_$][\w$]*(?:ALLOWLIST|ALLOW_LIST|ALLOWED|PERMITTED)[A-Za-z0-9_$]*\s*\.(?:includes|has)\s*\([^)]*\)\s*\)\s*(?:\{\s*)?(?:throw\b|return\b)/s;
|
|
392
|
+
function isExcludedSourcePath(path) {
|
|
393
|
+
return /(?:^|\/)(?:__tests__|test|tests|fixtures?|examples?|vendor|generated)(?:\/|$)/i.test(path) || /\.(?:test|spec|fixture)\.[cm]?[jt]sx?$/i.test(path) || /(?:^|\/)(?:test_[^/]+|[^/]+_test)\.py$/i.test(path);
|
|
394
|
+
}
|
|
395
|
+
function maskNonCode(contents) {
|
|
396
|
+
const chars = [...contents];
|
|
397
|
+
let quote = null;
|
|
398
|
+
let escaped2 = false;
|
|
399
|
+
let lineComment = false;
|
|
400
|
+
let blockComment = false;
|
|
401
|
+
for (let index = 0; index < chars.length; index += 1) {
|
|
402
|
+
const character = chars[index] ?? "";
|
|
403
|
+
const next = chars[index + 1] ?? "";
|
|
404
|
+
if (lineComment) {
|
|
405
|
+
if (character === "\n") lineComment = false;
|
|
406
|
+
else chars[index] = " ";
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
if (blockComment) {
|
|
410
|
+
if (character === "*" && next === "/") {
|
|
411
|
+
chars[index] = " ";
|
|
412
|
+
chars[index + 1] = " ";
|
|
413
|
+
blockComment = false;
|
|
414
|
+
index += 1;
|
|
415
|
+
} else if (character !== "\n") chars[index] = " ";
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
if (quote) {
|
|
419
|
+
if (character !== "\n") chars[index] = " ";
|
|
420
|
+
if (escaped2) escaped2 = false;
|
|
421
|
+
else if (character === "\\") escaped2 = true;
|
|
422
|
+
else if (character === quote) quote = null;
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (character === "/" && next === "/") {
|
|
426
|
+
chars[index] = " ";
|
|
427
|
+
chars[index + 1] = " ";
|
|
428
|
+
lineComment = true;
|
|
429
|
+
index += 1;
|
|
430
|
+
} else if (character === "/" && next === "*") {
|
|
431
|
+
chars[index] = " ";
|
|
432
|
+
chars[index + 1] = " ";
|
|
433
|
+
blockComment = true;
|
|
434
|
+
index += 1;
|
|
435
|
+
} else if (character === "'" || character === '"' || character === "`") {
|
|
436
|
+
chars[index] = " ";
|
|
437
|
+
quote = character;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
return chars.join("");
|
|
441
|
+
}
|
|
442
|
+
function lineStartDepths(contents) {
|
|
443
|
+
const masked = maskNonCode(contents);
|
|
444
|
+
const depths = [];
|
|
445
|
+
let depth = 0;
|
|
446
|
+
for (const line of masked.split("\n")) {
|
|
447
|
+
depths.push(depth);
|
|
448
|
+
for (const character of line) {
|
|
449
|
+
if (character === "{") depth += 1;
|
|
450
|
+
else if (character === "}") depth = Math.max(0, depth - 1);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
return depths;
|
|
454
|
+
}
|
|
455
|
+
function lineNumberAt(contents, index) {
|
|
456
|
+
return contents.slice(0, index).split("\n").length;
|
|
457
|
+
}
|
|
458
|
+
function finding(ruleId, file, index, severity, summary, label, confidence) {
|
|
459
|
+
return {
|
|
460
|
+
ruleId,
|
|
461
|
+
severity,
|
|
462
|
+
confidence,
|
|
463
|
+
summary,
|
|
464
|
+
evidence: {
|
|
465
|
+
path: file.path,
|
|
466
|
+
line: lineNumberAt(file.contents, index),
|
|
467
|
+
label
|
|
468
|
+
}
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
function escapeRegExp(value) {
|
|
472
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
473
|
+
}
|
|
474
|
+
function modelOutputAliasesBeforeLines(maskedLines, depths, lineOffsets, supportedCallIndices) {
|
|
475
|
+
const aliasesByDepth = /* @__PURE__ */ new Map();
|
|
476
|
+
const snapshots = [];
|
|
477
|
+
for (let lineIndex = 0; lineIndex < maskedLines.length; lineIndex += 1) {
|
|
478
|
+
const depth = depths[lineIndex] ?? 0;
|
|
479
|
+
for (const knownDepth of aliasesByDepth.keys()) {
|
|
480
|
+
if (knownDepth > depth) aliasesByDepth.delete(knownDepth);
|
|
481
|
+
}
|
|
482
|
+
const aliases = aliasesByDepth.get(depth) ?? /* @__PURE__ */ new Set();
|
|
483
|
+
aliasesByDepth.set(depth, aliases);
|
|
484
|
+
snapshots.push(new Set(aliases));
|
|
485
|
+
const assignment = (maskedLines[lineIndex] ?? "").match(IDENTIFIER_ASSIGNMENT_PATTERN);
|
|
486
|
+
const identifier = assignment?.[1];
|
|
487
|
+
const expression = assignment?.[2];
|
|
488
|
+
if (!identifier || !expression) continue;
|
|
489
|
+
const lineStart = lineOffsets[lineIndex] ?? 0;
|
|
490
|
+
const lineHasSupportedResponseAssignment = [...supportedCallIndices].some(
|
|
491
|
+
(index) => lineStart <= index && index < lineStart + (maskedLines[lineIndex]?.length ?? 0)
|
|
492
|
+
);
|
|
493
|
+
if (lineHasSupportedResponseAssignment || expressionContainsModelOutput(expression, aliases)) aliases.add(identifier);
|
|
494
|
+
else aliases.delete(identifier);
|
|
495
|
+
}
|
|
496
|
+
return snapshots;
|
|
497
|
+
}
|
|
498
|
+
function expressionContainsModelOutput(expression, modelOutputAliases) {
|
|
499
|
+
const output = expression.match(MODEL_OUTPUT_PATTERN);
|
|
500
|
+
if (output?.index !== void 0) {
|
|
501
|
+
const receiver = expression.slice(0, output.index);
|
|
502
|
+
const identifier = receiver.match(/([A-Za-z_$][\w$]*)[.\[\]\w$\s]*$/)?.[1];
|
|
503
|
+
if (identifier && modelOutputAliases.has(identifier)) return true;
|
|
504
|
+
}
|
|
505
|
+
for (const alias of modelOutputAliases) {
|
|
506
|
+
if (new RegExp(`\\b${escapeRegExp(alias)}\\b`).test(expression)) return true;
|
|
507
|
+
}
|
|
508
|
+
return false;
|
|
509
|
+
}
|
|
510
|
+
function hasFailClosedValidation(segment, identifier) {
|
|
511
|
+
const escaped2 = escapeRegExp(identifier);
|
|
512
|
+
const explicitParse = new RegExp(
|
|
513
|
+
`\\b[A-Za-z_$][\\w$]*(?:Schema|schema)\\s*(?:\\.strict\\s*\\(\\s*\\))?\\.parse\\s*\\(\\s*${escaped2}\\s*\\)`
|
|
514
|
+
);
|
|
515
|
+
const allowlistGuard = new RegExp(
|
|
516
|
+
`if\\s*\\(\\s*!\\s*[A-Za-z_$][\\w$]*(?:ALLOWLIST|ALLOW_LIST|ALLOWED|PERMITTED)[A-Za-z0-9_$]*\\s*\\.(?:includes|has)\\s*\\(\\s*${escaped2}\\.[A-Za-z_$][\\w$]*\\s*\\)\\s*\\)\\s*(?:\\{\\s*)?(?:throw\\b|return\\b)`,
|
|
517
|
+
"s"
|
|
518
|
+
);
|
|
519
|
+
return explicitParse.test(segment) || allowlistGuard.test(segment);
|
|
520
|
+
}
|
|
521
|
+
function detectUnvalidatedConsequentialAction(file) {
|
|
522
|
+
if (file.path.toLowerCase().endsWith(".py")) {
|
|
523
|
+
return detectPythonUnvalidatedConsequentialAction(file);
|
|
524
|
+
}
|
|
525
|
+
if (isExcludedSourcePath(file.path)) return [];
|
|
526
|
+
const findings = [];
|
|
527
|
+
const lines = file.contents.split("\n");
|
|
528
|
+
const maskedLines = maskNonCode(file.contents).split("\n");
|
|
529
|
+
const depths = lineStartDepths(file.contents);
|
|
530
|
+
const lineOffsets = [];
|
|
531
|
+
let runningOffset = 0;
|
|
532
|
+
for (const line of maskedLines) {
|
|
533
|
+
lineOffsets.push(runningOffset);
|
|
534
|
+
runningOffset += line.length + 1;
|
|
535
|
+
}
|
|
536
|
+
const aliasesBeforeLines = modelOutputAliasesBeforeLines(
|
|
537
|
+
maskedLines,
|
|
538
|
+
depths,
|
|
539
|
+
lineOffsets,
|
|
540
|
+
supportedJavaScriptModelCallIndices(file.contents)
|
|
541
|
+
);
|
|
542
|
+
let offset = 0;
|
|
543
|
+
for (let assignmentLine = 0; assignmentLine < lines.length; assignmentLine += 1) {
|
|
544
|
+
const line = maskedLines[assignmentLine] ?? "";
|
|
545
|
+
const assignment = line.match(IDENTIFIER_ASSIGNMENT_PATTERN);
|
|
546
|
+
const identifier = assignment?.[1];
|
|
547
|
+
const expression = assignment?.[2];
|
|
548
|
+
const structuredParse = expression?.match(STRUCTURED_PARSE_PATTERN);
|
|
549
|
+
if (!identifier || !expression || !structuredParse?.[1]) {
|
|
550
|
+
offset += line.length + 1;
|
|
551
|
+
continue;
|
|
552
|
+
}
|
|
553
|
+
if (!expressionContainsModelOutput(
|
|
554
|
+
structuredParse[1],
|
|
555
|
+
aliasesBeforeLines[assignmentLine] ?? /* @__PURE__ */ new Set()
|
|
556
|
+
)) {
|
|
557
|
+
offset += line.length + 1;
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
if (RUNTIME_PARSE_PATTERN.test(expression)) {
|
|
561
|
+
offset += line.length + 1;
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
let dispatchOffset = offset + line.length + 1;
|
|
565
|
+
const assignmentDepth = depths[assignmentLine] ?? 0;
|
|
566
|
+
for (let dispatchLine = assignmentLine + 1; dispatchLine < lines.length; dispatchLine += 1) {
|
|
567
|
+
const candidate = maskedLines[dispatchLine] ?? "";
|
|
568
|
+
const candidateDepth = depths[dispatchLine] ?? 0;
|
|
569
|
+
if (candidateDepth < assignmentDepth) break;
|
|
570
|
+
if (candidateDepth !== assignmentDepth) {
|
|
571
|
+
dispatchOffset += (lines[dispatchLine] ?? "").length + 1;
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
const dispatch = candidate.match(CONSEQUENTIAL_DISPATCH_PATTERN);
|
|
575
|
+
const dispatchExpression = dispatch ? candidate.slice((dispatch.index ?? 0) + dispatch[0].length) : "";
|
|
576
|
+
const fieldUse = new RegExp(
|
|
577
|
+
`\\b${escapeRegExp(identifier)}\\.([A-Za-z_$][\\w$]*)\\b`
|
|
578
|
+
).exec(dispatchExpression);
|
|
579
|
+
if (dispatch && fieldUse?.[1]) {
|
|
580
|
+
const between = maskedLines.slice(assignmentLine + 1, dispatchLine).join("\n");
|
|
581
|
+
if (!hasFailClosedValidation(between, identifier)) {
|
|
582
|
+
findings.push(
|
|
583
|
+
finding(
|
|
584
|
+
"LLM-VAL-001",
|
|
585
|
+
file,
|
|
586
|
+
dispatchOffset + (dispatch.index ?? 0),
|
|
587
|
+
"critical",
|
|
588
|
+
`Model-produced structured field ${identifier}.${fieldUse[1]} reaches a consequential dispatcher without observable runtime validation.`,
|
|
589
|
+
`Unvalidated ${identifier}.${fieldUse[1]} reaches consequential dispatch`,
|
|
590
|
+
"high"
|
|
591
|
+
)
|
|
592
|
+
);
|
|
593
|
+
}
|
|
594
|
+
break;
|
|
595
|
+
}
|
|
596
|
+
dispatchOffset += candidate.length + 1;
|
|
597
|
+
}
|
|
598
|
+
offset += line.length + 1;
|
|
599
|
+
}
|
|
600
|
+
return findings;
|
|
601
|
+
}
|
|
602
|
+
function pythonModelFacingToolMethods(file) {
|
|
603
|
+
if (isExcludedSourcePath(file.path)) return [];
|
|
604
|
+
const masked = maskPythonNonCode(file.contents);
|
|
605
|
+
const originalLines = file.contents.split("\n");
|
|
606
|
+
const maskedLines = masked.split("\n");
|
|
607
|
+
const offsets = [];
|
|
608
|
+
let offset = 0;
|
|
609
|
+
for (const line of originalLines) {
|
|
610
|
+
offsets.push(offset);
|
|
611
|
+
offset += line.length + 1;
|
|
612
|
+
}
|
|
613
|
+
const methods = [];
|
|
614
|
+
for (let classLine = 0; classLine < maskedLines.length; classLine += 1) {
|
|
615
|
+
const declaration = (maskedLines[classLine] ?? "").match(
|
|
616
|
+
/^(\s*)class\s+[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?\s*:/
|
|
617
|
+
);
|
|
618
|
+
if (!declaration) continue;
|
|
619
|
+
const classIndent = declaration[1]?.length ?? 0;
|
|
620
|
+
let classEndLine = maskedLines.length;
|
|
621
|
+
for (let later = classLine + 1; later < maskedLines.length; later += 1) {
|
|
622
|
+
const candidate = maskedLines[later] ?? "";
|
|
623
|
+
if (candidate.trim().length === 0) continue;
|
|
624
|
+
const indentation = candidate.match(/^\s*/)?.[0].length ?? 0;
|
|
625
|
+
if (indentation <= classIndent) {
|
|
626
|
+
classEndLine = later;
|
|
627
|
+
break;
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
const classOffset = offsets[classLine] ?? 0;
|
|
631
|
+
const classEndOffset = offsets[classEndLine] ?? file.contents.length;
|
|
632
|
+
const classBody = file.contents.slice(classOffset, classEndOffset);
|
|
633
|
+
const classBodyMasked = masked.slice(classOffset, classEndOffset);
|
|
634
|
+
if (!/\bargs_schema\s*(?::[^=\n]+)?=/.test(classBodyMasked)) continue;
|
|
635
|
+
for (const method of classBodyMasked.matchAll(
|
|
636
|
+
/\bdef\s+_run\s*\(([\s\S]{0,1200}?)\)\s*(?:->[^:\n]+)?\s*:/g
|
|
637
|
+
)) {
|
|
638
|
+
const methodIndex = method.index ?? 0;
|
|
639
|
+
const methodLineWithinClass = classBodyMasked.slice(0, methodIndex).split("\n").length - 1;
|
|
640
|
+
const methodLine = classLine + methodLineWithinClass;
|
|
641
|
+
const methodDeclarationLines = method[0].split("\n").length;
|
|
642
|
+
const methodDeclarationLine = maskedLines[methodLine] ?? "";
|
|
643
|
+
const methodIndent = methodDeclarationLine.match(/^\s*/)?.[0].length ?? 0;
|
|
644
|
+
let methodEndLine = classEndLine;
|
|
645
|
+
for (let later = methodLine + methodDeclarationLines; later < classEndLine; later += 1) {
|
|
646
|
+
const candidate = maskedLines[later] ?? "";
|
|
647
|
+
if (candidate.trim().length === 0) continue;
|
|
648
|
+
const indentation = candidate.match(/^\s*/)?.[0].length ?? 0;
|
|
649
|
+
if (indentation <= methodIndent) {
|
|
650
|
+
methodEndLine = later;
|
|
651
|
+
break;
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
const bodyOffset = offsets[methodLine] ?? classOffset + methodIndex;
|
|
655
|
+
const bodyEndOffset = offsets[methodEndLine] ?? classEndOffset;
|
|
656
|
+
const parameters = /* @__PURE__ */ new Set();
|
|
657
|
+
for (const parameter of (method[1] ?? "").split(",")) {
|
|
658
|
+
const identifier = parameter.trim().match(/^([A-Za-z_][A-Za-z0-9_]*)/)?.[1];
|
|
659
|
+
if (identifier && identifier !== "self") parameters.add(identifier);
|
|
660
|
+
}
|
|
661
|
+
methods.push({
|
|
662
|
+
constraintScope: classBody,
|
|
663
|
+
body: file.contents.slice(bodyOffset, bodyEndOffset),
|
|
664
|
+
bodyMasked: masked.slice(bodyOffset, bodyEndOffset),
|
|
665
|
+
bodyOffset,
|
|
666
|
+
parameters
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
return methods;
|
|
671
|
+
}
|
|
672
|
+
function pythonToolExecution(method) {
|
|
673
|
+
for (const invocation of method.bodyMasked.matchAll(
|
|
674
|
+
/\b[A-Za-z_][A-Za-z0-9_.]*\.invoke\s*\(/g
|
|
675
|
+
)) {
|
|
676
|
+
const openParen = (invocation.index ?? 0) + invocation[0].lastIndexOf("(");
|
|
677
|
+
const end = matchingCallEnd(method.body, openParen);
|
|
678
|
+
if (end === null) continue;
|
|
679
|
+
const call = method.body.slice(invocation.index, end);
|
|
680
|
+
if (!/\bmethod\s*=\s*['"][^'"]*execute[_-]?code[^'"]*['"]/i.test(call)) {
|
|
681
|
+
continue;
|
|
682
|
+
}
|
|
683
|
+
for (const parameter of method.parameters) {
|
|
684
|
+
if (!/^code$/i.test(parameter)) continue;
|
|
685
|
+
const escaped2 = escapeRegExp(parameter);
|
|
686
|
+
if (!new RegExp(`['"][A-Za-z_][A-Za-z0-9_]*['"]\\s*:\\s*${escaped2}\\b`).test(call)) {
|
|
687
|
+
continue;
|
|
688
|
+
}
|
|
689
|
+
const constrainedAnnotation = new RegExp(
|
|
690
|
+
`\\b${escaped2}\\s*:\\s*(?:Literal|Enum|Annotated)\\s*\\[`
|
|
691
|
+
);
|
|
692
|
+
const localValidator = new RegExp(
|
|
693
|
+
`\\bfield_validator\\s*\\(\\s*['"]${escaped2}['"]`
|
|
694
|
+
);
|
|
695
|
+
if (constrainedAnnotation.test(method.body) || localValidator.test(method.constraintScope)) {
|
|
696
|
+
continue;
|
|
697
|
+
}
|
|
698
|
+
return {
|
|
699
|
+
// The defect is introduced at the model-facing input declaration; the downstream
|
|
700
|
+
// dispatcher merely demonstrates consequence. Anchoring the finding here also avoids
|
|
701
|
+
// implying that every later invocation in the same toolkit is a separate defect.
|
|
702
|
+
index: method.bodyOffset,
|
|
703
|
+
parameter
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
return null;
|
|
708
|
+
}
|
|
709
|
+
function detectPythonUnvalidatedConsequentialAction(file) {
|
|
710
|
+
const findings = [];
|
|
711
|
+
for (const method of pythonModelFacingToolMethods(file)) {
|
|
712
|
+
const execution = pythonToolExecution(method);
|
|
713
|
+
if (!execution) continue;
|
|
714
|
+
findings.push(
|
|
715
|
+
finding(
|
|
716
|
+
"LLM-VAL-001",
|
|
717
|
+
file,
|
|
718
|
+
execution.index,
|
|
719
|
+
"critical",
|
|
720
|
+
`A model-facing structured tool parameter reaches a supported execution dispatcher without an observable closed constraint.`,
|
|
721
|
+
`Unconstrained structured field ${execution.parameter} reaches execution dispatch`,
|
|
722
|
+
"high"
|
|
723
|
+
)
|
|
724
|
+
);
|
|
725
|
+
}
|
|
726
|
+
return findings;
|
|
727
|
+
}
|
|
728
|
+
function hasConsequentialStructuredActionSurface(file) {
|
|
729
|
+
if (file.path.toLowerCase().endsWith(".py")) {
|
|
730
|
+
return pythonModelFacingToolMethods(file).some(
|
|
731
|
+
(method) => pythonToolExecution(method) !== null
|
|
732
|
+
);
|
|
733
|
+
}
|
|
734
|
+
if (isExcludedSourcePath(file.path)) return false;
|
|
735
|
+
const maskedLines = maskNonCode(file.contents).split("\n");
|
|
736
|
+
const depths = lineStartDepths(file.contents);
|
|
737
|
+
const lineOffsets = [];
|
|
738
|
+
let runningOffset = 0;
|
|
739
|
+
for (const line of maskedLines) {
|
|
740
|
+
lineOffsets.push(runningOffset);
|
|
741
|
+
runningOffset += line.length + 1;
|
|
742
|
+
}
|
|
743
|
+
const aliasesBeforeLines = modelOutputAliasesBeforeLines(
|
|
744
|
+
maskedLines,
|
|
745
|
+
depths,
|
|
746
|
+
lineOffsets,
|
|
747
|
+
supportedJavaScriptModelCallIndices(file.contents)
|
|
748
|
+
);
|
|
749
|
+
for (let lineIndex = 0; lineIndex < maskedLines.length; lineIndex += 1) {
|
|
750
|
+
const assignment = (maskedLines[lineIndex] ?? "").match(IDENTIFIER_ASSIGNMENT_PATTERN);
|
|
751
|
+
const expression = assignment?.[2];
|
|
752
|
+
const parsed = expression?.match(STRUCTURED_PARSE_PATTERN)?.[1];
|
|
753
|
+
if (!parsed || !expressionContainsModelOutput(parsed, aliasesBeforeLines[lineIndex] ?? /* @__PURE__ */ new Set())) {
|
|
754
|
+
continue;
|
|
755
|
+
}
|
|
756
|
+
const depth = depths[lineIndex] ?? 0;
|
|
757
|
+
for (let later = lineIndex + 1; later < maskedLines.length; later += 1) {
|
|
758
|
+
const laterDepth = depths[later] ?? 0;
|
|
759
|
+
if (laterDepth < depth) break;
|
|
760
|
+
if (laterDepth === depth && CONSEQUENTIAL_DISPATCH_PATTERN.test(maskedLines[later] ?? "")) {
|
|
761
|
+
return true;
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
return false;
|
|
766
|
+
}
|
|
767
|
+
function matchingCallEnd(contents, openParen) {
|
|
768
|
+
let depth = 0;
|
|
769
|
+
let quote = null;
|
|
770
|
+
let escaped2 = false;
|
|
771
|
+
let lineComment = false;
|
|
772
|
+
let blockComment = false;
|
|
773
|
+
for (let index = openParen; index < contents.length; index += 1) {
|
|
774
|
+
const character = contents[index] ?? "";
|
|
775
|
+
const next = contents[index + 1] ?? "";
|
|
776
|
+
if (lineComment) {
|
|
777
|
+
if (character === "\n") lineComment = false;
|
|
778
|
+
continue;
|
|
779
|
+
}
|
|
780
|
+
if (blockComment) {
|
|
781
|
+
if (character === "*" && next === "/") {
|
|
782
|
+
blockComment = false;
|
|
783
|
+
index += 1;
|
|
784
|
+
}
|
|
785
|
+
continue;
|
|
786
|
+
}
|
|
787
|
+
if (quote) {
|
|
788
|
+
if (escaped2) {
|
|
789
|
+
escaped2 = false;
|
|
790
|
+
} else if (character === "\\") {
|
|
791
|
+
escaped2 = true;
|
|
792
|
+
} else if (character === quote) {
|
|
793
|
+
quote = null;
|
|
794
|
+
}
|
|
795
|
+
continue;
|
|
796
|
+
}
|
|
797
|
+
if (character === "/" && next === "/") {
|
|
798
|
+
lineComment = true;
|
|
799
|
+
index += 1;
|
|
800
|
+
continue;
|
|
801
|
+
}
|
|
802
|
+
if (character === "/" && next === "*") {
|
|
803
|
+
blockComment = true;
|
|
804
|
+
index += 1;
|
|
805
|
+
continue;
|
|
806
|
+
}
|
|
807
|
+
if (character === "'" || character === '"' || character === "`") {
|
|
808
|
+
quote = character;
|
|
809
|
+
continue;
|
|
810
|
+
}
|
|
811
|
+
if (character === "(") depth += 1;
|
|
812
|
+
if (character === ")") {
|
|
813
|
+
depth -= 1;
|
|
814
|
+
if (depth === 0) return index + 1;
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
return null;
|
|
818
|
+
}
|
|
819
|
+
function isToolExposed(contents, maskedContents, toolName, afterIndex) {
|
|
820
|
+
const escaped2 = escapeRegExp(toolName);
|
|
821
|
+
const remainder = maskedContents.slice(afterIndex);
|
|
822
|
+
const supportedCalls = supportedJavaScriptModelCallIndices(contents);
|
|
823
|
+
for (const match of remainder.matchAll(new RegExp(
|
|
824
|
+
`\\b(?:generateText|streamText)\\s*\\([\\s\\S]{0,2000}?\\btools\\s*:\\s*\\{[^}]*\\b${escaped2}\\b`,
|
|
825
|
+
"gs"
|
|
826
|
+
))) {
|
|
827
|
+
if (supportedCalls.has(afterIndex + match.index)) return true;
|
|
828
|
+
}
|
|
829
|
+
return false;
|
|
830
|
+
}
|
|
831
|
+
var OBSERVABLE_SIDE_EFFECT_EXPORTS = {
|
|
832
|
+
fs: /* @__PURE__ */ new Set([
|
|
833
|
+
"appendFile",
|
|
834
|
+
"appendFileSync",
|
|
835
|
+
"rename",
|
|
836
|
+
"renameSync",
|
|
837
|
+
"rm",
|
|
838
|
+
"rmSync",
|
|
839
|
+
"unlink",
|
|
840
|
+
"unlinkSync",
|
|
841
|
+
"writeFile",
|
|
842
|
+
"writeFileSync"
|
|
843
|
+
]),
|
|
844
|
+
child_process: /* @__PURE__ */ new Set(["execFile", "execFileSync", "spawn", "spawnSync"])
|
|
845
|
+
};
|
|
846
|
+
function observableSideEffectBindings(contents, maskedContents) {
|
|
847
|
+
const direct = /* @__PURE__ */ new Set();
|
|
848
|
+
const namespaces = /* @__PURE__ */ new Map();
|
|
849
|
+
const namedImport = /\bimport\s*\{([^}]*)\}\s*from\s*(['"])(?:node:)?(fs|child_process)\2/g;
|
|
850
|
+
for (const match of contents.matchAll(namedImport)) {
|
|
851
|
+
if ((maskedContents[match.index] ?? " ") === " ") continue;
|
|
852
|
+
const exports = match[3] ? OBSERVABLE_SIDE_EFFECT_EXPORTS[match[3]] : void 0;
|
|
853
|
+
if (!exports) continue;
|
|
854
|
+
for (const specifier of (match[1] ?? "").split(",")) {
|
|
855
|
+
const binding = specifier.trim().match(/^([A-Za-z_$][\w$]*)(?:\s+as\s+([A-Za-z_$][\w$]*))?$/);
|
|
856
|
+
const imported = binding?.[1];
|
|
857
|
+
const local = binding?.[2] ?? imported;
|
|
858
|
+
if (imported && local && exports.has(imported)) direct.add(local);
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
const namespaceImport = /\bimport\s*\*\s*as\s*([A-Za-z_$][\w$]*)\s*from\s*(['"])(?:node:)?(fs|child_process)\2/g;
|
|
862
|
+
for (const match of contents.matchAll(namespaceImport)) {
|
|
863
|
+
if ((maskedContents[match.index] ?? " ") === " ") continue;
|
|
864
|
+
const namespace = match[1];
|
|
865
|
+
const exports = match[3] ? OBSERVABLE_SIDE_EFFECT_EXPORTS[match[3]] : void 0;
|
|
866
|
+
if (namespace && exports) namespaces.set(namespace, exports);
|
|
867
|
+
}
|
|
868
|
+
return { direct, namespaces };
|
|
869
|
+
}
|
|
870
|
+
function observableSideEffectIndex(declaration, bindings, helperNames = /* @__PURE__ */ new Set()) {
|
|
871
|
+
let earliest = null;
|
|
872
|
+
const consider = (index) => {
|
|
873
|
+
if (earliest === null || index < earliest) earliest = index;
|
|
874
|
+
};
|
|
875
|
+
for (const binding of bindings.direct) {
|
|
876
|
+
const match = new RegExp(`\\b${escapeRegExp(binding)}\\s*\\(`).exec(declaration);
|
|
877
|
+
if (match) consider(match.index);
|
|
878
|
+
}
|
|
879
|
+
for (const [namespace, exports] of bindings.namespaces) {
|
|
880
|
+
const operations = [...exports].map(escapeRegExp).join("|");
|
|
881
|
+
const match = new RegExp(
|
|
882
|
+
`\\b${escapeRegExp(namespace)}\\s*\\.\\s*(?:${operations})\\s*\\(`
|
|
883
|
+
).exec(declaration);
|
|
884
|
+
if (match) consider(match.index);
|
|
885
|
+
}
|
|
886
|
+
for (const helperName of helperNames) {
|
|
887
|
+
const match = new RegExp(`\\b${escapeRegExp(helperName)}\\s*\\(`).exec(declaration);
|
|
888
|
+
if (match) consider(match.index);
|
|
889
|
+
}
|
|
890
|
+
return earliest;
|
|
891
|
+
}
|
|
892
|
+
function dynamicOperationSelector(declaration) {
|
|
893
|
+
const field = /\b(action|operation)\s*:\s*Type\.(?:String|Union|Literal)\s*\(/i.exec(
|
|
894
|
+
declaration
|
|
895
|
+
);
|
|
896
|
+
const name = field?.[1];
|
|
897
|
+
if (!name) return null;
|
|
898
|
+
const use = new RegExp(`\\bparams\\.${escapeRegExp(name)}\\b`, "i").exec(declaration);
|
|
899
|
+
if (!use) return null;
|
|
900
|
+
return { name, index: field.index };
|
|
901
|
+
}
|
|
902
|
+
function matchingBraceEnd(contents, openBrace) {
|
|
903
|
+
let depth = 0;
|
|
904
|
+
for (let index = openBrace; index < contents.length; index += 1) {
|
|
905
|
+
const character = contents[index];
|
|
906
|
+
if (character === "{") depth += 1;
|
|
907
|
+
else if (character === "}") {
|
|
908
|
+
depth -= 1;
|
|
909
|
+
if (depth === 0) return index + 1;
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
return null;
|
|
913
|
+
}
|
|
914
|
+
function localSideEffectHelpers(maskedContents, bindings) {
|
|
915
|
+
const helpers = /* @__PURE__ */ new Map();
|
|
916
|
+
for (const declaration of maskedContents.matchAll(
|
|
917
|
+
/\bfunction\s+([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*(?::[^{\n]+)?\s*\{/g
|
|
918
|
+
)) {
|
|
919
|
+
const name = declaration[1];
|
|
920
|
+
if (!name) continue;
|
|
921
|
+
const openBrace = (declaration.index ?? 0) + declaration[0].lastIndexOf("{");
|
|
922
|
+
const end = matchingBraceEnd(maskedContents, openBrace);
|
|
923
|
+
if (end === null) continue;
|
|
924
|
+
const body = maskedContents.slice(openBrace + 1, end - 1);
|
|
925
|
+
const previous = helpers.get(name) ?? { declarations: 0, sideEffecting: false };
|
|
926
|
+
helpers.set(name, {
|
|
927
|
+
declarations: previous.declarations + 1,
|
|
928
|
+
sideEffecting: previous.sideEffecting || observableSideEffectIndex(body, bindings) !== null
|
|
929
|
+
});
|
|
930
|
+
}
|
|
931
|
+
return new Set(
|
|
932
|
+
[...helpers].filter(([, value]) => value.declarations === 1 && value.sideEffecting).map(([name]) => name)
|
|
933
|
+
);
|
|
934
|
+
}
|
|
935
|
+
function importedApiParameterScopes(contents, maskedContents) {
|
|
936
|
+
const importedTypes = /* @__PURE__ */ new Set();
|
|
937
|
+
for (const declaration of contents.matchAll(
|
|
938
|
+
/\bimport\s+(?:type\s+)?\{([^}]+)\}\s+from\s+['"]([^.'"][^'"]*)['"]/g
|
|
939
|
+
)) {
|
|
940
|
+
if ((maskedContents[declaration.index] ?? " ") === " ") continue;
|
|
941
|
+
for (const member of (declaration[1] ?? "").split(",")) {
|
|
942
|
+
const binding = member.trim().match(
|
|
943
|
+
/^(?:type\s+)?([A-Za-z_$][\w$]*)(?:\s+as\s+([A-Za-z_$][\w$]*))?$/
|
|
944
|
+
);
|
|
945
|
+
const local = binding?.[2] ?? binding?.[1];
|
|
946
|
+
if (local) importedTypes.add(local);
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
const scopes = [];
|
|
950
|
+
for (const declaration of maskedContents.matchAll(
|
|
951
|
+
/\bfunction(?:\s+[A-Za-z_$][\w$]*)?\s*\(([^)]*)\)\s*(?::[^{\n]+)?\s*\{/g
|
|
952
|
+
)) {
|
|
953
|
+
const openBrace = (declaration.index ?? 0) + declaration[0].lastIndexOf("{");
|
|
954
|
+
const end = matchingBraceEnd(maskedContents, openBrace);
|
|
955
|
+
if (end === null) continue;
|
|
956
|
+
for (const parameter of (declaration[1] ?? "").split(",")) {
|
|
957
|
+
const binding = parameter.trim().match(
|
|
958
|
+
/^([A-Za-z_$][\w$]*)\s*:\s*([A-Za-z_$][\w$]*)\b/
|
|
959
|
+
);
|
|
960
|
+
const receiver = binding?.[1];
|
|
961
|
+
const typeName = binding?.[2];
|
|
962
|
+
if (receiver && typeName && importedTypes.has(typeName)) {
|
|
963
|
+
scopes.push({ receiver, start: openBrace + 1, end: end - 1 });
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
return scopes;
|
|
968
|
+
}
|
|
969
|
+
function memberToolRegistrations(contents, maskedContents) {
|
|
970
|
+
const apiParameterScopes = importedApiParameterScopes(contents, maskedContents);
|
|
971
|
+
const registrations = [];
|
|
972
|
+
for (const registration of maskedContents.matchAll(MEMBER_TOOL_REGISTRATION_PATTERN)) {
|
|
973
|
+
const receiver = registration[1];
|
|
974
|
+
const registrationIndex = registration.index ?? 0;
|
|
975
|
+
if (!receiver || !apiParameterScopes.some(
|
|
976
|
+
(scope) => scope.receiver === receiver && registrationIndex >= scope.start && registrationIndex < scope.end
|
|
977
|
+
)) {
|
|
978
|
+
continue;
|
|
979
|
+
}
|
|
980
|
+
const openParen = (registration.index ?? 0) + registration[0].lastIndexOf("(");
|
|
981
|
+
const end = matchingCallEnd(maskedContents, openParen);
|
|
982
|
+
if (end === null) continue;
|
|
983
|
+
registrations.push({
|
|
984
|
+
receiver,
|
|
985
|
+
index: registrationIndex,
|
|
986
|
+
end,
|
|
987
|
+
declaration: maskedContents.slice(registration.index, end)
|
|
988
|
+
});
|
|
989
|
+
}
|
|
990
|
+
return registrations;
|
|
991
|
+
}
|
|
992
|
+
function registeredToolSideEffectSurface(file) {
|
|
993
|
+
if (isExcludedSourcePath(file.path)) return false;
|
|
994
|
+
const maskedContents = maskNonCode(file.contents);
|
|
995
|
+
const bindings = observableSideEffectBindings(file.contents, maskedContents);
|
|
996
|
+
const helpers = localSideEffectHelpers(maskedContents, bindings);
|
|
997
|
+
return memberToolRegistrations(file.contents, maskedContents).some((registration) => {
|
|
998
|
+
if (!/\b(?:execute|handler)\s*(?::|[=(])/.test(registration.declaration)) return false;
|
|
999
|
+
if (dynamicOperationSelector(registration.declaration) === null) return false;
|
|
1000
|
+
const direct = observableSideEffectIndex(registration.declaration, bindings);
|
|
1001
|
+
if (direct !== null) return true;
|
|
1002
|
+
return observableSideEffectIndex(registration.declaration, bindings, helpers) !== null && dynamicOperationSelector(registration.declaration) !== null;
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
function hasSideEffectingToolSurface(file) {
|
|
1006
|
+
if (isExcludedSourcePath(file.path)) return false;
|
|
1007
|
+
const masked = maskNonCode(file.contents);
|
|
1008
|
+
const bindings = observableSideEffectBindings(file.contents, masked);
|
|
1009
|
+
for (const registration of masked.matchAll(TOOL_ASSIGNMENT_PATTERN)) {
|
|
1010
|
+
const toolName = registration[1];
|
|
1011
|
+
if (!toolName) continue;
|
|
1012
|
+
const openParen = (registration.index ?? 0) + registration[0].lastIndexOf("(");
|
|
1013
|
+
const end = matchingCallEnd(file.contents, openParen);
|
|
1014
|
+
if (end === null) continue;
|
|
1015
|
+
if (observableSideEffectIndex(masked.slice(registration.index, end), bindings) !== null && isToolExposed(file.contents, masked, toolName, end)) {
|
|
1016
|
+
return true;
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
return registeredToolSideEffectSurface(file);
|
|
1020
|
+
}
|
|
1021
|
+
function detectSideEffectingToolWithoutAuthorityBoundary(file) {
|
|
1022
|
+
if (isExcludedSourcePath(file.path)) return [];
|
|
1023
|
+
const findings = [];
|
|
1024
|
+
const maskedContents = maskNonCode(file.contents);
|
|
1025
|
+
const sideEffectBindings = observableSideEffectBindings(file.contents, maskedContents);
|
|
1026
|
+
const sideEffectHelpers = localSideEffectHelpers(maskedContents, sideEffectBindings);
|
|
1027
|
+
for (const registration of maskedContents.matchAll(TOOL_ASSIGNMENT_PATTERN)) {
|
|
1028
|
+
const toolName = registration[1];
|
|
1029
|
+
if (!toolName) continue;
|
|
1030
|
+
const openParen = (registration.index ?? 0) + registration[0].lastIndexOf("(");
|
|
1031
|
+
const end = matchingCallEnd(file.contents, openParen);
|
|
1032
|
+
if (end === null) continue;
|
|
1033
|
+
const declaration = maskedContents.slice(registration.index, end);
|
|
1034
|
+
const sideEffectIndex = observableSideEffectIndex(declaration, sideEffectBindings);
|
|
1035
|
+
if (sideEffectIndex === null || !isToolExposed(file.contents, maskedContents, toolName, end)) continue;
|
|
1036
|
+
const beforeSideEffect = declaration.slice(0, sideEffectIndex);
|
|
1037
|
+
if (APPROVAL_FAIL_CLOSED_PATTERN.test(beforeSideEffect) || ALLOWLIST_FAIL_CLOSED_PATTERN.test(beforeSideEffect)) {
|
|
1038
|
+
continue;
|
|
1039
|
+
}
|
|
1040
|
+
findings.push(
|
|
1041
|
+
finding(
|
|
1042
|
+
"LLM-AGY-001",
|
|
1043
|
+
file,
|
|
1044
|
+
(registration.index ?? 0) + sideEffectIndex,
|
|
1045
|
+
"critical",
|
|
1046
|
+
`Exposed tool ${toolName} performs a recognized side effect without an observable fail-closed allowlist or human approval gate.`,
|
|
1047
|
+
`Exposed ${toolName} tool reaches side-effecting operation`,
|
|
1048
|
+
"high"
|
|
1049
|
+
)
|
|
1050
|
+
);
|
|
1051
|
+
}
|
|
1052
|
+
for (const registration of memberToolRegistrations(file.contents, maskedContents)) {
|
|
1053
|
+
if (!/\b(?:execute|handler)\s*(?::|[=(])/.test(registration.declaration)) continue;
|
|
1054
|
+
const directSideEffectIndex = observableSideEffectIndex(
|
|
1055
|
+
registration.declaration,
|
|
1056
|
+
sideEffectBindings
|
|
1057
|
+
);
|
|
1058
|
+
const helperSideEffectIndex = directSideEffectIndex === null ? observableSideEffectIndex(
|
|
1059
|
+
registration.declaration,
|
|
1060
|
+
sideEffectBindings,
|
|
1061
|
+
sideEffectHelpers
|
|
1062
|
+
) : null;
|
|
1063
|
+
const selector = dynamicOperationSelector(registration.declaration);
|
|
1064
|
+
const sideEffectIndex = directSideEffectIndex ?? helperSideEffectIndex;
|
|
1065
|
+
if (sideEffectIndex === null) continue;
|
|
1066
|
+
if (selector === null) continue;
|
|
1067
|
+
const beforeSideEffect = registration.declaration.slice(0, sideEffectIndex);
|
|
1068
|
+
if (APPROVAL_FAIL_CLOSED_PATTERN.test(beforeSideEffect) || ALLOWLIST_FAIL_CLOSED_PATTERN.test(beforeSideEffect)) {
|
|
1069
|
+
continue;
|
|
1070
|
+
}
|
|
1071
|
+
findings.push(
|
|
1072
|
+
finding(
|
|
1073
|
+
"LLM-AGY-001",
|
|
1074
|
+
file,
|
|
1075
|
+
registration.index,
|
|
1076
|
+
"critical",
|
|
1077
|
+
`A locally registered model-facing tool performs a recognized side effect without an observable fail-closed allowlist or human approval gate.`,
|
|
1078
|
+
"Registered tool reaches an import-resolved side-effecting operation",
|
|
1079
|
+
"high"
|
|
1080
|
+
)
|
|
1081
|
+
);
|
|
1082
|
+
}
|
|
1083
|
+
return findings;
|
|
1084
|
+
}
|
|
1085
|
+
var CEJEL_LLM_ACTION_RULES = [
|
|
1086
|
+
{
|
|
1087
|
+
id: "LLM-VAL-001",
|
|
1088
|
+
title: "Consequential structured action lacks validation",
|
|
1089
|
+
detectorConfidence: "high",
|
|
1090
|
+
evidenceContract: "A direct local JSON.parse of recognized model output supplies an object field to a named consequential dispatcher without an intervening fail-closed runtime parse or allowlist guard.",
|
|
1091
|
+
exclusions: [
|
|
1092
|
+
"Inter-procedural or dynamically assembled dispatch",
|
|
1093
|
+
"Read-only rendering and fixed operations",
|
|
1094
|
+
"Runtime validation whose implementation is external or cannot be resolved locally"
|
|
1095
|
+
],
|
|
1096
|
+
applies: hasConsequentialStructuredActionSurface,
|
|
1097
|
+
detect: detectUnvalidatedConsequentialAction
|
|
1098
|
+
},
|
|
1099
|
+
{
|
|
1100
|
+
id: "LLM-AGY-001",
|
|
1101
|
+
title: "Side-effecting tool lacks an authority boundary",
|
|
1102
|
+
detectorConfidence: "high",
|
|
1103
|
+
evidenceContract: "A complete local tool declaration calls an import-resolved Node filesystem or child-process mutation API and is exposed through a local tools registry without a preceding fail-closed approval or operation-allowlist guard.",
|
|
1104
|
+
exclusions: [
|
|
1105
|
+
"Read-only and pure tools",
|
|
1106
|
+
"Unbound business-operation names whose side effects cannot be established locally",
|
|
1107
|
+
"Dynamic or remotely supplied registries and policy middleware",
|
|
1108
|
+
"Tool declarations not observably exposed to a model or agent",
|
|
1109
|
+
"Generic registerTool calls or agent objects without an import-resolved supported model call"
|
|
1110
|
+
],
|
|
1111
|
+
applies: hasSideEffectingToolSurface,
|
|
1112
|
+
detect: detectSideEffectingToolWithoutAuthorityBoundary
|
|
1113
|
+
}
|
|
1114
|
+
];
|
|
1115
|
+
|
|
1116
|
+
// src/packs/llm/evaluation-rules.ts
|
|
1117
|
+
import { isAbsolute } from "path";
|
|
1118
|
+
|
|
1119
|
+
// src/packs/llm/python-evaluation-rules.ts
|
|
1120
|
+
function lineNumberAt2(contents, index) {
|
|
1121
|
+
return contents.slice(0, index).split("\n").length;
|
|
1122
|
+
}
|
|
1123
|
+
function escaped(identifier) {
|
|
1124
|
+
return identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1125
|
+
}
|
|
1126
|
+
function pythonClassBlocks(masked) {
|
|
1127
|
+
const declarations = [...masked.matchAll(
|
|
1128
|
+
/^([ \t]*)class\s+[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?\s*:/gm
|
|
1129
|
+
)];
|
|
1130
|
+
return declarations.map((declaration, declarationIndex) => {
|
|
1131
|
+
const indentation = declaration[1]?.length ?? 0;
|
|
1132
|
+
const next = declarations.slice(declarationIndex + 1).find(
|
|
1133
|
+
(candidate) => (candidate[1]?.length ?? 0) <= indentation
|
|
1134
|
+
);
|
|
1135
|
+
return {
|
|
1136
|
+
index: declaration.index,
|
|
1137
|
+
contents: masked.slice(declaration.index, next?.index ?? masked.length)
|
|
1138
|
+
};
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
function detectPythonConfiguredSelfJudge(file) {
|
|
1142
|
+
if (!file.path.toLowerCase().endsWith(".py") || isExcludedLlmSourcePath(file.path)) return [];
|
|
1143
|
+
const masked = maskPythonNonCode(file.contents);
|
|
1144
|
+
for (const classBlock of pythonClassBlocks(masked)) {
|
|
1145
|
+
const classContents = classBlock.contents;
|
|
1146
|
+
for (const alias of classContents.matchAll(
|
|
1147
|
+
/\bif\s+([A-Za-z_][A-Za-z0-9_]*)\s+is\s+None\s*:\s*\n[ \t]+\1\s*=\s*([A-Za-z_][A-Za-z0-9_]*)/gi
|
|
1148
|
+
)) {
|
|
1149
|
+
const judgeParameter = alias[1];
|
|
1150
|
+
const producerParameter = alias[2];
|
|
1151
|
+
if (!judgeParameter || !producerParameter) continue;
|
|
1152
|
+
if (!/judge/i.test(judgeParameter) || !/(?:llm|model)/i.test(producerParameter)) {
|
|
1153
|
+
continue;
|
|
1154
|
+
}
|
|
1155
|
+
const invokedAssignment = (parameter) => [...classContents.matchAll(
|
|
1156
|
+
new RegExp(
|
|
1157
|
+
`\\bself\\.([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*${escaped(parameter)}\\b`,
|
|
1158
|
+
"gi"
|
|
1159
|
+
)
|
|
1160
|
+
)].map((candidate) => candidate[1]).find(
|
|
1161
|
+
(attribute) => Boolean(attribute) && new RegExp(
|
|
1162
|
+
`\\bself\\.${escaped(attribute ?? "")}\\.(?:ainvoke|invoke|acomplete|complete)\\s*\\(`
|
|
1163
|
+
).test(classContents)
|
|
1164
|
+
);
|
|
1165
|
+
const judgeAttribute = invokedAssignment(judgeParameter);
|
|
1166
|
+
const producerAttribute = invokedAssignment(producerParameter);
|
|
1167
|
+
if (!judgeAttribute || !producerAttribute || judgeAttribute === producerAttribute) continue;
|
|
1168
|
+
if (!/judge/i.test(judgeAttribute)) continue;
|
|
1169
|
+
const judgeInvoked = new RegExp(
|
|
1170
|
+
`\\bself\\.${escaped(judgeAttribute)}\\.(?:ainvoke|invoke|acomplete|complete)\\s*\\(`
|
|
1171
|
+
).test(classContents);
|
|
1172
|
+
const producerInvoked = new RegExp(
|
|
1173
|
+
`\\bself\\.${escaped(producerAttribute)}\\.(?:ainvoke|invoke|acomplete|complete)\\s*\\(`
|
|
1174
|
+
).test(classContents);
|
|
1175
|
+
if (!judgeInvoked || !producerInvoked) continue;
|
|
1176
|
+
const verdictAssignment = [...classContents.matchAll(
|
|
1177
|
+
/\b([A-Za-z_][A-Za-z0-9_]*)\s*=\s*await\s+self\.([A-Za-z_][A-Za-z0-9_]*)\s*\(/gi
|
|
1178
|
+
)].find((candidate) => /judge/i.test(candidate[2] ?? ""));
|
|
1179
|
+
const verdictIdentifier = verdictAssignment?.[1];
|
|
1180
|
+
if (!verdictIdentifier || verdictAssignment.index === void 0) continue;
|
|
1181
|
+
const verdictTailWithLaterMethods = classContents.slice(
|
|
1182
|
+
verdictAssignment.index + verdictAssignment[0].length
|
|
1183
|
+
);
|
|
1184
|
+
const nextMethod = /\n[ \t]+(?:async\s+)?def\s+[A-Za-z_][A-Za-z0-9_]*\s*\(/.exec(
|
|
1185
|
+
verdictTailWithLaterMethods
|
|
1186
|
+
);
|
|
1187
|
+
const verdictTail = nextMethod ? verdictTailWithLaterMethods.slice(0, nextMethod.index) : verdictTailWithLaterMethods;
|
|
1188
|
+
const directlyRetainedVerdict = new RegExp(
|
|
1189
|
+
`\\bself\\.[A-Za-z_][A-Za-z0-9_.\\[\\]-]*\\.(?:judgement|judgment|verdict|judge_result)\\s*=\\s*${escaped(verdictIdentifier)}\\b`,
|
|
1190
|
+
"i"
|
|
1191
|
+
).test(verdictTail);
|
|
1192
|
+
const retainedThroughLocalResult = [...verdictTail.matchAll(
|
|
1193
|
+
/\b([A-Za-z_][A-Za-z0-9_]*)\s*=\s*self\.[A-Za-z_][A-Za-z0-9_.\[\]-]*/g
|
|
1194
|
+
)].some(
|
|
1195
|
+
(resultAlias) => Boolean(resultAlias[1]) && new RegExp(
|
|
1196
|
+
`\\b${escaped(resultAlias[1] ?? "")}\\.(?:judgement|judgment|verdict|judge_result)\\s*=\\s*${escaped(verdictIdentifier)}\\b`,
|
|
1197
|
+
"i"
|
|
1198
|
+
).test(verdictTail)
|
|
1199
|
+
);
|
|
1200
|
+
const retainedVerdict = directlyRetainedVerdict || retainedThroughLocalResult;
|
|
1201
|
+
if (!retainedVerdict) continue;
|
|
1202
|
+
const completionMatch = [...classContents.matchAll(
|
|
1203
|
+
/\bif\s+self\.[A-Za-z_][A-Za-z0-9_.]*use_judge[A-Za-z0-9_]*\s*:\s*\n[ \t]+await\s+self\.([A-Za-z_][A-Za-z0-9_]*)\s*\(/gi
|
|
1204
|
+
)].at(-1);
|
|
1205
|
+
const completionJudge = Boolean(
|
|
1206
|
+
completionMatch?.[1] && /judge/i.test(completionMatch[1])
|
|
1207
|
+
);
|
|
1208
|
+
if (!completionJudge) continue;
|
|
1209
|
+
const completionIndex = completionMatch?.index;
|
|
1210
|
+
if (completionIndex === void 0) continue;
|
|
1211
|
+
return [{
|
|
1212
|
+
ruleId: "LLM-EVL-002",
|
|
1213
|
+
severity: "warning",
|
|
1214
|
+
confidence: "high",
|
|
1215
|
+
summary: "A Python evaluation instance defaults its judge to the producer model and retains that judge verdict as the completed result without an independently configured judge.",
|
|
1216
|
+
evidence: {
|
|
1217
|
+
path: file.path,
|
|
1218
|
+
line: lineNumberAt2(
|
|
1219
|
+
file.contents,
|
|
1220
|
+
classBlock.index + completionIndex
|
|
1221
|
+
),
|
|
1222
|
+
label: "Configured judge aliases the producer model through completion"
|
|
1223
|
+
}
|
|
1224
|
+
}];
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
return [];
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
// src/packs/llm/evaluation-rules.ts
|
|
1231
|
+
var RESULT_EMITTER_PATTERN = /\b(?:writeFileSync|appendFileSync|Bun\.write)\s*\(/g;
|
|
1232
|
+
var MODEL_CALL_PATTERN = /(?:\.responses\.create|\.chat\.completions\.create|\.messages\.create|\b(?:generateText|streamText))\s*\(/g;
|
|
1233
|
+
var ASSIGNMENT_PATTERN = /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*([^;\n]+)/g;
|
|
1234
|
+
var DENOMINATOR_ALIAS_PATTERN = /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*([A-Za-z_$][\w$]*)\.length\s*;/g;
|
|
1235
|
+
var AGGREGATE_NAME_PATTERN = /(?:rate|score|accuracy|average|mean|percentage|pct)/i;
|
|
1236
|
+
var MODEL_LITERAL_PATTERN = /\bmodel\s*:\s*(['"`])([^'"`$]+)\1/;
|
|
1237
|
+
var MODEL_IDENTIFIER_PATTERN = /\bmodel\s*:\s*([A-Za-z_$][\w$]*)/;
|
|
1238
|
+
var LITERAL_BINDING_PATTERN = /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(['"`])([^'"`$]+)\2\s*;/g;
|
|
1239
|
+
var LINEAGE_MODEL_KEYS = /* @__PURE__ */ new Set([
|
|
1240
|
+
"model",
|
|
1241
|
+
"modelId",
|
|
1242
|
+
"modelVersion",
|
|
1243
|
+
"providerModelId",
|
|
1244
|
+
"producerModel"
|
|
1245
|
+
]);
|
|
1246
|
+
var LINEAGE_CONFIG_KEYS = /* @__PURE__ */ new Set([
|
|
1247
|
+
"promptDigest",
|
|
1248
|
+
"promptHash",
|
|
1249
|
+
"policyDigest",
|
|
1250
|
+
"policyHash",
|
|
1251
|
+
"configDigest",
|
|
1252
|
+
"configHash",
|
|
1253
|
+
"configVersion",
|
|
1254
|
+
"evaluationConfigVersion",
|
|
1255
|
+
"evaluationManifest",
|
|
1256
|
+
"repositoryCommit"
|
|
1257
|
+
]);
|
|
1258
|
+
var DENOMINATOR_KEYS = /* @__PURE__ */ new Set([
|
|
1259
|
+
"eligibleTotal",
|
|
1260
|
+
"eligibleCount",
|
|
1261
|
+
"totalCases",
|
|
1262
|
+
"totalCount",
|
|
1263
|
+
"denominator",
|
|
1264
|
+
"caseResults",
|
|
1265
|
+
"rawResults",
|
|
1266
|
+
"results"
|
|
1267
|
+
]);
|
|
1268
|
+
var OUTCOME_COUNT_PATTERN = /^(?:error|errors|refusal|refusals|abstention|abstentions|excluded|exclusions|excludedCount)$/i;
|
|
1269
|
+
var DISCRETE_CASE_KEY_PATTERN = /^(?:case|caseId|scenario|sample|sampleId|run|runId|input|expected|expectedOutput)$/i;
|
|
1270
|
+
var DISCRETE_OUTCOME_KEY_PATTERN = /^(?:ok|passed|success|successful|correct|score|verdict|result|actualOutput|output|latency|duration|error|failedPairs|contentChanged|editReflected)$/i;
|
|
1271
|
+
var CASE_COLLECTION_KEY_PATTERN = /^(?:cases|rows|samples|caseResults|results)$/i;
|
|
1272
|
+
var CASE_RESULT_KEY_PATTERN = /^(?:status|actualOutput|output|score|verdict|correct|success|error|latency|metrics)$/i;
|
|
1273
|
+
function hasIndependentAcceptanceSignal(file) {
|
|
1274
|
+
const masked = maskJavaScriptNonCode(file.contents);
|
|
1275
|
+
const emissions = localEmissions(file);
|
|
1276
|
+
for (const match of masked.matchAll(
|
|
1277
|
+
/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:await\s+)?(?:humanReview|humanAdjudication|manualReview|evidenceVerification|exactMatch|schemaCheck|propertyCheck|deterministicGrade|independentDecision)\s*\(/g
|
|
1278
|
+
)) {
|
|
1279
|
+
const identifier = match[1];
|
|
1280
|
+
if (!identifier) continue;
|
|
1281
|
+
if (emissions.some((emission) => {
|
|
1282
|
+
if (emission.index <= match.index) return false;
|
|
1283
|
+
const callStart = masked.indexOf("(", emission.index);
|
|
1284
|
+
const callEnd = callStart < 0 ? -1 : matchingDelimiter(masked, callStart, "(", ")");
|
|
1285
|
+
if (callEnd < 0) return false;
|
|
1286
|
+
const escaped3 = identifier.replaceAll("$", "\\$");
|
|
1287
|
+
return new RegExp(`\\b${escaped3}\\b`).test(masked.slice(callStart, callEnd + 1));
|
|
1288
|
+
})) return true;
|
|
1289
|
+
const tail = masked.slice(match.index + match[0].length);
|
|
1290
|
+
const escaped2 = identifier.replaceAll("$", "\\$");
|
|
1291
|
+
const gate = new RegExp(
|
|
1292
|
+
`\\bif\\s*\\([^)]*\\b${escaped2}\\b[^)]*\\)\\s*(?:\\{[\\s\\S]{0,300}?\\b(?:throw|return|writeFileSync|appendFileSync|Bun\\.write)\\b|(?:throw|return)\\b)`
|
|
1293
|
+
);
|
|
1294
|
+
if (gate.test(tail)) return true;
|
|
1295
|
+
}
|
|
1296
|
+
return /\bif\s*\([^)]*(?:humanReview|humanAdjudication|manualReview|evidenceVerification|exactMatch|schemaCheck|propertyCheck|deterministicGrade|independentDecision)\s*\([^)]*\)[^)]*\)\s*(?:\{[\s\S]{0,300}?\b(?:throw|return|writeFileSync|appendFileSync|Bun\.write)\b|(?:throw|return)\b)/.test(
|
|
1297
|
+
masked
|
|
1298
|
+
);
|
|
1299
|
+
}
|
|
1300
|
+
function completeLocalSource(file) {
|
|
1301
|
+
if (!file.path || isAbsolute(file.path)) return false;
|
|
1302
|
+
const segments = file.path.replaceAll("\\", "/").split("/");
|
|
1303
|
+
if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) return false;
|
|
1304
|
+
return !segments.some(
|
|
1305
|
+
(segment) => /^(?:test|tests|__tests__|fixtures?|examples?|docs?|node_modules|vendor|generated)$/i.test(
|
|
1306
|
+
segment
|
|
1307
|
+
)
|
|
1308
|
+
);
|
|
1309
|
+
}
|
|
1310
|
+
function hasSupportedEvaluationImport(file) {
|
|
1311
|
+
return hasUnmaskedJavaScriptMatch(
|
|
1312
|
+
file.contents,
|
|
1313
|
+
/(?:from\s+['"](?:openai|@anthropic-ai\/sdk|ai)['"]|require\(\s*['"](?:openai|@anthropic-ai\/sdk|ai)['"]\s*\))/
|
|
1314
|
+
);
|
|
1315
|
+
}
|
|
1316
|
+
function lineNumberAt3(contents, index) {
|
|
1317
|
+
return contents.slice(0, index).split("\n").length;
|
|
1318
|
+
}
|
|
1319
|
+
function matchingDelimiter(contents, start, open, close) {
|
|
1320
|
+
let depth = 0;
|
|
1321
|
+
let quote = null;
|
|
1322
|
+
let escaped2 = false;
|
|
1323
|
+
for (let index = start; index < contents.length; index += 1) {
|
|
1324
|
+
const character = contents[index];
|
|
1325
|
+
if (!character) break;
|
|
1326
|
+
if (quote) {
|
|
1327
|
+
if (escaped2) escaped2 = false;
|
|
1328
|
+
else if (character === "\\") escaped2 = true;
|
|
1329
|
+
else if (character === quote) quote = null;
|
|
1330
|
+
continue;
|
|
1331
|
+
}
|
|
1332
|
+
if (character === '"' || character === "'" || character === "`") {
|
|
1333
|
+
quote = character;
|
|
1334
|
+
continue;
|
|
1335
|
+
}
|
|
1336
|
+
if (character === open) depth += 1;
|
|
1337
|
+
if (character === close) {
|
|
1338
|
+
depth -= 1;
|
|
1339
|
+
if (depth === 0) return index;
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
return -1;
|
|
1343
|
+
}
|
|
1344
|
+
function objectProperties(objectBody) {
|
|
1345
|
+
const properties = /* @__PURE__ */ new Set();
|
|
1346
|
+
for (const match of objectBody.matchAll(/(?:^|,)\s*([A-Za-z_$][\w$]*)\s*(?=:|,|$)/g)) {
|
|
1347
|
+
const property = match[1];
|
|
1348
|
+
if (property) properties.add(property);
|
|
1349
|
+
}
|
|
1350
|
+
return properties;
|
|
1351
|
+
}
|
|
1352
|
+
function boundObjectLiterals(contents) {
|
|
1353
|
+
const objects = [];
|
|
1354
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
1355
|
+
for (const match of contents.matchAll(
|
|
1356
|
+
/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)(?:\s*:[^=\n]+)?\s*=\s*\{/g
|
|
1357
|
+
)) {
|
|
1358
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
1359
|
+
const name = match[1];
|
|
1360
|
+
if (!name) continue;
|
|
1361
|
+
const start = contents.indexOf("{", match.index);
|
|
1362
|
+
const end = matchingDelimiter(contents, start, "{", "}");
|
|
1363
|
+
if (end < 0) continue;
|
|
1364
|
+
objects.push({
|
|
1365
|
+
name,
|
|
1366
|
+
index: match.index,
|
|
1367
|
+
end,
|
|
1368
|
+
properties: objectProperties(masked.slice(start + 1, end))
|
|
1369
|
+
});
|
|
1370
|
+
}
|
|
1371
|
+
return objects;
|
|
1372
|
+
}
|
|
1373
|
+
function resolvedIdentifierEmissionOrReturnIndex(contents, identifier, afterIndex) {
|
|
1374
|
+
const escaped2 = identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1375
|
+
const tail = maskJavaScriptNonCode(contents.slice(afterIndex));
|
|
1376
|
+
const redeclaration = new RegExp(
|
|
1377
|
+
`\\b(?:const|let|var)\\s+${escaped2}\\b`
|
|
1378
|
+
).exec(tail);
|
|
1379
|
+
const observableTail = redeclaration ? tail.slice(0, redeclaration.index) : tail;
|
|
1380
|
+
const serializedEmission = new RegExp(
|
|
1381
|
+
`\\b(?:writeFileSync|appendFileSync|Bun\\.write|console\\.(?:log|info))\\s*\\([\\s\\S]{0,500}?JSON\\.stringify\\s*\\(\\s*${escaped2}\\b`
|
|
1382
|
+
).exec(observableTail);
|
|
1383
|
+
const returned = new RegExp(`\\breturn\\s+${escaped2}\\s*;?`).exec(observableTail);
|
|
1384
|
+
const indices = [serializedEmission?.index, returned?.index].filter((index) => index !== void 0).map((index) => afterIndex + index);
|
|
1385
|
+
return indices.length > 0 ? Math.min(...indices) : null;
|
|
1386
|
+
}
|
|
1387
|
+
function assignedProperties(contents, identifier, afterIndex, beforeIndex = contents.length) {
|
|
1388
|
+
const properties = /* @__PURE__ */ new Set();
|
|
1389
|
+
const escaped2 = identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1390
|
+
const maskedTail = maskJavaScriptNonCode(contents.slice(afterIndex, beforeIndex));
|
|
1391
|
+
const redeclaration = new RegExp(
|
|
1392
|
+
`\\b(?:const|let|var)\\s+${escaped2}\\b`
|
|
1393
|
+
).exec(maskedTail);
|
|
1394
|
+
const masked = redeclaration ? maskedTail.slice(0, redeclaration.index) : maskedTail;
|
|
1395
|
+
const pattern = new RegExp(`\\b${escaped2}\\.([A-Za-z_$][\\w$]*)\\s*=`, "g");
|
|
1396
|
+
for (const match of masked.matchAll(pattern)) {
|
|
1397
|
+
if (match[1]) properties.add(match[1]);
|
|
1398
|
+
}
|
|
1399
|
+
return properties;
|
|
1400
|
+
}
|
|
1401
|
+
function configurationInputLocus(contents, beforeIndex) {
|
|
1402
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
1403
|
+
let earliest = null;
|
|
1404
|
+
for (const declaration of contents.matchAll(
|
|
1405
|
+
/\bimport\s*\{([^}]+)\}\s*from\s*['"][^'"]+['"]/g
|
|
1406
|
+
)) {
|
|
1407
|
+
if (declaration.index >= beforeIndex || (masked[declaration.index] ?? " ") === " ") continue;
|
|
1408
|
+
for (const member of (declaration[1] ?? "").split(",")) {
|
|
1409
|
+
const binding = member.trim().match(
|
|
1410
|
+
/^(?:type\s+)?([A-Za-z_$][\w$]*)(?:\s+as\s+([A-Za-z_$][\w$]*))?$/
|
|
1411
|
+
);
|
|
1412
|
+
const local = binding?.[2] ?? binding?.[1];
|
|
1413
|
+
if (!local || !/(?:prompt|policy|config|system)/i.test(local)) continue;
|
|
1414
|
+
const tail = masked.slice(declaration.index + declaration[0].length, beforeIndex);
|
|
1415
|
+
if (!new RegExp(`\\b${local.replaceAll("$", "\\$")}\\b`).test(tail)) continue;
|
|
1416
|
+
earliest = earliest === null ? declaration.index : Math.min(earliest, declaration.index);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
return earliest;
|
|
1420
|
+
}
|
|
1421
|
+
function javaScriptFunctionScopes(masked) {
|
|
1422
|
+
const scopes = [];
|
|
1423
|
+
const patterns = [
|
|
1424
|
+
/^\s*(?:async\s+)?(?:function\s+)?([A-Za-z_$][\w$]*)\s*\(([^)]*)\)\s*(?::[^{\n]+)?\s*\{/gm,
|
|
1425
|
+
/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\([^)]*\)\s*(?::[^=\n]+)?=>\s*\{/g
|
|
1426
|
+
];
|
|
1427
|
+
for (const pattern of patterns) {
|
|
1428
|
+
for (const declaration of masked.matchAll(pattern)) {
|
|
1429
|
+
const functionName = declaration[1];
|
|
1430
|
+
if (!functionName || /^(?:if|for|while|switch|catch|with)$/i.test(functionName)) {
|
|
1431
|
+
continue;
|
|
1432
|
+
}
|
|
1433
|
+
const start = masked.indexOf("{", declaration.index);
|
|
1434
|
+
const end = matchingDelimiter(masked, start, "{", "}");
|
|
1435
|
+
if (start >= 0 && end >= 0) scopes.push({ start, end });
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
return scopes;
|
|
1439
|
+
}
|
|
1440
|
+
function hasScopedInvocationBefore(contents, invocations, resultIndex) {
|
|
1441
|
+
const scopes = javaScriptFunctionScopes(maskJavaScriptNonCode(contents));
|
|
1442
|
+
const resultScope = scopes.filter((scope) => resultIndex > scope.start && resultIndex < scope.end).sort((left, right) => right.start - left.start)[0];
|
|
1443
|
+
if (resultScope) {
|
|
1444
|
+
return invocations.some(
|
|
1445
|
+
(invocation) => invocation.index > resultScope.start && invocation.index < resultIndex && !scopes.some(
|
|
1446
|
+
(scope) => scope.start > resultScope.start && scope.end < resultScope.end && invocation.index > scope.start && invocation.index < scope.end
|
|
1447
|
+
)
|
|
1448
|
+
);
|
|
1449
|
+
}
|
|
1450
|
+
return invocations.some(
|
|
1451
|
+
(invocation) => invocation.index < resultIndex && !scopes.some(
|
|
1452
|
+
(scope) => invocation.index > scope.start && invocation.index < scope.end
|
|
1453
|
+
)
|
|
1454
|
+
);
|
|
1455
|
+
}
|
|
1456
|
+
function parameterBackedPerCaseProvenanceFinding(file, objects, invocations) {
|
|
1457
|
+
const masked = maskJavaScriptNonCode(file.contents);
|
|
1458
|
+
for (const declaration of masked.matchAll(
|
|
1459
|
+
/^\s*(?:async\s+)?(?:function\s+)?([A-Za-z_$][\w$]*)\s*\(([^)]*)\)\s*(?::[^{\n]+)?\s*\{/gm
|
|
1460
|
+
)) {
|
|
1461
|
+
const functionName = declaration[1];
|
|
1462
|
+
if (!functionName || /^(?:if|for|while|switch|catch|with)$/i.test(functionName)) {
|
|
1463
|
+
continue;
|
|
1464
|
+
}
|
|
1465
|
+
const openBrace = masked.indexOf("{", declaration.index);
|
|
1466
|
+
const end = matchingDelimiter(masked, openBrace, "{", "}");
|
|
1467
|
+
if (end < 0) continue;
|
|
1468
|
+
const body = masked.slice(openBrace + 1, end);
|
|
1469
|
+
const parameters = (declaration[2] ?? "").split(",").map((parameter) => parameter.trim().match(/^([A-Za-z_$][\w$]*)/)?.[1]).filter((parameter) => Boolean(parameter));
|
|
1470
|
+
for (const root of parameters) {
|
|
1471
|
+
const escapedRoot = root.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1472
|
+
const returned = new RegExp(`\\breturn\\s+${escapedRoot}\\s*;?`).exec(body);
|
|
1473
|
+
if (!returned) continue;
|
|
1474
|
+
const returnIndex = openBrace + 1 + returned.index;
|
|
1475
|
+
if (!invocations.some(
|
|
1476
|
+
(invocation) => invocation.index > openBrace && invocation.index < returnIndex
|
|
1477
|
+
)) {
|
|
1478
|
+
continue;
|
|
1479
|
+
}
|
|
1480
|
+
for (const candidate of objects) {
|
|
1481
|
+
if (candidate.index <= openBrace || candidate.index >= end) continue;
|
|
1482
|
+
const escapedCase = candidate.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1483
|
+
const push = new RegExp(
|
|
1484
|
+
`\\b${escapedRoot}(?:\\.[A-Za-z_$][\\w$]*|\\[[^\\]]+\\]){1,5}\\.push\\s*\\(\\s*${escapedCase}\\s*\\)`
|
|
1485
|
+
).exec(body);
|
|
1486
|
+
if (!push) continue;
|
|
1487
|
+
const pushIndex = openBrace + 1 + push.index;
|
|
1488
|
+
const properties = assignedProperties(
|
|
1489
|
+
file.contents,
|
|
1490
|
+
candidate.name,
|
|
1491
|
+
candidate.end,
|
|
1492
|
+
pushIndex
|
|
1493
|
+
);
|
|
1494
|
+
const hasEvaluationIdentity = [...properties].some(
|
|
1495
|
+
(property) => /^(?:uuid|evaluationId|evaluationRunId|runId|caseId)$/i.test(property)
|
|
1496
|
+
);
|
|
1497
|
+
const resultPropertyCount = [...properties].filter(
|
|
1498
|
+
(property) => CASE_RESULT_KEY_PATTERN.test(property)
|
|
1499
|
+
).length;
|
|
1500
|
+
if (!hasEvaluationIdentity || resultPropertyCount < 3) continue;
|
|
1501
|
+
const allProperties = /* @__PURE__ */ new Set([...candidate.properties, ...properties]);
|
|
1502
|
+
if (hasAny(allProperties, LINEAGE_CONFIG_KEYS)) continue;
|
|
1503
|
+
return finding2(
|
|
1504
|
+
"LLM-PRV-001",
|
|
1505
|
+
file,
|
|
1506
|
+
candidate.index,
|
|
1507
|
+
"info",
|
|
1508
|
+
"A parameter-backed evaluation collection retains per-case results without prompt/policy evaluation-configuration lineage.",
|
|
1509
|
+
"Per-case evaluation record stored without reproducible configuration provenance"
|
|
1510
|
+
);
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
return null;
|
|
1515
|
+
}
|
|
1516
|
+
function detectBoundedEvaluationResultProvenance(files) {
|
|
1517
|
+
const findings = [];
|
|
1518
|
+
for (const file of files) {
|
|
1519
|
+
if (!completeLocalSource(file) || !hasSupportedEvaluationImport(file)) continue;
|
|
1520
|
+
const invocations = modelInvocations(file.contents);
|
|
1521
|
+
if (invocations.length === 0) continue;
|
|
1522
|
+
const findingsBeforeFile = findings.length;
|
|
1523
|
+
const objects = boundObjectLiterals(file.contents);
|
|
1524
|
+
let found = false;
|
|
1525
|
+
for (const object of objects) {
|
|
1526
|
+
const emissionIndex = resolvedIdentifierEmissionOrReturnIndex(
|
|
1527
|
+
file.contents,
|
|
1528
|
+
object.name,
|
|
1529
|
+
object.end
|
|
1530
|
+
);
|
|
1531
|
+
if (emissionIndex === null || !hasScopedInvocationBefore(file.contents, invocations, emissionIndex)) {
|
|
1532
|
+
continue;
|
|
1533
|
+
}
|
|
1534
|
+
const hasModel = hasAny(object.properties, LINEAGE_MODEL_KEYS);
|
|
1535
|
+
const hasCase = [...object.properties].some(
|
|
1536
|
+
(property) => DISCRETE_CASE_KEY_PATTERN.test(property)
|
|
1537
|
+
);
|
|
1538
|
+
const outcomeCount = [...object.properties].filter(
|
|
1539
|
+
(property) => DISCRETE_OUTCOME_KEY_PATTERN.test(property)
|
|
1540
|
+
).length;
|
|
1541
|
+
const hasConfiguration = hasAny(object.properties, LINEAGE_CONFIG_KEYS);
|
|
1542
|
+
if (!hasModel || !hasCase || outcomeCount < 2 || hasConfiguration) continue;
|
|
1543
|
+
findings.push(
|
|
1544
|
+
finding2(
|
|
1545
|
+
"LLM-PRV-001",
|
|
1546
|
+
file,
|
|
1547
|
+
configurationInputLocus(file.contents, object.index) ?? object.index,
|
|
1548
|
+
"info",
|
|
1549
|
+
"A locally emitted discrete LLM evaluation result does not retain prompt/policy evaluation-configuration lineage.",
|
|
1550
|
+
"Discrete evaluation result emitted without reproducible configuration provenance"
|
|
1551
|
+
)
|
|
1552
|
+
);
|
|
1553
|
+
found = true;
|
|
1554
|
+
break;
|
|
1555
|
+
}
|
|
1556
|
+
if (found) continue;
|
|
1557
|
+
for (const root of objects) {
|
|
1558
|
+
const rootProperties = assignedProperties(file.contents, root.name, root.end);
|
|
1559
|
+
const hasEvaluationIdentity = [...rootProperties].some(
|
|
1560
|
+
(property) => /^(?:evaluationId|evaluationRunId|runId|datasetId)$/i.test(property)
|
|
1561
|
+
);
|
|
1562
|
+
if (!hasEvaluationIdentity) continue;
|
|
1563
|
+
const collection = [...rootProperties].find(
|
|
1564
|
+
(property) => CASE_COLLECTION_KEY_PATTERN.test(property)
|
|
1565
|
+
);
|
|
1566
|
+
if (!collection) continue;
|
|
1567
|
+
const escapedRoot = root.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1568
|
+
const escapedCollection = collection.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1569
|
+
if (!new RegExp(`\\b${escapedRoot}\\.${escapedCollection}\\s*=\\s*\\[\\s*\\]`).test(
|
|
1570
|
+
maskJavaScriptNonCode(file.contents)
|
|
1571
|
+
)) continue;
|
|
1572
|
+
const emissionIndex = resolvedIdentifierEmissionOrReturnIndex(
|
|
1573
|
+
file.contents,
|
|
1574
|
+
root.name,
|
|
1575
|
+
root.end
|
|
1576
|
+
);
|
|
1577
|
+
if (emissionIndex === null || !hasScopedInvocationBefore(file.contents, invocations, emissionIndex)) {
|
|
1578
|
+
continue;
|
|
1579
|
+
}
|
|
1580
|
+
const perCase = objects.find((candidate) => {
|
|
1581
|
+
if (candidate.index <= root.index) return false;
|
|
1582
|
+
const properties = assignedProperties(file.contents, candidate.name, candidate.end);
|
|
1583
|
+
const resultPropertyCount = [...properties].filter(
|
|
1584
|
+
(property) => CASE_RESULT_KEY_PATTERN.test(property)
|
|
1585
|
+
).length;
|
|
1586
|
+
if (resultPropertyCount < 3) return false;
|
|
1587
|
+
const escapedCase = candidate.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1588
|
+
return new RegExp(
|
|
1589
|
+
`\\b${escapedRoot}\\.${escapedCollection}(?:\\[[^\\]]+\\])?(?:\\.[A-Za-z_$][\\w$]*)?\\.push\\s*\\(\\s*${escapedCase}\\s*\\)`
|
|
1590
|
+
).test(maskJavaScriptNonCode(file.contents));
|
|
1591
|
+
});
|
|
1592
|
+
if (!perCase) continue;
|
|
1593
|
+
const allProperties = /* @__PURE__ */ new Set([
|
|
1594
|
+
...rootProperties,
|
|
1595
|
+
...assignedProperties(file.contents, perCase.name, perCase.end)
|
|
1596
|
+
]);
|
|
1597
|
+
if (hasAny(allProperties, LINEAGE_CONFIG_KEYS)) continue;
|
|
1598
|
+
findings.push(
|
|
1599
|
+
finding2(
|
|
1600
|
+
"LLM-PRV-001",
|
|
1601
|
+
file,
|
|
1602
|
+
perCase.index,
|
|
1603
|
+
"info",
|
|
1604
|
+
"A locally returned per-case LLM evaluation result does not retain prompt/policy evaluation-configuration lineage.",
|
|
1605
|
+
"Per-case evaluation results returned without reproducible configuration provenance"
|
|
1606
|
+
)
|
|
1607
|
+
);
|
|
1608
|
+
break;
|
|
1609
|
+
}
|
|
1610
|
+
if (findings.length > findingsBeforeFile) continue;
|
|
1611
|
+
const parameterBacked = parameterBackedPerCaseProvenanceFinding(
|
|
1612
|
+
file,
|
|
1613
|
+
objects,
|
|
1614
|
+
invocations
|
|
1615
|
+
);
|
|
1616
|
+
if (parameterBacked) findings.push(parameterBacked);
|
|
1617
|
+
}
|
|
1618
|
+
return findings;
|
|
1619
|
+
}
|
|
1620
|
+
function directObjectEmission(contents, emitterIndex) {
|
|
1621
|
+
const callStart = contents.indexOf("(", emitterIndex);
|
|
1622
|
+
if (callStart < 0) return null;
|
|
1623
|
+
const callEnd = matchingDelimiter(contents, callStart, "(", ")");
|
|
1624
|
+
if (callEnd < 0) return null;
|
|
1625
|
+
const call = contents.slice(callStart + 1, callEnd);
|
|
1626
|
+
const stringifyIndex = call.indexOf("JSON.stringify");
|
|
1627
|
+
if (stringifyIndex < 0) return null;
|
|
1628
|
+
const objectStart = call.indexOf("{", stringifyIndex);
|
|
1629
|
+
if (objectStart < 0) return null;
|
|
1630
|
+
const objectEnd = matchingDelimiter(call, objectStart, "{", "}");
|
|
1631
|
+
if (objectEnd < 0) return null;
|
|
1632
|
+
return {
|
|
1633
|
+
index: emitterIndex,
|
|
1634
|
+
properties: objectProperties(call.slice(objectStart + 1, objectEnd))
|
|
1635
|
+
};
|
|
1636
|
+
}
|
|
1637
|
+
function localEmissions(file) {
|
|
1638
|
+
const emissions = [];
|
|
1639
|
+
const masked = maskJavaScriptNonCode(file.contents);
|
|
1640
|
+
for (const match of file.contents.matchAll(RESULT_EMITTER_PATTERN)) {
|
|
1641
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
1642
|
+
const emission = directObjectEmission(file.contents, match.index);
|
|
1643
|
+
if (emission) emissions.push(emission);
|
|
1644
|
+
}
|
|
1645
|
+
return emissions;
|
|
1646
|
+
}
|
|
1647
|
+
function hasAny(properties, candidates) {
|
|
1648
|
+
for (const property of properties) if (candidates.has(property)) return true;
|
|
1649
|
+
return false;
|
|
1650
|
+
}
|
|
1651
|
+
function finding2(ruleId, file, index, severity, summary, label) {
|
|
1652
|
+
return {
|
|
1653
|
+
ruleId,
|
|
1654
|
+
severity,
|
|
1655
|
+
confidence: "high",
|
|
1656
|
+
summary,
|
|
1657
|
+
evidence: {
|
|
1658
|
+
path: file.path,
|
|
1659
|
+
line: lineNumberAt3(file.contents, index),
|
|
1660
|
+
label
|
|
1661
|
+
}
|
|
1662
|
+
};
|
|
1663
|
+
}
|
|
1664
|
+
function detectMissingProvenance(files) {
|
|
1665
|
+
const findings = [];
|
|
1666
|
+
for (const file of files) {
|
|
1667
|
+
if (!completeLocalSource(file) || !hasSupportedEvaluationImport(file)) continue;
|
|
1668
|
+
const aggregateAssignments = aggregateAssignmentsIn(file.contents);
|
|
1669
|
+
if (aggregateAssignments.size === 0) continue;
|
|
1670
|
+
for (const emission of localEmissions(file)) {
|
|
1671
|
+
const emittedAggregates = [...aggregateAssignments].filter(
|
|
1672
|
+
([name]) => emission.properties.has(name)
|
|
1673
|
+
);
|
|
1674
|
+
if (emittedAggregates.length === 0) continue;
|
|
1675
|
+
const firstAggregateIndex = Math.min(
|
|
1676
|
+
...emittedAggregates.map(([, assignment]) => assignment.index)
|
|
1677
|
+
);
|
|
1678
|
+
if (!hasLocalLlmEvaluationInvocation(file, firstAggregateIndex)) continue;
|
|
1679
|
+
const hasModelLineage = hasAny(emission.properties, LINEAGE_MODEL_KEYS);
|
|
1680
|
+
const hasConfigurationLineage = hasAny(emission.properties, LINEAGE_CONFIG_KEYS);
|
|
1681
|
+
if (hasModelLineage && hasConfigurationLineage) continue;
|
|
1682
|
+
findings.push(
|
|
1683
|
+
finding2(
|
|
1684
|
+
"LLM-PRV-001",
|
|
1685
|
+
file,
|
|
1686
|
+
emission.index,
|
|
1687
|
+
"info",
|
|
1688
|
+
"A locally emitted LLM evaluation aggregate does not retain both model and prompt/policy evaluation-configuration lineage.",
|
|
1689
|
+
"Evaluation result emitted without complete reproducible system provenance"
|
|
1690
|
+
)
|
|
1691
|
+
);
|
|
1692
|
+
}
|
|
1693
|
+
}
|
|
1694
|
+
return findings;
|
|
1695
|
+
}
|
|
1696
|
+
function detectMissingProvenanceIncludingBoundedResults(files) {
|
|
1697
|
+
const aggregateFindings = detectMissingProvenance(files);
|
|
1698
|
+
const aggregatePaths = new Set(aggregateFindings.map((item) => item.evidence.path));
|
|
1699
|
+
return [
|
|
1700
|
+
...aggregateFindings,
|
|
1701
|
+
...detectBoundedEvaluationResultProvenance(files).filter(
|
|
1702
|
+
(item) => !aggregatePaths.has(item.evidence.path)
|
|
1703
|
+
)
|
|
1704
|
+
];
|
|
1705
|
+
}
|
|
1706
|
+
function aggregateAssignmentsIn(contents) {
|
|
1707
|
+
const assignments = /* @__PURE__ */ new Map();
|
|
1708
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
1709
|
+
for (const match of contents.matchAll(ASSIGNMENT_PATTERN)) {
|
|
1710
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
1711
|
+
const name = match[1];
|
|
1712
|
+
const expression = match[2];
|
|
1713
|
+
if (!name || !expression) continue;
|
|
1714
|
+
if (!AGGREGATE_NAME_PATTERN.test(name)) continue;
|
|
1715
|
+
if (/\.length\s*\/|\/\s*[A-Za-z_$][\w$]*\.length\b|\.reduce\s*\(/.test(expression)) {
|
|
1716
|
+
const denominatorCollections = /* @__PURE__ */ new Set();
|
|
1717
|
+
for (const denominator of expression.matchAll(
|
|
1718
|
+
/\/\s*([A-Za-z_$][\w$]*)\.length\b/g
|
|
1719
|
+
)) {
|
|
1720
|
+
if (denominator[1]) denominatorCollections.add(denominator[1]);
|
|
1721
|
+
}
|
|
1722
|
+
assignments.set(name, { index: match.index, denominatorCollections });
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
return assignments;
|
|
1726
|
+
}
|
|
1727
|
+
function denominatorAliasAssignments(contents) {
|
|
1728
|
+
const assignments = /* @__PURE__ */ new Map();
|
|
1729
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
1730
|
+
for (const match of contents.matchAll(DENOMINATOR_ALIAS_PATTERN)) {
|
|
1731
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
1732
|
+
const alias = match[1];
|
|
1733
|
+
const collection = match[2];
|
|
1734
|
+
if (alias && collection) assignments.set(alias, { index: match.index, collection });
|
|
1735
|
+
}
|
|
1736
|
+
return assignments;
|
|
1737
|
+
}
|
|
1738
|
+
function hasOutcomeCounts(properties) {
|
|
1739
|
+
return [...properties].some((property) => OUTCOME_COUNT_PATTERN.test(property));
|
|
1740
|
+
}
|
|
1741
|
+
function hasLocalLlmEvaluationInvocation(file, beforeIndex) {
|
|
1742
|
+
return modelInvocations(file.contents).some((invocation) => invocation.index < beforeIndex);
|
|
1743
|
+
}
|
|
1744
|
+
function detectMissingDenominator(files) {
|
|
1745
|
+
const findings = [];
|
|
1746
|
+
for (const file of files) {
|
|
1747
|
+
if (!completeLocalSource(file) || !hasSupportedEvaluationImport(file)) continue;
|
|
1748
|
+
const aggregateAssignments = aggregateAssignmentsIn(file.contents);
|
|
1749
|
+
const denominatorAliases = denominatorAliasAssignments(file.contents);
|
|
1750
|
+
if (aggregateAssignments.size === 0) continue;
|
|
1751
|
+
const iteratesCases = /\.(?:map|filter|reduce|forEach)\s*\(|\bfor\s*\([^)]*\bof\b/.test(
|
|
1752
|
+
file.contents
|
|
1753
|
+
);
|
|
1754
|
+
if (!iteratesCases) continue;
|
|
1755
|
+
for (const emission of localEmissions(file)) {
|
|
1756
|
+
const emittedAggregates = [...aggregateAssignments].filter(
|
|
1757
|
+
([name]) => emission.properties.has(name)
|
|
1758
|
+
);
|
|
1759
|
+
if (emittedAggregates.length === 0) continue;
|
|
1760
|
+
const firstAggregateIndex = Math.min(
|
|
1761
|
+
...emittedAggregates.map(([, assignment]) => assignment.index)
|
|
1762
|
+
);
|
|
1763
|
+
if (!hasLocalLlmEvaluationInvocation(file, firstAggregateIndex)) continue;
|
|
1764
|
+
if (hasAny(emission.properties, DENOMINATOR_KEYS)) continue;
|
|
1765
|
+
const emittedAliasHasLineage = [...emission.properties].some((property) => {
|
|
1766
|
+
const alias = denominatorAliases.get(property);
|
|
1767
|
+
if (!alias || alias.index >= emission.index) return false;
|
|
1768
|
+
return emittedAggregates.some(
|
|
1769
|
+
([, aggregate]) => aggregate.denominatorCollections.has(alias.collection)
|
|
1770
|
+
);
|
|
1771
|
+
});
|
|
1772
|
+
if (emittedAliasHasLineage) continue;
|
|
1773
|
+
if (hasOutcomeCounts(emission.properties)) continue;
|
|
1774
|
+
findings.push(
|
|
1775
|
+
finding2(
|
|
1776
|
+
"LLM-EVL-001",
|
|
1777
|
+
file,
|
|
1778
|
+
emission.index,
|
|
1779
|
+
"warning",
|
|
1780
|
+
"A locally computed evaluation aggregate is emitted without its eligible-case denominator or raw case-level results.",
|
|
1781
|
+
"Evaluation aggregate emitted without reproducible denominator"
|
|
1782
|
+
)
|
|
1783
|
+
);
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
return findings;
|
|
1787
|
+
}
|
|
1788
|
+
function literalBindings(contents) {
|
|
1789
|
+
const bindings = /* @__PURE__ */ new Map();
|
|
1790
|
+
for (const match of contents.matchAll(LITERAL_BINDING_PATTERN)) {
|
|
1791
|
+
const identifier = match[1];
|
|
1792
|
+
const value = match[3];
|
|
1793
|
+
if (identifier && value) bindings.set(identifier, value);
|
|
1794
|
+
}
|
|
1795
|
+
return bindings;
|
|
1796
|
+
}
|
|
1797
|
+
function modelIdentity(call, bindings) {
|
|
1798
|
+
const literal = call.match(MODEL_LITERAL_PATTERN)?.[2];
|
|
1799
|
+
if (literal) return literal;
|
|
1800
|
+
const identifier = call.match(MODEL_IDENTIFIER_PATTERN)?.[1];
|
|
1801
|
+
return identifier ? bindings.get(identifier) ?? null : null;
|
|
1802
|
+
}
|
|
1803
|
+
function modelInvocations(contents) {
|
|
1804
|
+
const bindings = literalBindings(contents);
|
|
1805
|
+
const invocations = [];
|
|
1806
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
1807
|
+
const supportedIndices = supportedJavaScriptModelCallIndices(contents);
|
|
1808
|
+
for (const match of contents.matchAll(MODEL_CALL_PATTERN)) {
|
|
1809
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
1810
|
+
if (!supportedIndices.has(match.index)) continue;
|
|
1811
|
+
const callStart = contents.indexOf("(", match.index);
|
|
1812
|
+
if (callStart < 0) continue;
|
|
1813
|
+
const callEnd = matchingDelimiter(contents, callStart, "(", ")");
|
|
1814
|
+
if (callEnd < 0) continue;
|
|
1815
|
+
const call = contents.slice(match.index, callEnd + 1);
|
|
1816
|
+
const judge = /\b(?:system|instructions?)\s*:\s*(['"`])[^'"`]*(?:judge|grader|evaluate|score)[^'"`]*\1/i.test(
|
|
1817
|
+
call
|
|
1818
|
+
);
|
|
1819
|
+
invocations.push({ index: match.index, identity: modelIdentity(call, bindings), judge });
|
|
1820
|
+
}
|
|
1821
|
+
return invocations;
|
|
1822
|
+
}
|
|
1823
|
+
function detectSoleSelfJudge(files) {
|
|
1824
|
+
const findings = [];
|
|
1825
|
+
for (const file of files) {
|
|
1826
|
+
if (!completeLocalSource(file) || !hasSupportedEvaluationImport(file)) continue;
|
|
1827
|
+
if (hasIndependentAcceptanceSignal(file)) continue;
|
|
1828
|
+
const invocations = modelInvocations(file.contents);
|
|
1829
|
+
const judges = invocations.filter((invocation) => invocation.judge);
|
|
1830
|
+
const producers = invocations.filter((invocation) => !invocation.judge);
|
|
1831
|
+
if (judges.length !== 1 || producers.length !== 1) continue;
|
|
1832
|
+
const judge = judges[0];
|
|
1833
|
+
const producer = producers[0];
|
|
1834
|
+
if (!judge?.identity || !producer?.identity || judge.identity !== producer.identity) continue;
|
|
1835
|
+
if (localEmissions(file).length === 0) continue;
|
|
1836
|
+
findings.push(
|
|
1837
|
+
finding2(
|
|
1838
|
+
"LLM-EVL-002",
|
|
1839
|
+
file,
|
|
1840
|
+
judge.index,
|
|
1841
|
+
"warning",
|
|
1842
|
+
`The producer and sole model-assisted judge resolve to the same configured model (${judge.identity}), with no local independent adjudicator.`,
|
|
1843
|
+
`Judge invocation reuses producer model ${judge.identity}`
|
|
1844
|
+
)
|
|
1845
|
+
);
|
|
1846
|
+
}
|
|
1847
|
+
return findings;
|
|
1848
|
+
}
|
|
1849
|
+
function detectSoleSelfJudgeAcrossLanguages(files) {
|
|
1850
|
+
return [
|
|
1851
|
+
...detectSoleSelfJudge(files),
|
|
1852
|
+
...files.flatMap((file) => detectPythonConfiguredSelfJudge(file))
|
|
1853
|
+
];
|
|
1854
|
+
}
|
|
1855
|
+
function hasAggregateEvaluationSurface(files) {
|
|
1856
|
+
return files.some((file) => {
|
|
1857
|
+
if (!completeLocalSource(file) || !hasSupportedEvaluationImport(file)) return false;
|
|
1858
|
+
const aggregates = aggregateAssignmentsIn(file.contents);
|
|
1859
|
+
if (aggregates.size === 0) return false;
|
|
1860
|
+
return localEmissions(file).some((emission) => {
|
|
1861
|
+
const emitted = [...aggregates].filter(([name]) => emission.properties.has(name));
|
|
1862
|
+
if (emitted.length === 0) return false;
|
|
1863
|
+
const first = Math.min(...emitted.map(([, assignment]) => assignment.index));
|
|
1864
|
+
return hasLocalLlmEvaluationInvocation(file, first);
|
|
1865
|
+
});
|
|
1866
|
+
});
|
|
1867
|
+
}
|
|
1868
|
+
function hasProvenanceEvaluationSurface(files) {
|
|
1869
|
+
return hasAggregateEvaluationSurface(files) || detectBoundedEvaluationResultProvenance(files).length > 0;
|
|
1870
|
+
}
|
|
1871
|
+
function hasModelJudgeSurface(files) {
|
|
1872
|
+
return files.some((file) => {
|
|
1873
|
+
if (!completeLocalSource(file) || !hasSupportedEvaluationImport(file)) return false;
|
|
1874
|
+
const invocations = modelInvocations(file.contents);
|
|
1875
|
+
return invocations.some((invocation) => invocation.judge) && invocations.some((invocation) => !invocation.judge) && localEmissions(file).length > 0;
|
|
1876
|
+
}) || files.some((file) => detectPythonConfiguredSelfJudge(file).length > 0);
|
|
1877
|
+
}
|
|
1878
|
+
var CEJEL_LLM_EVALUATION_RULES = [
|
|
1879
|
+
{
|
|
1880
|
+
id: "LLM-PRV-001",
|
|
1881
|
+
title: "Declared evaluation lacks reproducible system provenance",
|
|
1882
|
+
detectorConfidence: "high",
|
|
1883
|
+
evidenceContract: "A recognized local model invocation precedes a complete local evaluation result that is emitted or returned while lacking model or prompt/policy evaluation-configuration lineage. Supported forms are aggregate-to-emitter paths, identifier-bound discrete results with model/case/outcome fields, and incrementally built per-case collections with an evaluation identity and resolved return.",
|
|
1884
|
+
exclusions: [
|
|
1885
|
+
"External or dynamically assembled result reporters",
|
|
1886
|
+
"Exploratory code without a locally emitted or returned evaluation result",
|
|
1887
|
+
"Generic returned objects and model metadata without declared case/outcome semantics",
|
|
1888
|
+
"Tests, examples, fixtures, documentation, generated code, and unresolved paths"
|
|
1889
|
+
],
|
|
1890
|
+
applies: hasProvenanceEvaluationSurface,
|
|
1891
|
+
detect: detectMissingProvenanceIncludingBoundedResults
|
|
1892
|
+
},
|
|
1893
|
+
{
|
|
1894
|
+
id: "LLM-EVL-001",
|
|
1895
|
+
title: "Evaluation result omits its eligible denominator or exclusions",
|
|
1896
|
+
detectorConfidence: "high",
|
|
1897
|
+
evidenceContract: "A complete local case-iteration, aggregate-calculation, and direct result-emission path is visible, while the emitted object lacks a denominator or raw case results.",
|
|
1898
|
+
exclusions: [
|
|
1899
|
+
"Counts without an aggregate rate or average",
|
|
1900
|
+
"External or dynamically assembled result reporters",
|
|
1901
|
+
"Tests, examples, fixtures, documentation, generated code, and unresolved paths"
|
|
1902
|
+
],
|
|
1903
|
+
applies: hasAggregateEvaluationSurface,
|
|
1904
|
+
detect: detectMissingDenominator
|
|
1905
|
+
},
|
|
1906
|
+
{
|
|
1907
|
+
id: "LLM-EVL-002",
|
|
1908
|
+
title: "Evaluated system is its own sole judge",
|
|
1909
|
+
detectorConfidence: "high",
|
|
1910
|
+
evidenceContract: "Exactly one local producer and one explicit model-assisted judge resolve to the same literal model identity, a result is locally emitted, and no observable independent review signal participates in evaluation acceptance or gating.",
|
|
1911
|
+
exclusions: [
|
|
1912
|
+
"Unresolved or provider-managed model identities",
|
|
1913
|
+
"Distinct judge models and locally declared independent adjudication",
|
|
1914
|
+
"Tests, examples, fixtures, documentation, generated code, and unresolved paths"
|
|
1915
|
+
],
|
|
1916
|
+
applies: hasModelJudgeSurface,
|
|
1917
|
+
detect: detectSoleSelfJudgeAcrossLanguages
|
|
1918
|
+
}
|
|
1919
|
+
];
|
|
1920
|
+
function detectCejelLlmEvaluationRules(files) {
|
|
1921
|
+
return CEJEL_LLM_EVALUATION_RULES.flatMap((rule) => rule.detect(files));
|
|
1922
|
+
}
|
|
1923
|
+
|
|
1924
|
+
// src/packs/llm/files.ts
|
|
1925
|
+
import { execFileSync } from "child_process";
|
|
1926
|
+
import { lstatSync, readdirSync, statSync } from "fs";
|
|
1927
|
+
import { join, relative, resolve, sep } from "path";
|
|
1928
|
+
var SKIPPED_DIRECTORIES = /* @__PURE__ */ new Set([
|
|
1929
|
+
".git",
|
|
1930
|
+
".next",
|
|
1931
|
+
".terraform",
|
|
1932
|
+
".venv",
|
|
1933
|
+
"__pycache__",
|
|
1934
|
+
"build",
|
|
1935
|
+
"coverage",
|
|
1936
|
+
"dist",
|
|
1937
|
+
"node_modules",
|
|
1938
|
+
"site-packages",
|
|
1939
|
+
"target",
|
|
1940
|
+
"vendor",
|
|
1941
|
+
"venv"
|
|
1942
|
+
]);
|
|
1943
|
+
var HARD_EXCLUDED_PATH_PATTERN = /(^|\/)(?:\.git|\.venv|venv|site-packages|node_modules|dist|build|\.next|__pycache__|vendor|\.terraform|coverage)(?:\/|$)/;
|
|
1944
|
+
var MAX_ENUMERATED_FILE_BYTES = 512e3;
|
|
1945
|
+
function isInsideRoot(root, path) {
|
|
1946
|
+
const resolved = resolve(path);
|
|
1947
|
+
return resolved === root || resolved.startsWith(`${root}${sep}`);
|
|
1948
|
+
}
|
|
1949
|
+
function isRegularFile(path) {
|
|
1950
|
+
try {
|
|
1951
|
+
return lstatSync(path).isFile();
|
|
1952
|
+
} catch {
|
|
1953
|
+
return false;
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
function listTrackedFiles(repoPath) {
|
|
1957
|
+
try {
|
|
1958
|
+
execFileSync("git", ["rev-parse", "--is-inside-work-tree"], {
|
|
1959
|
+
cwd: repoPath,
|
|
1960
|
+
encoding: "utf8",
|
|
1961
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
1962
|
+
});
|
|
1963
|
+
const output = execFileSync("git", ["ls-files", "--cached"], {
|
|
1964
|
+
cwd: repoPath,
|
|
1965
|
+
encoding: "utf8",
|
|
1966
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
1967
|
+
}).trim();
|
|
1968
|
+
if (!output) return [];
|
|
1969
|
+
const root = resolve(repoPath);
|
|
1970
|
+
return output.split("\n").filter((path) => {
|
|
1971
|
+
if (path.length === 0 || HARD_EXCLUDED_PATH_PATTERN.test(path)) return false;
|
|
1972
|
+
const fullPath = resolve(root, path);
|
|
1973
|
+
return isInsideRoot(root, fullPath) && isRegularFile(fullPath);
|
|
1974
|
+
}).sort();
|
|
1975
|
+
} catch {
|
|
1976
|
+
return null;
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
function visitDirectory(root, directory, files) {
|
|
1980
|
+
let entries;
|
|
1981
|
+
try {
|
|
1982
|
+
entries = readdirSync(directory, { withFileTypes: true });
|
|
1983
|
+
} catch {
|
|
1984
|
+
return;
|
|
1985
|
+
}
|
|
1986
|
+
for (const entry of entries) {
|
|
1987
|
+
if (SKIPPED_DIRECTORIES.has(entry.name)) continue;
|
|
1988
|
+
const fullPath = join(directory, entry.name);
|
|
1989
|
+
if (!isInsideRoot(root, fullPath)) continue;
|
|
1990
|
+
if (entry.isDirectory()) {
|
|
1991
|
+
visitDirectory(root, fullPath, files);
|
|
1992
|
+
continue;
|
|
1993
|
+
}
|
|
1994
|
+
if (!entry.isFile()) continue;
|
|
1995
|
+
try {
|
|
1996
|
+
if (statSync(fullPath).size > MAX_ENUMERATED_FILE_BYTES) continue;
|
|
1997
|
+
} catch {
|
|
1998
|
+
continue;
|
|
1999
|
+
}
|
|
2000
|
+
files.push(relative(root, fullPath));
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
function listCejelLlmPackFiles(repoPath) {
|
|
2004
|
+
const root = resolve(repoPath);
|
|
2005
|
+
let stat;
|
|
2006
|
+
try {
|
|
2007
|
+
stat = lstatSync(root);
|
|
2008
|
+
} catch {
|
|
2009
|
+
throw new Error(`Cejel LLM pack repository path does not exist: ${repoPath}`);
|
|
2010
|
+
}
|
|
2011
|
+
if (!stat.isDirectory()) {
|
|
2012
|
+
throw new Error(`Cejel LLM pack repository path is not a directory: ${repoPath}`);
|
|
2013
|
+
}
|
|
2014
|
+
const tracked = listTrackedFiles(root);
|
|
2015
|
+
if (tracked !== null) return tracked;
|
|
2016
|
+
const files = [];
|
|
2017
|
+
visitDirectory(root, root, files);
|
|
2018
|
+
return files.filter((path) => !HARD_EXCLUDED_PATH_PATTERN.test(path)).sort();
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
// src/packs/llm/python-rules.ts
|
|
2022
|
+
var PYTHON_INTEGRATION_PATTERN = /(?:^|\n)\s*(?:from\s+(?:openai|anthropic)\s+import\s+|import\s+(?:openai|anthropic)\b)/;
|
|
2023
|
+
var PYTHON_MODEL_CALL_PATTERN = /\.(?:responses\.create|chat\.completions\.create|messages\.create)\s*\(/;
|
|
2024
|
+
var PYTHON_MODEL_OUTPUT_PATTERN = /(?:\.output_text\b|\.choices\s*\[[^\]]+\]\s*\.message\.content\b|\.content\s*\[[^\]]+\]\s*\.text\b)/;
|
|
2025
|
+
var PYTHON_IDENTIFIER_ASSIGNMENT_PATTERN = /^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.+)$/;
|
|
2026
|
+
var PYTHON_SENSITIVE_ENV_PATTERN = /(?:os\.environ\s*\[\s*['"]([A-Z0-9_]*(?:PASSWORD|PASSWD|SECRET|PRIVATE_KEY|DATABASE_URL|ACCESS_TOKEN|REFRESH_TOKEN|AUTH_TOKEN|SESSION_TOKEN)[A-Z0-9_]*)['"]\s*\]|os\.getenv\(\s*['"]([A-Z0-9_]*(?:PASSWORD|PASSWD|SECRET|PRIVATE_KEY|DATABASE_URL|ACCESS_TOKEN|REFRESH_TOKEN|AUTH_TOKEN|SESSION_TOKEN)[A-Z0-9_]*)['"])/;
|
|
2027
|
+
var PYTHON_DANGEROUS_SINKS = [
|
|
2028
|
+
{ name: "dynamic evaluation", pattern: /(?<![.\w])(?:eval|exec)\s*\(/ },
|
|
2029
|
+
{ name: "shell execution", pattern: /\bos\.system\s*\(/ },
|
|
2030
|
+
{
|
|
2031
|
+
name: "subprocess execution",
|
|
2032
|
+
pattern: /\bsubprocess\.(?:run|call|check_call|check_output|Popen)\s*\(/
|
|
2033
|
+
}
|
|
2034
|
+
];
|
|
2035
|
+
function pythonSdkImports(masked) {
|
|
2036
|
+
const constructors = /* @__PURE__ */ new Set();
|
|
2037
|
+
const modules = /* @__PURE__ */ new Set();
|
|
2038
|
+
for (const match of masked.matchAll(
|
|
2039
|
+
/(?:^|\n)\s*from\s+(openai|anthropic)\s+import\s+(OpenAI|Anthropic)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/g
|
|
2040
|
+
)) {
|
|
2041
|
+
if (match[2]) constructors.add(match[3] ?? match[2]);
|
|
2042
|
+
}
|
|
2043
|
+
for (const match of masked.matchAll(
|
|
2044
|
+
/(?:^|\n)\s*import\s+(openai|anthropic)(?:\s+as\s+([A-Za-z_][A-Za-z0-9_]*))?/g
|
|
2045
|
+
)) {
|
|
2046
|
+
if (match[1]) modules.add(match[2] ?? match[1]);
|
|
2047
|
+
}
|
|
2048
|
+
return { constructors, modules };
|
|
2049
|
+
}
|
|
2050
|
+
function pythonParameters(line) {
|
|
2051
|
+
const parameters = line.match(/^\s*(?:async\s+)?def\s+[A-Za-z_][A-Za-z0-9_]*\s*\(([^)]*)\)/)?.[1];
|
|
2052
|
+
if (parameters === void 0) return [];
|
|
2053
|
+
return parameters.split(",").flatMap((parameter) => {
|
|
2054
|
+
const name = parameter.trim().match(/^(?:\*{1,2})?([A-Za-z_][A-Za-z0-9_]*)/)?.[1];
|
|
2055
|
+
return name ? [name] : [];
|
|
2056
|
+
});
|
|
2057
|
+
}
|
|
2058
|
+
function splitPythonTopLevel(value, delimiter) {
|
|
2059
|
+
const parts = [];
|
|
2060
|
+
let start = 0;
|
|
2061
|
+
let round = 0;
|
|
2062
|
+
let square = 0;
|
|
2063
|
+
let curly = 0;
|
|
2064
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
2065
|
+
const character = value[index];
|
|
2066
|
+
if (character === "(") round += 1;
|
|
2067
|
+
else if (character === ")") round -= 1;
|
|
2068
|
+
else if (character === "[") square += 1;
|
|
2069
|
+
else if (character === "]") square -= 1;
|
|
2070
|
+
else if (character === "{") curly += 1;
|
|
2071
|
+
else if (character === "}") curly -= 1;
|
|
2072
|
+
else if (character === delimiter && round === 0 && square === 0 && curly === 0) {
|
|
2073
|
+
parts.push(value.slice(start, index));
|
|
2074
|
+
start = index + 1;
|
|
2075
|
+
}
|
|
2076
|
+
}
|
|
2077
|
+
parts.push(value.slice(start));
|
|
2078
|
+
return parts;
|
|
2079
|
+
}
|
|
2080
|
+
function pythonTargetIdentifiers(target) {
|
|
2081
|
+
let normalized = target.trim();
|
|
2082
|
+
while (normalized.startsWith("(") && normalized.endsWith(")") || normalized.startsWith("[") && normalized.endsWith("]")) {
|
|
2083
|
+
normalized = normalized.slice(1, -1).trim();
|
|
2084
|
+
}
|
|
2085
|
+
const tuple = splitPythonTopLevel(normalized, ",");
|
|
2086
|
+
if (tuple.length > 1) {
|
|
2087
|
+
return {
|
|
2088
|
+
identifiers: tuple.flatMap((part) => pythonTargetIdentifiers(part).identifiers),
|
|
2089
|
+
simple: false
|
|
2090
|
+
};
|
|
2091
|
+
}
|
|
2092
|
+
normalized = normalized.replace(/^\*+/, "").trim();
|
|
2093
|
+
const annotation = splitPythonTopLevel(normalized, ":");
|
|
2094
|
+
if (annotation.length > 1) normalized = annotation[0]?.trim() ?? "";
|
|
2095
|
+
const identifier = /^[A-Za-z_][A-Za-z0-9_]*$/.test(normalized) ? normalized : void 0;
|
|
2096
|
+
return { identifiers: identifier ? [identifier] : [], simple: Boolean(identifier) };
|
|
2097
|
+
}
|
|
2098
|
+
function pythonAssignment(statement) {
|
|
2099
|
+
const equalityIndices = [];
|
|
2100
|
+
let round = 0;
|
|
2101
|
+
let square = 0;
|
|
2102
|
+
let curly = 0;
|
|
2103
|
+
for (let index = 0; index < statement.length; index += 1) {
|
|
2104
|
+
const character = statement[index];
|
|
2105
|
+
if (character === "(") round += 1;
|
|
2106
|
+
else if (character === ")") round -= 1;
|
|
2107
|
+
else if (character === "[") square += 1;
|
|
2108
|
+
else if (character === "]") square -= 1;
|
|
2109
|
+
else if (character === "{") curly += 1;
|
|
2110
|
+
else if (character === "}") curly -= 1;
|
|
2111
|
+
else if (character === "=" && round === 0 && square === 0 && curly === 0 && !/[=<>!:+\-*/%@&|^]/.test(statement[index - 1] ?? "") && statement[index + 1] !== "=") equalityIndices.push(index);
|
|
2112
|
+
}
|
|
2113
|
+
if (equalityIndices.length === 0) return void 0;
|
|
2114
|
+
const targets = equalityIndices.map((equalityIndex, index) => {
|
|
2115
|
+
const start = index === 0 ? 0 : (equalityIndices[index - 1] ?? -1) + 1;
|
|
2116
|
+
return pythonTargetIdentifiers(statement.slice(start, equalityIndex));
|
|
2117
|
+
});
|
|
2118
|
+
return {
|
|
2119
|
+
targets,
|
|
2120
|
+
expression: statement.slice((equalityIndices.at(-1) ?? -1) + 1).trim()
|
|
2121
|
+
};
|
|
2122
|
+
}
|
|
2123
|
+
function supportedPythonModelCallIndices(contents) {
|
|
2124
|
+
const masked = maskPythonNonCode(contents);
|
|
2125
|
+
const imports = pythonSdkImports(masked);
|
|
2126
|
+
const scopes = [
|
|
2127
|
+
{
|
|
2128
|
+
indent: -1,
|
|
2129
|
+
bindings: new Map([
|
|
2130
|
+
...[...imports.modules].map((name) => [name, "module"]),
|
|
2131
|
+
...[...imports.constructors].map((name) => [name, "constructor"])
|
|
2132
|
+
])
|
|
2133
|
+
}
|
|
2134
|
+
];
|
|
2135
|
+
const indices = /* @__PURE__ */ new Set();
|
|
2136
|
+
let offset = 0;
|
|
2137
|
+
const visibleBinding = (name) => {
|
|
2138
|
+
for (let index = scopes.length - 1; index >= 0; index -= 1) {
|
|
2139
|
+
const value = scopes[index]?.bindings.get(name);
|
|
2140
|
+
if (value !== void 0) return value;
|
|
2141
|
+
}
|
|
2142
|
+
return void 0;
|
|
2143
|
+
};
|
|
2144
|
+
for (const line of masked.split("\n")) {
|
|
2145
|
+
const indentation = line.match(/^\s*/)?.[0].length ?? 0;
|
|
2146
|
+
if (line.trim().length > 0) {
|
|
2147
|
+
while (scopes.length > 1 && indentation <= (scopes.at(-1)?.indent ?? -1)) scopes.pop();
|
|
2148
|
+
}
|
|
2149
|
+
let statementOffset = 0;
|
|
2150
|
+
for (const statement of line.split(";")) {
|
|
2151
|
+
for (const match of statement.matchAll(new RegExp(PYTHON_MODEL_CALL_PATTERN, "g"))) {
|
|
2152
|
+
const prefix = statement.slice(0, match.index);
|
|
2153
|
+
const receiver = prefix.match(/([A-Za-z_][A-Za-z0-9_]*)\s*$/)?.[1];
|
|
2154
|
+
const directConstructor = prefix.match(/([A-Za-z_][A-Za-z0-9_]*)\s*\(\s*\)\s*$/)?.[1];
|
|
2155
|
+
const receiverBinding = receiver ? visibleBinding(receiver) : void 0;
|
|
2156
|
+
if (receiverBinding === "client" || receiverBinding === "module" || directConstructor && visibleBinding(directConstructor) === "constructor") {
|
|
2157
|
+
indices.add(offset + statementOffset + match.index);
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
2160
|
+
const assignment = pythonAssignment(statement);
|
|
2161
|
+
if (assignment) {
|
|
2162
|
+
const constructor = [...imports.constructors].some(
|
|
2163
|
+
(name) => visibleBinding(name) === "constructor" && new RegExp(`^${name}\\s*\\(`).test(assignment.expression)
|
|
2164
|
+
);
|
|
2165
|
+
const moduleConstructor = [...imports.modules].some(
|
|
2166
|
+
(name) => visibleBinding(name) === "module" && new RegExp(`^${name}\\.(?:OpenAI|Anthropic)\\s*\\(`).test(assignment.expression)
|
|
2167
|
+
);
|
|
2168
|
+
const establishesClient = constructor || moduleConstructor;
|
|
2169
|
+
for (const target of assignment.targets) {
|
|
2170
|
+
for (const identifier of target.identifiers) {
|
|
2171
|
+
scopes.at(-1)?.bindings.set(
|
|
2172
|
+
identifier,
|
|
2173
|
+
target.simple && establishesClient ? "client" : "other"
|
|
2174
|
+
);
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
}
|
|
2178
|
+
statementOffset += statement.length + 1;
|
|
2179
|
+
}
|
|
2180
|
+
const parameters = pythonParameters(line);
|
|
2181
|
+
if (parameters.length > 0) {
|
|
2182
|
+
scopes.push({
|
|
2183
|
+
indent: indentation,
|
|
2184
|
+
bindings: new Map(parameters.map((name) => [name, "other"]))
|
|
2185
|
+
});
|
|
2186
|
+
} else if (/^\s*class\s+[A-Za-z_][A-Za-z0-9_]*\b/.test(line)) {
|
|
2187
|
+
scopes.push({ indent: indentation, bindings: /* @__PURE__ */ new Map() });
|
|
2188
|
+
}
|
|
2189
|
+
offset += line.length + 1;
|
|
2190
|
+
}
|
|
2191
|
+
return indices;
|
|
2192
|
+
}
|
|
2193
|
+
function isExcludedSourcePath2(path) {
|
|
2194
|
+
return /(?:^|\/)(?:__tests__|test|tests|fixtures?|examples?|vendor|generated)(?:\/|$)/i.test(
|
|
2195
|
+
path
|
|
2196
|
+
) || /(?:^|\/)(?:test_[^/]+|[^/]+_test)\.py$/i.test(path);
|
|
2197
|
+
}
|
|
2198
|
+
function lineNumberAt4(contents, index) {
|
|
2199
|
+
return contents.slice(0, index).split("\n").length;
|
|
2200
|
+
}
|
|
2201
|
+
function pythonFinding(ruleId, file, index, severity, confidence, summary, label) {
|
|
2202
|
+
return {
|
|
2203
|
+
ruleId,
|
|
2204
|
+
severity,
|
|
2205
|
+
confidence,
|
|
2206
|
+
summary,
|
|
2207
|
+
evidence: {
|
|
2208
|
+
path: file.path,
|
|
2209
|
+
line: lineNumberAt4(file.contents, index),
|
|
2210
|
+
label
|
|
2211
|
+
}
|
|
2212
|
+
};
|
|
2213
|
+
}
|
|
2214
|
+
function containsIdentifier(expression, identifiers) {
|
|
2215
|
+
for (const identifier of identifiers) {
|
|
2216
|
+
const escaped2 = identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2217
|
+
if (new RegExp(`\\b${escaped2}\\b`).test(expression)) return true;
|
|
2218
|
+
}
|
|
2219
|
+
return false;
|
|
2220
|
+
}
|
|
2221
|
+
function visiblePythonOutputIdentifiers(scopes) {
|
|
2222
|
+
const visible = /* @__PURE__ */ new Map();
|
|
2223
|
+
for (const scope of scopes) {
|
|
2224
|
+
for (const [identifier, output] of scope.aliases) visible.set(identifier, output);
|
|
2225
|
+
}
|
|
2226
|
+
return new Set([...visible].filter(([, output]) => output).map(([identifier]) => identifier));
|
|
2227
|
+
}
|
|
2228
|
+
function expressionContainsBoundPythonModelOutput(expression, boundIdentifiers) {
|
|
2229
|
+
const normalized = expression.trim();
|
|
2230
|
+
const output = normalized.match(PYTHON_MODEL_OUTPUT_PATTERN);
|
|
2231
|
+
if (!output || output.index === void 0) return false;
|
|
2232
|
+
const receiver = normalized.slice(0, output.index);
|
|
2233
|
+
const identifier = receiver.match(/([A-Za-z_][A-Za-z0-9_]*)[.\[\]\w\s]*$/)?.[1];
|
|
2234
|
+
return Boolean(identifier && boundIdentifiers.has(identifier));
|
|
2235
|
+
}
|
|
2236
|
+
function hasSupportedPythonLlmIntegration(file) {
|
|
2237
|
+
const maskedContents = maskPythonNonCode(file.contents);
|
|
2238
|
+
return !isExcludedSourcePath2(file.path) && PYTHON_INTEGRATION_PATTERN.test(maskedContents) && supportedPythonModelCallIndices(file.contents).size > 0;
|
|
2239
|
+
}
|
|
2240
|
+
function detectPythonUnsafeModelOutputSink(file) {
|
|
2241
|
+
if (!hasSupportedPythonLlmIntegration(file)) return [];
|
|
2242
|
+
const maskedContents = maskPythonNonCode(file.contents);
|
|
2243
|
+
const scopes = [
|
|
2244
|
+
{ indent: -1, aliases: /* @__PURE__ */ new Map() }
|
|
2245
|
+
];
|
|
2246
|
+
const findings = [];
|
|
2247
|
+
const supportedCalls = supportedPythonModelCallIndices(file.contents);
|
|
2248
|
+
let offset = 0;
|
|
2249
|
+
for (const line of maskedContents.split("\n")) {
|
|
2250
|
+
const indentation = line.match(/^\s*/)?.[0].length ?? 0;
|
|
2251
|
+
if (line.trim().length > 0) {
|
|
2252
|
+
while (scopes.length > 1 && indentation <= (scopes.at(-1)?.indent ?? -1)) scopes.pop();
|
|
2253
|
+
}
|
|
2254
|
+
const assignment = line.match(PYTHON_IDENTIFIER_ASSIGNMENT_PATTERN);
|
|
2255
|
+
const identifier = assignment?.[1];
|
|
2256
|
+
const expression = assignment?.[2];
|
|
2257
|
+
const visibleBeforeAssignment = visiblePythonOutputIdentifiers(scopes);
|
|
2258
|
+
if (identifier && expression) {
|
|
2259
|
+
const lineHasSupportedResponseAssignment = [...supportedCalls].some(
|
|
2260
|
+
(index) => offset <= index && index < offset + line.length
|
|
2261
|
+
);
|
|
2262
|
+
scopes.at(-1)?.aliases.set(
|
|
2263
|
+
identifier,
|
|
2264
|
+
lineHasSupportedResponseAssignment || expressionContainsBoundPythonModelOutput(expression, visibleBeforeAssignment)
|
|
2265
|
+
);
|
|
2266
|
+
}
|
|
2267
|
+
const outputIdentifiers = visiblePythonOutputIdentifiers(scopes);
|
|
2268
|
+
for (const sink of PYTHON_DANGEROUS_SINKS) {
|
|
2269
|
+
const sinkMatch = line.match(sink.pattern);
|
|
2270
|
+
if (!sinkMatch) continue;
|
|
2271
|
+
if (!expressionContainsBoundPythonModelOutput(line, outputIdentifiers) && !containsIdentifier(line, outputIdentifiers)) {
|
|
2272
|
+
continue;
|
|
2273
|
+
}
|
|
2274
|
+
findings.push(
|
|
2275
|
+
pythonFinding(
|
|
2276
|
+
"LLM-IOH-001",
|
|
2277
|
+
file,
|
|
2278
|
+
offset + (sinkMatch.index ?? 0),
|
|
2279
|
+
"critical",
|
|
2280
|
+
"high",
|
|
2281
|
+
`Observable model output is passed directly to ${sink.name}.`,
|
|
2282
|
+
`Model output reaches ${sink.name}`
|
|
2283
|
+
)
|
|
2284
|
+
);
|
|
2285
|
+
}
|
|
2286
|
+
if (/^\s*(?:(?:async\s+)?def|class)\s+[A-Za-z_][A-Za-z0-9_]*\b/.test(line)) {
|
|
2287
|
+
scopes.push({ indent: indentation, aliases: /* @__PURE__ */ new Map() });
|
|
2288
|
+
}
|
|
2289
|
+
offset += line.length + 1;
|
|
2290
|
+
}
|
|
2291
|
+
return findings;
|
|
2292
|
+
}
|
|
2293
|
+
function detectPythonUnboundedAgentLoop(file) {
|
|
2294
|
+
if (!hasSupportedPythonLlmIntegration(file)) return [];
|
|
2295
|
+
const findings = [];
|
|
2296
|
+
const lines = maskPythonNonCode(file.contents).split("\n");
|
|
2297
|
+
const supportedCalls = supportedPythonModelCallIndices(file.contents);
|
|
2298
|
+
let offset = 0;
|
|
2299
|
+
for (let lineIndex = 0; lineIndex < lines.length; lineIndex += 1) {
|
|
2300
|
+
const line = lines[lineIndex] ?? "";
|
|
2301
|
+
const loop = line.match(/^(\s*)while\s+True\s*:\s*(?:#.*)?$/);
|
|
2302
|
+
if (!loop) {
|
|
2303
|
+
offset += line.length + 1;
|
|
2304
|
+
continue;
|
|
2305
|
+
}
|
|
2306
|
+
const loopIndent = loop[1]?.length ?? 0;
|
|
2307
|
+
const bodyStartOffset = offset + line.length + 1;
|
|
2308
|
+
let bodyEndOffset = bodyStartOffset;
|
|
2309
|
+
const bodyLines = [];
|
|
2310
|
+
for (let bodyIndex = lineIndex + 1; bodyIndex < lines.length; bodyIndex += 1) {
|
|
2311
|
+
const candidate = lines[bodyIndex] ?? "";
|
|
2312
|
+
if (candidate.trim().length === 0 || candidate.trimStart().startsWith("#")) {
|
|
2313
|
+
bodyEndOffset += candidate.length + 1;
|
|
2314
|
+
bodyLines.push(candidate);
|
|
2315
|
+
continue;
|
|
2316
|
+
}
|
|
2317
|
+
const indent = candidate.match(/^\s*/)?.[0].length ?? 0;
|
|
2318
|
+
if (indent <= loopIndent) break;
|
|
2319
|
+
bodyEndOffset += candidate.length + 1;
|
|
2320
|
+
bodyLines.push(candidate);
|
|
2321
|
+
}
|
|
2322
|
+
if (![...supportedCalls].some((index) => bodyStartOffset <= index && index < bodyEndOffset)) {
|
|
2323
|
+
offset += line.length + 1;
|
|
2324
|
+
continue;
|
|
2325
|
+
}
|
|
2326
|
+
const significant = bodyLines.filter((candidate) => candidate.trim().length > 0);
|
|
2327
|
+
const topIndent = Math.min(...significant.map(
|
|
2328
|
+
(candidate) => candidate.match(/^\s*/)?.[0].length ?? Number.POSITIVE_INFINITY
|
|
2329
|
+
));
|
|
2330
|
+
const hasObservableExit = bodyLines.some((candidate, candidateIndex) => {
|
|
2331
|
+
const indent = candidate.match(/^\s*/)?.[0].length ?? 0;
|
|
2332
|
+
const guard = candidate.trim().match(/^if\b(.+):\s*(.*)$/);
|
|
2333
|
+
if (indent !== topIndent || !guard) return false;
|
|
2334
|
+
if (/^(?:break|return|raise)\b/.test(guard[2] ?? "")) return true;
|
|
2335
|
+
const next = bodyLines.slice(candidateIndex + 1).find((item) => item.trim().length > 0);
|
|
2336
|
+
if (!next) return false;
|
|
2337
|
+
const nextIndent = next.match(/^\s*/)?.[0].length ?? 0;
|
|
2338
|
+
return nextIndent > topIndent && /^(?:break|return|raise)\b/.test(next.trim());
|
|
2339
|
+
});
|
|
2340
|
+
if (hasObservableExit) {
|
|
2341
|
+
offset += line.length + 1;
|
|
2342
|
+
continue;
|
|
2343
|
+
}
|
|
2344
|
+
findings.push(
|
|
2345
|
+
pythonFinding(
|
|
2346
|
+
"LLM-AGY-002",
|
|
2347
|
+
file,
|
|
2348
|
+
offset + (line.indexOf("while") >= 0 ? line.indexOf("while") : 0),
|
|
2349
|
+
"warning",
|
|
2350
|
+
"medium",
|
|
2351
|
+
"An official SDK model call occurs inside a literal while True loop; no static step bound is observable.",
|
|
2352
|
+
"Model call inside unconditional Python loop body"
|
|
2353
|
+
)
|
|
2354
|
+
);
|
|
2355
|
+
offset += line.length + 1;
|
|
2356
|
+
}
|
|
2357
|
+
return findings;
|
|
2358
|
+
}
|
|
2359
|
+
function matchingPythonCallEnd(contents, openParen) {
|
|
2360
|
+
let depth = 0;
|
|
2361
|
+
let quote = null;
|
|
2362
|
+
let escaped2 = false;
|
|
2363
|
+
let comment = false;
|
|
2364
|
+
for (let index = openParen; index < contents.length; index += 1) {
|
|
2365
|
+
const character = contents[index] ?? "";
|
|
2366
|
+
if (comment) {
|
|
2367
|
+
if (character === "\n") comment = false;
|
|
2368
|
+
continue;
|
|
2369
|
+
}
|
|
2370
|
+
if (quote) {
|
|
2371
|
+
if (quote.length === 3 && contents.startsWith(quote, index)) {
|
|
2372
|
+
quote = null;
|
|
2373
|
+
index += 2;
|
|
2374
|
+
} else if (quote.length === 1) {
|
|
2375
|
+
if (escaped2) escaped2 = false;
|
|
2376
|
+
else if (character === "\\") escaped2 = true;
|
|
2377
|
+
else if (character === quote) quote = null;
|
|
2378
|
+
}
|
|
2379
|
+
continue;
|
|
2380
|
+
}
|
|
2381
|
+
if (character === "#") {
|
|
2382
|
+
comment = true;
|
|
2383
|
+
continue;
|
|
2384
|
+
}
|
|
2385
|
+
const triple = contents.slice(index, index + 3);
|
|
2386
|
+
if (triple === "'''" || triple === '"""') {
|
|
2387
|
+
quote = triple;
|
|
2388
|
+
index += 2;
|
|
2389
|
+
continue;
|
|
2390
|
+
}
|
|
2391
|
+
if (character === "'" || character === '"') {
|
|
2392
|
+
quote = character;
|
|
2393
|
+
continue;
|
|
2394
|
+
}
|
|
2395
|
+
if (character === "(") depth += 1;
|
|
2396
|
+
if (character === ")") {
|
|
2397
|
+
depth -= 1;
|
|
2398
|
+
if (depth === 0) return index + 1;
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
return null;
|
|
2402
|
+
}
|
|
2403
|
+
function pythonModelCallArguments(contents) {
|
|
2404
|
+
const calls = [];
|
|
2405
|
+
const masked = maskPythonNonCode(contents);
|
|
2406
|
+
const supportedCalls = supportedPythonModelCallIndices(contents);
|
|
2407
|
+
for (const match of masked.matchAll(new RegExp(PYTHON_MODEL_CALL_PATTERN, "g"))) {
|
|
2408
|
+
if (!supportedCalls.has(match.index)) continue;
|
|
2409
|
+
const openParen = match.index + match[0].lastIndexOf("(");
|
|
2410
|
+
const end = matchingPythonCallEnd(contents, openParen);
|
|
2411
|
+
if (end === null) continue;
|
|
2412
|
+
calls.push({ text: contents.slice(openParen + 1, end - 1), index: openParen + 1 });
|
|
2413
|
+
}
|
|
2414
|
+
return calls;
|
|
2415
|
+
}
|
|
2416
|
+
function detectPythonSensitivePromptData(file) {
|
|
2417
|
+
if (!hasSupportedPythonLlmIntegration(file)) return [];
|
|
2418
|
+
const findings = [];
|
|
2419
|
+
for (const call of pythonModelCallArguments(file.contents)) {
|
|
2420
|
+
const masked = maskPythonNonCode(call.text);
|
|
2421
|
+
const sensitive = [...call.text.matchAll(new RegExp(PYTHON_SENSITIVE_ENV_PATTERN, "g"))].find(
|
|
2422
|
+
(candidate) => candidate.index !== void 0 && (masked[candidate.index] ?? " ") !== " "
|
|
2423
|
+
);
|
|
2424
|
+
if (!sensitive) continue;
|
|
2425
|
+
const variable = sensitive[1] ?? sensitive[2] ?? "secret-like environment value";
|
|
2426
|
+
findings.push(
|
|
2427
|
+
pythonFinding(
|
|
2428
|
+
"LLM-DAT-001",
|
|
2429
|
+
file,
|
|
2430
|
+
call.index + (sensitive.index ?? 0),
|
|
2431
|
+
"critical",
|
|
2432
|
+
"high",
|
|
2433
|
+
"A secret-like environment value is placed directly in a model request.",
|
|
2434
|
+
`Sensitive environment value ${variable} in model request`
|
|
2435
|
+
)
|
|
2436
|
+
);
|
|
2437
|
+
}
|
|
2438
|
+
return findings;
|
|
2439
|
+
}
|
|
2440
|
+
var CEJEL_LLM_PYTHON_RULES = [
|
|
2441
|
+
{
|
|
2442
|
+
id: "LLM-IOH-001",
|
|
2443
|
+
title: "Python model output passed to a consequential sink",
|
|
2444
|
+
detectorConfidence: "high",
|
|
2445
|
+
evidenceContract: "An official OpenAI or Anthropic Python integration, a supported response-output shape or still-live direct alias in the visible function or class scope, and a direct eval, exec, os.system, or subprocess sink are observable in one non-test source file.",
|
|
2446
|
+
exclusions: [
|
|
2447
|
+
"Inter-procedural or cross-file data flow",
|
|
2448
|
+
"Identifiers transformed or reassigned after model-output extraction",
|
|
2449
|
+
"Shell wrappers and sink APIs outside the enumerated Python standard-library forms"
|
|
2450
|
+
],
|
|
2451
|
+
applies: (file) => detectPythonUnsafeModelOutputSink(file).length > 0,
|
|
2452
|
+
detect: detectPythonUnsafeModelOutputSink
|
|
2453
|
+
},
|
|
2454
|
+
{
|
|
2455
|
+
id: "LLM-AGY-002",
|
|
2456
|
+
title: "Unbounded Python agent loop",
|
|
2457
|
+
detectorConfidence: "medium",
|
|
2458
|
+
evidenceContract: "A literal while True loop has a complete local indentation-delimited body containing an official OpenAI or Anthropic Python SDK model call and no observable conditional counter, deadline, cancellation, budget, return, raise, or break guard.",
|
|
2459
|
+
exclusions: [
|
|
2460
|
+
"Runtime bounds or cancellation enforced outside the source file",
|
|
2461
|
+
"Framework-specific iteration controls not visible in loop syntax"
|
|
2462
|
+
],
|
|
2463
|
+
applies: (file) => detectPythonUnboundedAgentLoop(file).length > 0,
|
|
2464
|
+
detect: detectPythonUnboundedAgentLoop
|
|
2465
|
+
},
|
|
2466
|
+
{
|
|
2467
|
+
id: "LLM-DAT-001",
|
|
2468
|
+
title: "Sensitive Python environment data passed to a model request",
|
|
2469
|
+
detectorConfidence: "high",
|
|
2470
|
+
evidenceContract: "A narrowly named os.environ or os.getenv secret-like value occurs inside the executable, syntactically matched arguments of an official SDK model call after comments and strings are excluded from call extraction.",
|
|
2471
|
+
exclusions: [
|
|
2472
|
+
"Provider API keys used only for client construction",
|
|
2473
|
+
"Values passed through aliases, helpers, or external configuration layers",
|
|
2474
|
+
"General personal-data or semantic sensitivity classification"
|
|
2475
|
+
],
|
|
2476
|
+
applies: (file) => detectPythonSensitivePromptData(file).length > 0,
|
|
2477
|
+
detect: detectPythonSensitivePromptData
|
|
2478
|
+
}
|
|
2479
|
+
];
|
|
2480
|
+
|
|
2481
|
+
// src/packs/llm/rules.ts
|
|
2482
|
+
var MODEL_OUTPUT_PATTERN2 = /(?:\.output_text\b|\.choices\s*\[[^\]]+\]\s*\.message\s*\.content\b|\.content\s*\[[^\]]+\]\s*\.text\b)/;
|
|
2483
|
+
var MODEL_CALL_PATTERN2 = /(?:\.responses\.create\s*\(|\.chat\.completions\.create\s*\(|\.messages\.create\s*\(|\bgenerateText\s*\(|\bstreamText\s*\()/;
|
|
2484
|
+
var BASE_DANGEROUS_SINKS = [
|
|
2485
|
+
{
|
|
2486
|
+
name: "dynamic evaluation",
|
|
2487
|
+
pattern: /(?<![.\w$])(?:eval|Function)\s*\(/,
|
|
2488
|
+
severity: "critical"
|
|
2489
|
+
},
|
|
2490
|
+
{
|
|
2491
|
+
name: "raw HTML assignment",
|
|
2492
|
+
pattern: /(?:\.innerHTML\s*=|dangerouslySetInnerHTML\s*=)/,
|
|
2493
|
+
severity: "warning"
|
|
2494
|
+
}
|
|
2495
|
+
];
|
|
2496
|
+
function escapeRegExp2(value) {
|
|
2497
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2498
|
+
}
|
|
2499
|
+
function importedProcessSinks(contents, maskedContents) {
|
|
2500
|
+
const identifiers = /* @__PURE__ */ new Set();
|
|
2501
|
+
const namespaces = /* @__PURE__ */ new Set();
|
|
2502
|
+
const childProcessMembers = /* @__PURE__ */ new Set(["exec", "execSync", "spawn", "spawnSync"]);
|
|
2503
|
+
for (const match of contents.matchAll(
|
|
2504
|
+
/import\s*\{([^}]+)\}\s*from\s*['"](?:node:)?child_process['"]/g
|
|
2505
|
+
)) {
|
|
2506
|
+
if ((maskedContents[match.index] ?? " ") === " ") continue;
|
|
2507
|
+
for (const member of (match[1] ?? "").split(",")) {
|
|
2508
|
+
const binding = member.trim().match(/^([A-Za-z_$][\w$]*)(?:\s+as\s+([A-Za-z_$][\w$]*))?$/);
|
|
2509
|
+
if (binding?.[1] && childProcessMembers.has(binding[1])) identifiers.add(binding[2] ?? binding[1]);
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
for (const match of contents.matchAll(
|
|
2513
|
+
/import\s+\*\s+as\s+([A-Za-z_$][\w$]*)\s+from\s*['"](?:node:)?child_process['"]/g
|
|
2514
|
+
)) {
|
|
2515
|
+
if ((maskedContents[match.index] ?? " ") === " ") continue;
|
|
2516
|
+
if (match[1]) namespaces.add(match[1]);
|
|
2517
|
+
}
|
|
2518
|
+
for (const match of contents.matchAll(
|
|
2519
|
+
/(?:const|let|var)\s+\{([^}]+)\}\s*=\s*require\s*\(\s*['"](?:node:)?child_process['"]\s*\)/g
|
|
2520
|
+
)) {
|
|
2521
|
+
if ((maskedContents[match.index] ?? " ") === " ") continue;
|
|
2522
|
+
for (const member of (match[1] ?? "").split(",")) {
|
|
2523
|
+
const binding = member.trim().match(/^([A-Za-z_$][\w$]*)(?:\s*:\s*([A-Za-z_$][\w$]*))?$/);
|
|
2524
|
+
if (binding?.[1] && childProcessMembers.has(binding[1])) identifiers.add(binding[2] ?? binding[1]);
|
|
2525
|
+
}
|
|
2526
|
+
}
|
|
2527
|
+
for (const match of contents.matchAll(
|
|
2528
|
+
/(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*require\s*\(\s*['"](?:node:)?child_process['"]\s*\)/g
|
|
2529
|
+
)) {
|
|
2530
|
+
if ((maskedContents[match.index] ?? " ") === " ") continue;
|
|
2531
|
+
if (match[1]) namespaces.add(match[1]);
|
|
2532
|
+
}
|
|
2533
|
+
for (const match of contents.matchAll(
|
|
2534
|
+
/import\s*\{([^}]+)\}\s*from\s*['"]execa['"]/g
|
|
2535
|
+
)) {
|
|
2536
|
+
if ((maskedContents[match.index] ?? " ") === " ") continue;
|
|
2537
|
+
for (const member of (match[1] ?? "").split(",")) {
|
|
2538
|
+
const binding = member.trim().match(/^(execa)(?:\s+as\s+([A-Za-z_$][\w$]*))?$/);
|
|
2539
|
+
if (binding?.[1]) identifiers.add(binding[2] ?? binding[1]);
|
|
2540
|
+
}
|
|
2541
|
+
}
|
|
2542
|
+
const alternatives = [
|
|
2543
|
+
...[...identifiers].map((identifier) => `(?<![.\\w$])${escapeRegExp2(identifier)}`),
|
|
2544
|
+
...[...namespaces].map(
|
|
2545
|
+
(namespace) => `\\b${escapeRegExp2(namespace)}\\.(?:exec|execSync|spawn|spawnSync)`
|
|
2546
|
+
)
|
|
2547
|
+
];
|
|
2548
|
+
return alternatives.length === 0 ? [] : [{
|
|
2549
|
+
name: "shell execution",
|
|
2550
|
+
pattern: new RegExp(`(?:${alternatives.join("|")})\\s*\\(`),
|
|
2551
|
+
severity: "critical"
|
|
2552
|
+
}];
|
|
2553
|
+
}
|
|
2554
|
+
function javaScriptExecutableHelperParameterSinks(file) {
|
|
2555
|
+
if (isExcludedSourcePath3(file.path)) return [];
|
|
2556
|
+
const sinks = [];
|
|
2557
|
+
const masked = maskJavaScriptNonCode(file.contents);
|
|
2558
|
+
const declaration = /(?:\bexport\s+)?\bfunction\s+([A-Za-z_$][\w$]*)\s*\(([^)]*)\)\s*(?::[^{\n]+)?\s*\{/g;
|
|
2559
|
+
for (const match of file.contents.matchAll(declaration)) {
|
|
2560
|
+
if ((masked[match.index] ?? " ") === " ") continue;
|
|
2561
|
+
const functionName = match[1];
|
|
2562
|
+
if (!functionName) continue;
|
|
2563
|
+
const openBrace = file.contents.indexOf("{", match.index);
|
|
2564
|
+
const end = matchingDelimiterEnd(file.contents, openBrace, "{", "}");
|
|
2565
|
+
if (end === null) continue;
|
|
2566
|
+
const body = file.contents.slice(openBrace + 1, end - 1);
|
|
2567
|
+
const parameters = (match[2] ?? "").split(",").map((parameter) => parameter.trim().match(/^([A-Za-z_$][\w$]*)/)?.[1]).filter((parameter) => Boolean(parameter));
|
|
2568
|
+
for (const parameterName of parameters) {
|
|
2569
|
+
const escaped2 = escapeRegExp2(parameterName);
|
|
2570
|
+
const directDynamicEvaluation = new RegExp(
|
|
2571
|
+
`(?:eval|Function)\\s*\\([^)]*\\b${escaped2}\\b|\\(\\s*0\\s*,\\s*eval\\s*\\)\\s*\\([^)]*\\b${escaped2}\\b`
|
|
2572
|
+
).test(body);
|
|
2573
|
+
const executableTemplate = [...body.matchAll(/`(?:\\[\s\S]|\$\{[\s\S]*?\}|[^`])*`/g)].some((template) => {
|
|
2574
|
+
const text = template[0];
|
|
2575
|
+
const tainted = /* @__PURE__ */ new Set();
|
|
2576
|
+
const assignments = [...text.matchAll(
|
|
2577
|
+
/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*([\s\S]*?);/g
|
|
2578
|
+
)];
|
|
2579
|
+
for (const assignment of assignments) {
|
|
2580
|
+
const name = assignment[1];
|
|
2581
|
+
const expression = assignment[2];
|
|
2582
|
+
if (name && expression && new RegExp(`\\$\\{[^}]{0,160}\\b${escaped2}\\b[^}]*\\}`).test(expression)) {
|
|
2583
|
+
tainted.add(name);
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
for (let pass = 0; pass < assignments.length; pass += 1) {
|
|
2587
|
+
for (const assignment of assignments) {
|
|
2588
|
+
const name = assignment[1];
|
|
2589
|
+
const expression = assignment[2];
|
|
2590
|
+
if (name && expression && containsIdentifier2(expression, tainted)) {
|
|
2591
|
+
tainted.add(name);
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
2595
|
+
if (tainted.size === 0) return false;
|
|
2596
|
+
return [...text.matchAll(
|
|
2597
|
+
/(?:(?:eval|Function)\s*|\(\s*0\s*,\s*eval\s*\)\s*)\(([\s\S]{0,500}?);/g
|
|
2598
|
+
)].some((evaluation) => containsIdentifier2(evaluation[1] ?? "", tainted));
|
|
2599
|
+
});
|
|
2600
|
+
if (!directDynamicEvaluation && !executableTemplate) continue;
|
|
2601
|
+
sinks.push({ functionName, parameterName, index: match.index });
|
|
2602
|
+
}
|
|
2603
|
+
}
|
|
2604
|
+
return sinks;
|
|
2605
|
+
}
|
|
2606
|
+
var SENSITIVE_ENV_PATTERN = /process\.env\.(?:[A-Z0-9_]*(?:PASSWORD|PASSWD|SECRET|PRIVATE_KEY|DATABASE_URL|ACCESS_TOKEN|REFRESH_TOKEN|AUTH_TOKEN|SESSION_TOKEN)[A-Z0-9_]*)\b/;
|
|
2607
|
+
var UNBOUNDED_LOOP_PATTERN = /\bwhile\s*\(\s*true\s*\)|\bfor\s*\(\s*;\s*;\s*\)/g;
|
|
2608
|
+
var IDENTIFIER_ASSIGNMENT_PATTERN2 = /^\s*(?:(?:const|let|var)\s+)?([A-Za-z_$][\w$]*)\s*=\s*(.+)$/;
|
|
2609
|
+
function isExcludedSourcePath3(path) {
|
|
2610
|
+
return /(?:^|\/)(?:__tests__|test|tests|fixtures?|examples?|vendor|generated)(?:\/|$)/i.test(
|
|
2611
|
+
path
|
|
2612
|
+
) || /\.(?:test|spec|fixture)\.[cm]?[jt]sx?$/i.test(path);
|
|
2613
|
+
}
|
|
2614
|
+
function lineNumberAt5(contents, index) {
|
|
2615
|
+
return contents.slice(0, index).split("\n").length;
|
|
2616
|
+
}
|
|
2617
|
+
function finding3(ruleId, file, index, severity, summary, label, confidence) {
|
|
2618
|
+
return {
|
|
2619
|
+
ruleId,
|
|
2620
|
+
severity,
|
|
2621
|
+
confidence,
|
|
2622
|
+
summary,
|
|
2623
|
+
evidence: {
|
|
2624
|
+
path: file.path,
|
|
2625
|
+
line: lineNumberAt5(file.contents, index),
|
|
2626
|
+
label
|
|
2627
|
+
}
|
|
2628
|
+
};
|
|
2629
|
+
}
|
|
2630
|
+
function containsIdentifier2(expression, identifiers) {
|
|
2631
|
+
for (const identifier of identifiers) {
|
|
2632
|
+
const escaped2 = identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2633
|
+
if (new RegExp(`\\b${escaped2}\\b`).test(expression)) return true;
|
|
2634
|
+
}
|
|
2635
|
+
return false;
|
|
2636
|
+
}
|
|
2637
|
+
function visibleOutputIdentifiers(scopes) {
|
|
2638
|
+
const visible = /* @__PURE__ */ new Map();
|
|
2639
|
+
for (const scope of scopes) for (const [identifier, output] of scope) visible.set(identifier, output);
|
|
2640
|
+
return new Set([...visible].filter(([, output]) => output).map(([identifier]) => identifier));
|
|
2641
|
+
}
|
|
2642
|
+
function expressionContainsBoundModelOutput(expression, boundIdentifiers) {
|
|
2643
|
+
const normalized = expression.trim().replace(/;$/, "").trim();
|
|
2644
|
+
const output = normalized.match(MODEL_OUTPUT_PATTERN2);
|
|
2645
|
+
if (!output || output.index === void 0) return false;
|
|
2646
|
+
const receiver = normalized.slice(0, output.index);
|
|
2647
|
+
const identifier = receiver.match(/([A-Za-z_$][\w$]*)[.\[\]\w$\s]*$/)?.[1];
|
|
2648
|
+
return Boolean(identifier && boundIdentifiers.has(identifier));
|
|
2649
|
+
}
|
|
2650
|
+
function detectUnsafeSink(file) {
|
|
2651
|
+
if (isExcludedSourcePath3(file.path)) return [];
|
|
2652
|
+
const maskedContents = maskJavaScriptNonCode(file.contents);
|
|
2653
|
+
const executableHelpers = javaScriptExecutableHelperParameterSinks(file);
|
|
2654
|
+
const scopes = [/* @__PURE__ */ new Map()];
|
|
2655
|
+
const dangerousSinks = [
|
|
2656
|
+
...BASE_DANGEROUS_SINKS,
|
|
2657
|
+
...importedProcessSinks(file.contents, maskedContents),
|
|
2658
|
+
...executableHelpers.map(
|
|
2659
|
+
(helper) => ({
|
|
2660
|
+
name: `local executable helper ${helper.functionName}`,
|
|
2661
|
+
pattern: new RegExp(`\\b${escapeRegExp2(helper.functionName)}\\s*\\(`),
|
|
2662
|
+
severity: "critical"
|
|
2663
|
+
})
|
|
2664
|
+
)
|
|
2665
|
+
];
|
|
2666
|
+
const findings = [];
|
|
2667
|
+
const supportedCallIndices = supportedJavaScriptModelCallIndices(file.contents);
|
|
2668
|
+
let offset = 0;
|
|
2669
|
+
for (const line of maskedContents.split("\n")) {
|
|
2670
|
+
const assignment = line.match(IDENTIFIER_ASSIGNMENT_PATTERN2);
|
|
2671
|
+
const identifier = assignment?.[1];
|
|
2672
|
+
const expression = assignment?.[2];
|
|
2673
|
+
const visibleBeforeAssignment = visibleOutputIdentifiers(scopes);
|
|
2674
|
+
if (identifier && expression) {
|
|
2675
|
+
const lineHasSupportedResponseAssignment = [...supportedCallIndices].some(
|
|
2676
|
+
(index) => offset <= index && index < offset + line.length
|
|
2677
|
+
);
|
|
2678
|
+
scopes.at(-1)?.set(
|
|
2679
|
+
identifier,
|
|
2680
|
+
lineHasSupportedResponseAssignment || expressionContainsBoundModelOutput(expression, visibleBeforeAssignment)
|
|
2681
|
+
);
|
|
2682
|
+
}
|
|
2683
|
+
const outputIdentifiers = visibleOutputIdentifiers(scopes);
|
|
2684
|
+
for (const sink of dangerousSinks) {
|
|
2685
|
+
const sinkMatch = line.match(sink.pattern);
|
|
2686
|
+
if (!sinkMatch) continue;
|
|
2687
|
+
const containsModelOutput = expressionContainsBoundModelOutput(line, outputIdentifiers) || containsIdentifier2(line, outputIdentifiers);
|
|
2688
|
+
if (!containsModelOutput) continue;
|
|
2689
|
+
findings.push(
|
|
2690
|
+
finding3(
|
|
2691
|
+
"LLM-IOH-001",
|
|
2692
|
+
file,
|
|
2693
|
+
offset + (sinkMatch.index ?? 0),
|
|
2694
|
+
sink.severity,
|
|
2695
|
+
`Observable model output is passed directly to ${sink.name}.`,
|
|
2696
|
+
`Model output reaches ${sink.name}`,
|
|
2697
|
+
"high"
|
|
2698
|
+
)
|
|
2699
|
+
);
|
|
2700
|
+
}
|
|
2701
|
+
for (const character of line) {
|
|
2702
|
+
if (character === "{") scopes.push(/* @__PURE__ */ new Map());
|
|
2703
|
+
else if (character === "}" && scopes.length > 1) scopes.pop();
|
|
2704
|
+
}
|
|
2705
|
+
offset += line.length + 1;
|
|
2706
|
+
}
|
|
2707
|
+
return findings;
|
|
2708
|
+
}
|
|
2709
|
+
function matchingDelimiterEnd(contents, start, open, close) {
|
|
2710
|
+
let depth = 0;
|
|
2711
|
+
let quote = null;
|
|
2712
|
+
let escaped2 = false;
|
|
2713
|
+
let lineComment = false;
|
|
2714
|
+
let blockComment = false;
|
|
2715
|
+
for (let index = start; index < contents.length; index += 1) {
|
|
2716
|
+
const character = contents[index] ?? "";
|
|
2717
|
+
const next = contents[index + 1] ?? "";
|
|
2718
|
+
if (lineComment) {
|
|
2719
|
+
if (character === "\n") lineComment = false;
|
|
2720
|
+
continue;
|
|
2721
|
+
}
|
|
2722
|
+
if (blockComment) {
|
|
2723
|
+
if (character === "*" && next === "/") {
|
|
2724
|
+
blockComment = false;
|
|
2725
|
+
index += 1;
|
|
2726
|
+
}
|
|
2727
|
+
continue;
|
|
2728
|
+
}
|
|
2729
|
+
if (quote) {
|
|
2730
|
+
if (escaped2) escaped2 = false;
|
|
2731
|
+
else if (character === "\\") escaped2 = true;
|
|
2732
|
+
else if (character === quote) quote = null;
|
|
2733
|
+
continue;
|
|
2734
|
+
}
|
|
2735
|
+
if (character === "/" && next === "/") {
|
|
2736
|
+
lineComment = true;
|
|
2737
|
+
index += 1;
|
|
2738
|
+
continue;
|
|
2739
|
+
}
|
|
2740
|
+
if (character === "/" && next === "*") {
|
|
2741
|
+
blockComment = true;
|
|
2742
|
+
index += 1;
|
|
2743
|
+
continue;
|
|
2744
|
+
}
|
|
2745
|
+
if (character === "'" || character === '"' || character === "`") {
|
|
2746
|
+
quote = character;
|
|
2747
|
+
continue;
|
|
2748
|
+
}
|
|
2749
|
+
if (character === open) depth += 1;
|
|
2750
|
+
if (character === close) {
|
|
2751
|
+
depth -= 1;
|
|
2752
|
+
if (depth === 0) return index + 1;
|
|
2753
|
+
}
|
|
2754
|
+
}
|
|
2755
|
+
return null;
|
|
2756
|
+
}
|
|
2757
|
+
function hasObservableJavaScriptLoopExit(body) {
|
|
2758
|
+
let braceDepth = 0;
|
|
2759
|
+
for (let index = 0; index < body.length; index += 1) {
|
|
2760
|
+
const character = body[index];
|
|
2761
|
+
if (character === "{") {
|
|
2762
|
+
braceDepth += 1;
|
|
2763
|
+
continue;
|
|
2764
|
+
}
|
|
2765
|
+
if (character === "}") {
|
|
2766
|
+
braceDepth -= 1;
|
|
2767
|
+
continue;
|
|
2768
|
+
}
|
|
2769
|
+
if (braceDepth !== 0 || body.slice(index, index + 2) !== "if" || /[\w$]/.test(body[index - 1] ?? "") || /[\w$]/.test(body[index + 2] ?? "")) continue;
|
|
2770
|
+
const conditionStart = body.indexOf("(", index + 2);
|
|
2771
|
+
if (conditionStart < 0) continue;
|
|
2772
|
+
const conditionEnd = matchingDelimiterEnd(body, conditionStart, "(", ")");
|
|
2773
|
+
if (conditionEnd === null) continue;
|
|
2774
|
+
let statementStart = conditionEnd;
|
|
2775
|
+
while (/\s/.test(body[statementStart] ?? "")) statementStart += 1;
|
|
2776
|
+
if (body[statementStart] === "{") {
|
|
2777
|
+
const statementEnd = matchingDelimiterEnd(body, statementStart, "{", "}");
|
|
2778
|
+
if (statementEnd === null) continue;
|
|
2779
|
+
const statement = body.slice(statementStart + 1, statementEnd - 1);
|
|
2780
|
+
const exit = statement.search(/\b(?:break|return|throw)\b/);
|
|
2781
|
+
if (exit >= 0 && !/\b(?:for|while|switch)\b/.test(statement.slice(0, exit))) return true;
|
|
2782
|
+
continue;
|
|
2783
|
+
}
|
|
2784
|
+
if (/^(?:break|return|throw)\b/.test(body.slice(statementStart))) return true;
|
|
2785
|
+
}
|
|
2786
|
+
return false;
|
|
2787
|
+
}
|
|
2788
|
+
function detectUnboundedLoop(file) {
|
|
2789
|
+
if (isExcludedSourcePath3(file.path)) return [];
|
|
2790
|
+
const maskedContents = maskJavaScriptNonCode(file.contents);
|
|
2791
|
+
const findings = [];
|
|
2792
|
+
for (const match of maskedContents.matchAll(UNBOUNDED_LOOP_PATTERN)) {
|
|
2793
|
+
let bodyStart = match.index + match[0].length;
|
|
2794
|
+
while (/\s/.test(maskedContents[bodyStart] ?? "")) bodyStart += 1;
|
|
2795
|
+
if (maskedContents[bodyStart] !== "{") continue;
|
|
2796
|
+
const bodyEnd = matchingDelimiterEnd(maskedContents, bodyStart, "{", "}");
|
|
2797
|
+
if (bodyEnd === null) continue;
|
|
2798
|
+
const body = maskedContents.slice(bodyStart + 1, bodyEnd - 1);
|
|
2799
|
+
if (hasObservableJavaScriptLoopExit(body)) continue;
|
|
2800
|
+
const supportedCalls = supportedJavaScriptModelCallIndices(file.contents);
|
|
2801
|
+
if (![...supportedCalls].some((index) => bodyStart < index && index < bodyEnd)) continue;
|
|
2802
|
+
findings.push(
|
|
2803
|
+
finding3(
|
|
2804
|
+
"LLM-AGY-002",
|
|
2805
|
+
file,
|
|
2806
|
+
match.index,
|
|
2807
|
+
"warning",
|
|
2808
|
+
"A recognized model call occurs inside a literal unconditional loop; no static step bound is observable.",
|
|
2809
|
+
"Model call inside unconditional loop body",
|
|
2810
|
+
"medium"
|
|
2811
|
+
)
|
|
2812
|
+
);
|
|
2813
|
+
}
|
|
2814
|
+
return findings;
|
|
2815
|
+
}
|
|
2816
|
+
function modelCallArguments(contents) {
|
|
2817
|
+
const calls = [];
|
|
2818
|
+
const masked = maskJavaScriptNonCode(contents);
|
|
2819
|
+
const supportedCalls = supportedJavaScriptModelCallIndices(contents);
|
|
2820
|
+
for (const match of masked.matchAll(new RegExp(MODEL_CALL_PATTERN2, "g"))) {
|
|
2821
|
+
if (!supportedCalls.has(match.index)) continue;
|
|
2822
|
+
const openParen = match.index + match[0].lastIndexOf("(");
|
|
2823
|
+
const end = matchingDelimiterEnd(contents, openParen, "(", ")");
|
|
2824
|
+
if (end === null) continue;
|
|
2825
|
+
calls.push({ text: contents.slice(openParen + 1, end - 1), index: openParen + 1 });
|
|
2826
|
+
}
|
|
2827
|
+
return calls;
|
|
2828
|
+
}
|
|
2829
|
+
function detectSensitivePromptData(file) {
|
|
2830
|
+
if (isExcludedSourcePath3(file.path)) return [];
|
|
2831
|
+
const findings = [];
|
|
2832
|
+
for (const call of modelCallArguments(file.contents)) {
|
|
2833
|
+
const masked = maskJavaScriptNonCode(call.text);
|
|
2834
|
+
const candidates = [...call.text.matchAll(new RegExp(SENSITIVE_ENV_PATTERN, "g"))];
|
|
2835
|
+
const sensitive = candidates.find((candidate) => {
|
|
2836
|
+
const index = candidate.index;
|
|
2837
|
+
if (index === void 0) return false;
|
|
2838
|
+
if ((masked[index] ?? " ") !== " ") return true;
|
|
2839
|
+
const prefix = call.text.slice(0, index);
|
|
2840
|
+
const backticks = [...prefix.matchAll(/(?<!\\)`/g)].length;
|
|
2841
|
+
return backticks % 2 === 1 && prefix.lastIndexOf("${") > prefix.lastIndexOf("}");
|
|
2842
|
+
});
|
|
2843
|
+
if (!sensitive) continue;
|
|
2844
|
+
const relativeIndex = sensitive.index ?? 0;
|
|
2845
|
+
findings.push(
|
|
2846
|
+
finding3(
|
|
2847
|
+
"LLM-DAT-001",
|
|
2848
|
+
file,
|
|
2849
|
+
call.index + relativeIndex,
|
|
2850
|
+
"critical",
|
|
2851
|
+
"A secret-like environment value is placed directly in a model request.",
|
|
2852
|
+
`Sensitive environment value ${sensitive[0]} in model request`,
|
|
2853
|
+
"high"
|
|
2854
|
+
)
|
|
2855
|
+
);
|
|
2856
|
+
}
|
|
2857
|
+
return findings;
|
|
2858
|
+
}
|
|
2859
|
+
function hasUnsafeSinkSurface(file) {
|
|
2860
|
+
return detectUnsafeSink(file).length > 0;
|
|
2861
|
+
}
|
|
2862
|
+
var CEJEL_LLM_V1_RULES = [
|
|
2863
|
+
{
|
|
2864
|
+
id: "LLM-IOH-001",
|
|
2865
|
+
title: "Model output passed to a consequential sink",
|
|
2866
|
+
detectorConfidence: "high",
|
|
2867
|
+
evidenceContract: "A supported model-output expression, or an identifier whose latest assignment in the visible lexical scope is directly from one, appears on the same line as a supported sink; process sinks additionally require a recognized local import or require binding. A local helper is also supported when one of its parameters is observably interpolated into an executable template or passed to dynamic evaluation.",
|
|
2868
|
+
exclusions: [
|
|
2869
|
+
"Inter-procedural data flow",
|
|
2870
|
+
"Identifiers transformed, reassigned, or referenced outside their observable lexical scope",
|
|
2871
|
+
"Method calls named exec without recognized child_process or execa import evidence, including RegExp.prototype.exec",
|
|
2872
|
+
"Sinks other than the enumerated dynamic-evaluation, shell, and raw-HTML forms",
|
|
2873
|
+
"Executable helpers whose model-output argument is not observable in the same source file"
|
|
2874
|
+
],
|
|
2875
|
+
applies: hasUnsafeSinkSurface,
|
|
2876
|
+
detect: detectUnsafeSink
|
|
2877
|
+
},
|
|
2878
|
+
{
|
|
2879
|
+
id: "LLM-AGY-002",
|
|
2880
|
+
title: "Unbounded agent loop",
|
|
2881
|
+
detectorConfidence: "medium",
|
|
2882
|
+
evidenceContract: "A literal while(true) or for(;;) loop has a complete local brace-delimited body containing a recognized model call and no observable conditional counter, deadline, cancellation, budget, return, throw, or break guard.",
|
|
2883
|
+
exclusions: [
|
|
2884
|
+
"Loops bounded by runtime controls outside the source file",
|
|
2885
|
+
"Framework-specific iteration limits not visible in the loop syntax",
|
|
2886
|
+
"Local unconditional-loop bodies with an observable conditional finite counter, deadline, cancellation, budget, return, throw, or break guard"
|
|
2887
|
+
],
|
|
2888
|
+
applies: (file) => detectUnboundedLoop(file).length > 0,
|
|
2889
|
+
detect: detectUnboundedLoop
|
|
2890
|
+
},
|
|
2891
|
+
{
|
|
2892
|
+
id: "LLM-DAT-001",
|
|
2893
|
+
title: "Sensitive data passed to a model request",
|
|
2894
|
+
detectorConfidence: "high",
|
|
2895
|
+
evidenceContract: "A narrowly named secret-like process.env value occurs inside the executable, syntactically matched arguments of a recognized model call; comments and string-literal lookalikes are masked before call extraction.",
|
|
2896
|
+
exclusions: [
|
|
2897
|
+
"Provider API keys used only to construct a client",
|
|
2898
|
+
"Sensitive values passed through helpers or aliases",
|
|
2899
|
+
"General personal-data classification"
|
|
2900
|
+
],
|
|
2901
|
+
applies: (file) => detectSensitivePromptData(file).length > 0,
|
|
2902
|
+
detect: detectSensitivePromptData
|
|
2903
|
+
}
|
|
2904
|
+
];
|
|
2905
|
+
|
|
2906
|
+
// src/packs/llm/types.ts
|
|
2907
|
+
var CEJEL_LLM_PACK_ID = "free-llm";
|
|
2908
|
+
var CEJEL_LLM_PACK_VERSION = "cejel-free-llm-v0-alpha-1";
|
|
2909
|
+
var CEJEL_LLM_RULE_IDS = [
|
|
2910
|
+
"LLM-IOH-001",
|
|
2911
|
+
"LLM-VAL-001",
|
|
2912
|
+
"LLM-AGY-001",
|
|
2913
|
+
"LLM-AGY-002",
|
|
2914
|
+
"LLM-DAT-001",
|
|
2915
|
+
"LLM-PRV-001",
|
|
2916
|
+
"LLM-EVL-001",
|
|
2917
|
+
"LLM-EVL-002"
|
|
2918
|
+
];
|
|
2919
|
+
var CEJEL_LLM_ENABLED_RULE_IDS = [
|
|
2920
|
+
"LLM-IOH-001",
|
|
2921
|
+
"LLM-VAL-001",
|
|
2922
|
+
"LLM-AGY-001",
|
|
2923
|
+
"LLM-AGY-002",
|
|
2924
|
+
"LLM-DAT-001",
|
|
2925
|
+
"LLM-PRV-001",
|
|
2926
|
+
"LLM-EVL-001",
|
|
2927
|
+
"LLM-EVL-002"
|
|
2928
|
+
];
|
|
2929
|
+
var CejelLlmRuleIdSchema = external_exports.enum(CEJEL_LLM_RULE_IDS);
|
|
2930
|
+
var CejelLlmEnabledRuleIdSchema = external_exports.enum(CEJEL_LLM_ENABLED_RULE_IDS);
|
|
2931
|
+
var CejelLlmSeveritySchema = external_exports.enum(["critical", "warning", "info"]);
|
|
2932
|
+
var CejelLlmConfidenceSchema = external_exports.enum(["high", "medium", "low"]);
|
|
2933
|
+
var CejelLlmRuleStateSchema = external_exports.enum([
|
|
2934
|
+
"finding",
|
|
2935
|
+
"verified_control",
|
|
2936
|
+
"not_applicable",
|
|
2937
|
+
"insufficient_data"
|
|
2938
|
+
]);
|
|
2939
|
+
var CejelLlmPackStatusSchema = external_exports.enum([
|
|
2940
|
+
"assessed",
|
|
2941
|
+
"assessed_with_limitations",
|
|
2942
|
+
"not_applicable"
|
|
2943
|
+
]);
|
|
2944
|
+
var CejelLlmEvidenceSchema = external_exports.object({
|
|
2945
|
+
path: external_exports.string().min(1).max(700).refine(
|
|
2946
|
+
(path) => !path.startsWith("/") && !/^[A-Za-z]:[\\/]/.test(path) && !path.split(/[\\/]/).some((member) => member === ".."),
|
|
2947
|
+
{ message: "Evidence paths must be repository-relative and cannot traverse parents." }
|
|
2948
|
+
),
|
|
2949
|
+
line: external_exports.number().int().positive().nullable(),
|
|
2950
|
+
label: external_exports.string().min(1).max(180)
|
|
2951
|
+
}).strict();
|
|
2952
|
+
var CejelLlmFindingSchema = external_exports.object({
|
|
2953
|
+
// The finding schema represents the complete frozen catalogue. Whether a rule is enabled for
|
|
2954
|
+
// a detector version is recorded separately in coverage.enabledRuleIds; coupling the artifact
|
|
2955
|
+
// shape to today's subset would make adding a calibrated rule a schema-breaking change.
|
|
2956
|
+
ruleId: CejelLlmRuleIdSchema,
|
|
2957
|
+
severity: CejelLlmSeveritySchema,
|
|
2958
|
+
confidence: CejelLlmConfidenceSchema,
|
|
2959
|
+
summary: external_exports.string().min(1).max(500),
|
|
2960
|
+
evidence: CejelLlmEvidenceSchema
|
|
2961
|
+
}).strict();
|
|
2962
|
+
var CejelLlmRuleResultSchema = external_exports.object({
|
|
2963
|
+
ruleId: CejelLlmRuleIdSchema,
|
|
2964
|
+
state: CejelLlmRuleStateSchema,
|
|
2965
|
+
confidence: CejelLlmConfidenceSchema,
|
|
2966
|
+
findings: external_exports.array(CejelLlmFindingSchema),
|
|
2967
|
+
notes: external_exports.string().min(1).max(1e3)
|
|
2968
|
+
}).strict().superRefine((result, context) => {
|
|
2969
|
+
if (result.state === "finding" && result.findings.length === 0) {
|
|
2970
|
+
context.addIssue({
|
|
2971
|
+
code: external_exports.ZodIssueCode.custom,
|
|
2972
|
+
message: "A finding state must carry at least one finding.",
|
|
2973
|
+
path: ["findings"]
|
|
2974
|
+
});
|
|
2975
|
+
}
|
|
2976
|
+
if (result.state !== "finding" && result.findings.length > 0) {
|
|
2977
|
+
context.addIssue({
|
|
2978
|
+
code: external_exports.ZodIssueCode.custom,
|
|
2979
|
+
message: "Only a finding state may carry findings.",
|
|
2980
|
+
path: ["findings"]
|
|
2981
|
+
});
|
|
2982
|
+
}
|
|
2983
|
+
if (result.findings.some((finding4) => finding4.ruleId !== result.ruleId)) {
|
|
2984
|
+
context.addIssue({
|
|
2985
|
+
code: external_exports.ZodIssueCode.custom,
|
|
2986
|
+
message: "Nested findings must match the rule result id.",
|
|
2987
|
+
path: ["findings"]
|
|
2988
|
+
});
|
|
2989
|
+
}
|
|
2990
|
+
});
|
|
2991
|
+
var CejelLlmCoverageSchema = external_exports.object({
|
|
2992
|
+
supportedLanguages: external_exports.array(external_exports.enum(["JavaScript/TypeScript", "Python"])).min(1).refine((languages) => new Set(languages).size === languages.length, {
|
|
2993
|
+
message: "Supported languages must be unique."
|
|
2994
|
+
}),
|
|
2995
|
+
sourceFilesConsidered: external_exports.number().int().nonnegative(),
|
|
2996
|
+
sourceFilesWithLlmIndicators: external_exports.number().int().nonnegative(),
|
|
2997
|
+
detectedIntegrations: external_exports.array(external_exports.string().min(1)),
|
|
2998
|
+
enabledRuleIds: external_exports.array(CejelLlmEnabledRuleIdSchema),
|
|
2999
|
+
deferredRuleIds: external_exports.array(CejelLlmRuleIdSchema),
|
|
3000
|
+
limitations: external_exports.array(external_exports.string().min(1))
|
|
3001
|
+
}).strict();
|
|
3002
|
+
var CejelLlmPackResultSchema = external_exports.object({
|
|
3003
|
+
packId: external_exports.literal(CEJEL_LLM_PACK_ID),
|
|
3004
|
+
packVersion: external_exports.literal(CEJEL_LLM_PACK_VERSION),
|
|
3005
|
+
status: CejelLlmPackStatusSchema,
|
|
3006
|
+
findings: external_exports.array(CejelLlmFindingSchema),
|
|
3007
|
+
ruleResults: external_exports.array(CejelLlmRuleResultSchema),
|
|
3008
|
+
coverage: CejelLlmCoverageSchema,
|
|
3009
|
+
notes: external_exports.string().min(1).max(1e3)
|
|
3010
|
+
}).strict().superRefine((result, context) => {
|
|
3011
|
+
const expectedIds = [...CEJEL_LLM_ENABLED_RULE_IDS];
|
|
3012
|
+
const actualIds = result.ruleResults.map((rule) => rule.ruleId);
|
|
3013
|
+
if (actualIds.length !== expectedIds.length || actualIds.some((id, index) => id !== expectedIds[index])) {
|
|
3014
|
+
context.addIssue({
|
|
3015
|
+
code: external_exports.ZodIssueCode.custom,
|
|
3016
|
+
message: "Rule results must contain every enabled rule exactly once in catalogue order.",
|
|
3017
|
+
path: ["ruleResults"]
|
|
3018
|
+
});
|
|
3019
|
+
}
|
|
3020
|
+
if (result.status === "not_applicable" && result.findings.length > 0) {
|
|
3021
|
+
context.addIssue({
|
|
3022
|
+
code: external_exports.ZodIssueCode.custom,
|
|
3023
|
+
message: "A not-applicable pack cannot carry findings.",
|
|
3024
|
+
path: ["findings"]
|
|
3025
|
+
});
|
|
3026
|
+
}
|
|
3027
|
+
const expectedEnabled = [...CEJEL_LLM_ENABLED_RULE_IDS];
|
|
3028
|
+
if (result.coverage.enabledRuleIds.length !== expectedEnabled.length || result.coverage.enabledRuleIds.some((id, index) => id !== expectedEnabled[index])) {
|
|
3029
|
+
context.addIssue({
|
|
3030
|
+
code: external_exports.ZodIssueCode.custom,
|
|
3031
|
+
message: "Coverage enabled rule ids must match the detector catalogue exactly.",
|
|
3032
|
+
path: ["coverage", "enabledRuleIds"]
|
|
3033
|
+
});
|
|
3034
|
+
}
|
|
3035
|
+
const enabled = new Set(expectedEnabled);
|
|
3036
|
+
const expectedDeferred = CEJEL_LLM_RULE_IDS.filter((id) => !enabled.has(id));
|
|
3037
|
+
if (result.coverage.deferredRuleIds.length !== expectedDeferred.length || result.coverage.deferredRuleIds.some((id, index) => id !== expectedDeferred[index])) {
|
|
3038
|
+
context.addIssue({
|
|
3039
|
+
code: external_exports.ZodIssueCode.custom,
|
|
3040
|
+
message: "Coverage deferred rule ids must be the catalogue complement in order.",
|
|
3041
|
+
path: ["coverage", "deferredRuleIds"]
|
|
3042
|
+
});
|
|
3043
|
+
}
|
|
3044
|
+
const canonicalFindings = (findings) => findings.map((finding4) => JSON.stringify(finding4)).sort();
|
|
3045
|
+
const nestedFindings = result.ruleResults.flatMap((rule) => rule.findings);
|
|
3046
|
+
if (JSON.stringify(canonicalFindings(result.findings)) !== JSON.stringify(canonicalFindings(nestedFindings))) {
|
|
3047
|
+
context.addIssue({
|
|
3048
|
+
code: external_exports.ZodIssueCode.custom,
|
|
3049
|
+
message: "Top-level findings must exactly equal the nested rule findings.",
|
|
3050
|
+
path: ["findings"]
|
|
3051
|
+
});
|
|
3052
|
+
}
|
|
3053
|
+
if (result.findings.some((finding4) => finding4.confidence === "low")) {
|
|
3054
|
+
context.addIssue({
|
|
3055
|
+
code: external_exports.ZodIssueCode.custom,
|
|
3056
|
+
message: "Low-confidence candidates cannot be emitted as public artifact findings.",
|
|
3057
|
+
path: ["findings"]
|
|
3058
|
+
});
|
|
3059
|
+
}
|
|
3060
|
+
const states = result.ruleResults.map((rule) => rule.state);
|
|
3061
|
+
if (result.status === "not_applicable" && states.some((state) => state !== "not_applicable")) {
|
|
3062
|
+
context.addIssue({
|
|
3063
|
+
code: external_exports.ZodIssueCode.custom,
|
|
3064
|
+
message: "A not-applicable pack requires every rule to be not_applicable.",
|
|
3065
|
+
path: ["ruleResults"]
|
|
3066
|
+
});
|
|
3067
|
+
}
|
|
3068
|
+
if (result.status === "assessed" && states.some((state) => state === "not_applicable" || state === "insufficient_data")) {
|
|
3069
|
+
context.addIssue({
|
|
3070
|
+
code: external_exports.ZodIssueCode.custom,
|
|
3071
|
+
message: "An assessed pack cannot contain unassessed rule states.",
|
|
3072
|
+
path: ["ruleResults"]
|
|
3073
|
+
});
|
|
3074
|
+
}
|
|
3075
|
+
});
|
|
3076
|
+
|
|
3077
|
+
// src/packs/llm/detector.ts
|
|
3078
|
+
var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
3079
|
+
".js",
|
|
3080
|
+
".jsx",
|
|
3081
|
+
".mjs",
|
|
3082
|
+
".cjs",
|
|
3083
|
+
".ts",
|
|
3084
|
+
".tsx",
|
|
3085
|
+
".mts",
|
|
3086
|
+
".cts",
|
|
3087
|
+
".py"
|
|
3088
|
+
]);
|
|
3089
|
+
var MAX_SOURCE_BYTES = 1e6;
|
|
3090
|
+
var ENABLED_RULE_ID_SET = new Set(CEJEL_LLM_ENABLED_RULE_IDS);
|
|
3091
|
+
var INTEGRATION_PATTERNS = [
|
|
3092
|
+
{ name: "OpenAI SDK", pattern: /(?:from\s+['"]openai['"]|require\(\s*['"]openai['"]\s*\))/ },
|
|
3093
|
+
{
|
|
3094
|
+
name: "Anthropic SDK",
|
|
3095
|
+
pattern: /(?:from\s+['"]@anthropic-ai\/sdk['"]|require\(\s*['"]@anthropic-ai\/sdk['"]\s*\))/
|
|
3096
|
+
},
|
|
3097
|
+
{ name: "Vercel AI SDK", pattern: /(?:from\s+['"]ai['"]|require\(\s*['"]ai['"]\s*\))/ },
|
|
3098
|
+
{
|
|
3099
|
+
name: "LangChain",
|
|
3100
|
+
pattern: /(?:from\s+['"](?:@langchain\/|langchain\/)|require\(\s*['"](?:@langchain\/|langchain\/))/
|
|
3101
|
+
},
|
|
3102
|
+
{
|
|
3103
|
+
name: "OpenAI Python SDK",
|
|
3104
|
+
pattern: /(?:^|\n)\s*(?:from\s+openai\s+import\s+|import\s+openai\b)/
|
|
3105
|
+
},
|
|
3106
|
+
{
|
|
3107
|
+
name: "Anthropic Python SDK",
|
|
3108
|
+
pattern: /(?:^|\n)\s*(?:from\s+anthropic\s+import\s+|import\s+anthropic\b)/
|
|
3109
|
+
}
|
|
3110
|
+
];
|
|
3111
|
+
var RULE_CONFIDENCE = new Map(
|
|
3112
|
+
[...CEJEL_LLM_V1_RULES, ...CEJEL_LLM_ACTION_RULES, ...CEJEL_LLM_EVALUATION_RULES].map(
|
|
3113
|
+
(rule) => [rule.id, rule.detectorConfidence]
|
|
3114
|
+
)
|
|
3115
|
+
);
|
|
3116
|
+
function isPythonSource(file) {
|
|
3117
|
+
return extname(file.path).toLowerCase() === ".py";
|
|
3118
|
+
}
|
|
3119
|
+
function supportedSourcePath(path) {
|
|
3120
|
+
return SUPPORTED_EXTENSIONS.has(extname(path).toLowerCase());
|
|
3121
|
+
}
|
|
3122
|
+
function readSourceFile(repoPath, relativePath) {
|
|
3123
|
+
const root = resolve2(repoPath);
|
|
3124
|
+
const fullPath = resolve2(root, relativePath);
|
|
3125
|
+
if (fullPath !== root && !fullPath.startsWith(`${root}${sep2}`)) return null;
|
|
3126
|
+
try {
|
|
3127
|
+
const stat = lstatSync2(fullPath);
|
|
3128
|
+
if (!stat.isFile() || stat.size > MAX_SOURCE_BYTES) return null;
|
|
3129
|
+
return { path: relativePath, contents: readFileSync(fullPath, "utf8") };
|
|
3130
|
+
} catch {
|
|
3131
|
+
return null;
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
function detectedIntegrations(files) {
|
|
3135
|
+
const detected = /* @__PURE__ */ new Set();
|
|
3136
|
+
for (const file of files) {
|
|
3137
|
+
if (isExcludedLlmSourcePath(file.path)) continue;
|
|
3138
|
+
for (const integration of INTEGRATION_PATTERNS) {
|
|
3139
|
+
const detectedHere = isPythonSource(file) ? integration.pattern.test(maskPythonNonCode(file.contents)) : hasUnmaskedJavaScriptMatch(file.contents, integration.pattern);
|
|
3140
|
+
if (detectedHere) detected.add(integration.name);
|
|
3141
|
+
}
|
|
3142
|
+
}
|
|
3143
|
+
return [...detected].sort();
|
|
3144
|
+
}
|
|
3145
|
+
function snapshotCejelLlmPackInput(repoPath) {
|
|
3146
|
+
const repoFiles = listCejelLlmPackFiles(repoPath);
|
|
3147
|
+
const hash = createHash("sha256");
|
|
3148
|
+
for (const path of repoFiles.filter(supportedSourcePath)) {
|
|
3149
|
+
const file = readSourceFile(repoPath, path);
|
|
3150
|
+
if (!file) continue;
|
|
3151
|
+
hash.update(path, "utf8");
|
|
3152
|
+
hash.update("\0");
|
|
3153
|
+
hash.update(file.contents, "utf8");
|
|
3154
|
+
hash.update("\0");
|
|
3155
|
+
}
|
|
3156
|
+
return { repoFiles, sourceSha256: hash.digest("hex") };
|
|
3157
|
+
}
|
|
3158
|
+
function collectCejelLlmPack(repoPath, repoFiles) {
|
|
3159
|
+
const sourceFiles = repoFiles.filter(supportedSourcePath).map((path) => readSourceFile(repoPath, path)).filter((file) => file !== null);
|
|
3160
|
+
const integrations = detectedIntegrations(sourceFiles);
|
|
3161
|
+
const javascriptSourceFiles = sourceFiles.filter((file) => !isPythonSource(file));
|
|
3162
|
+
const javascriptLlmFiles = javascriptSourceFiles.filter(
|
|
3163
|
+
(file) => !isExcludedLlmSourcePath(file.path) && hasSupportedJavaScriptModelCall(file.contents)
|
|
3164
|
+
);
|
|
3165
|
+
const pythonLlmFiles = sourceFiles.filter(
|
|
3166
|
+
(file) => isPythonSource(file) && hasSupportedPythonLlmIntegration(file)
|
|
3167
|
+
);
|
|
3168
|
+
const actionSurfaceFiles = sourceFiles.filter(
|
|
3169
|
+
(file) => CEJEL_LLM_ACTION_RULES.some((rule) => rule.applies(file))
|
|
3170
|
+
);
|
|
3171
|
+
const evaluationSurfaceFiles = sourceFiles.filter(
|
|
3172
|
+
(file) => CEJEL_LLM_EVALUATION_RULES.some((rule) => rule.applies([file]))
|
|
3173
|
+
);
|
|
3174
|
+
const hasEvaluationSurface = CEJEL_LLM_EVALUATION_RULES.some(
|
|
3175
|
+
(rule) => rule.applies(sourceFiles)
|
|
3176
|
+
);
|
|
3177
|
+
const applicable = javascriptLlmFiles.length > 0 || pythonLlmFiles.length > 0 || actionSurfaceFiles.length > 0 || hasEvaluationSurface;
|
|
3178
|
+
const findings = applicable ? [
|
|
3179
|
+
...javascriptLlmFiles.flatMap(
|
|
3180
|
+
(file) => CEJEL_LLM_V1_RULES.flatMap((rule) => rule.detect(file))
|
|
3181
|
+
),
|
|
3182
|
+
// A model-facing tool registration is itself a frozen-contract activator. Action rules
|
|
3183
|
+
// therefore inspect every supported source file rather than depending on an unrelated
|
|
3184
|
+
// direct official-SDK call in the same file.
|
|
3185
|
+
...sourceFiles.flatMap(
|
|
3186
|
+
(file) => CEJEL_LLM_ACTION_RULES.flatMap((rule) => rule.detect(file))
|
|
3187
|
+
),
|
|
3188
|
+
...pythonLlmFiles.flatMap(
|
|
3189
|
+
(file) => CEJEL_LLM_PYTHON_RULES.flatMap((rule) => rule.detect(file))
|
|
3190
|
+
),
|
|
3191
|
+
// Evaluation rules inspect all supported sources and enforce their own bounded,
|
|
3192
|
+
// language-specific evidence contracts. Repository-level applicability alone is never
|
|
3193
|
+
// evidence that an unrelated metrics writer is an LLM evaluation.
|
|
3194
|
+
...detectCejelLlmEvaluationRules(sourceFiles)
|
|
3195
|
+
] : [];
|
|
3196
|
+
const ruleResults = CEJEL_LLM_ENABLED_RULE_IDS.map((ruleId) => {
|
|
3197
|
+
const ruleFindings = findings.filter((finding4) => finding4.ruleId === ruleId);
|
|
3198
|
+
const confidence = RULE_CONFIDENCE.get(ruleId) ?? "low";
|
|
3199
|
+
if (!applicable) {
|
|
3200
|
+
return {
|
|
3201
|
+
ruleId,
|
|
3202
|
+
state: "not_applicable",
|
|
3203
|
+
confidence,
|
|
3204
|
+
findings: [],
|
|
3205
|
+
notes: "No supported LLM call path was detected, so this rule was not assessed."
|
|
3206
|
+
};
|
|
3207
|
+
}
|
|
3208
|
+
if (ruleFindings.length > 0) {
|
|
3209
|
+
return {
|
|
3210
|
+
ruleId,
|
|
3211
|
+
state: "finding",
|
|
3212
|
+
confidence,
|
|
3213
|
+
findings: ruleFindings,
|
|
3214
|
+
notes: "The rule evidence contract matched observable source evidence."
|
|
3215
|
+
};
|
|
3216
|
+
}
|
|
3217
|
+
const ruleApplicable = CEJEL_LLM_V1_RULES.filter((rule) => rule.id === ruleId).some(
|
|
3218
|
+
(rule) => javascriptLlmFiles.some((file) => rule.applies(file))
|
|
3219
|
+
) || CEJEL_LLM_PYTHON_RULES.filter((rule) => rule.id === ruleId).some(
|
|
3220
|
+
(rule) => pythonLlmFiles.some((file) => rule.applies(file))
|
|
3221
|
+
) || CEJEL_LLM_ACTION_RULES.filter((rule) => rule.id === ruleId).some(
|
|
3222
|
+
(rule) => sourceFiles.some((file) => rule.applies(file))
|
|
3223
|
+
) || CEJEL_LLM_EVALUATION_RULES.filter((rule) => rule.id === ruleId).some(
|
|
3224
|
+
(rule) => rule.applies(sourceFiles)
|
|
3225
|
+
);
|
|
3226
|
+
if (!ruleApplicable) {
|
|
3227
|
+
return {
|
|
3228
|
+
ruleId,
|
|
3229
|
+
state: "not_applicable",
|
|
3230
|
+
confidence,
|
|
3231
|
+
findings: [],
|
|
3232
|
+
notes: "The rule-specific surface was not observed in supported local source."
|
|
3233
|
+
};
|
|
3234
|
+
}
|
|
3235
|
+
return {
|
|
3236
|
+
ruleId,
|
|
3237
|
+
state: "insufficient_data",
|
|
3238
|
+
confidence,
|
|
3239
|
+
findings: [],
|
|
3240
|
+
notes: "No finding matched, but this alpha detector does not yet prove that a control exists."
|
|
3241
|
+
};
|
|
3242
|
+
});
|
|
3243
|
+
const status = applicable ? "assessed_with_limitations" : "not_applicable";
|
|
3244
|
+
const indicatorPaths = /* @__PURE__ */ new Set([
|
|
3245
|
+
...javascriptLlmFiles.map((file) => file.path),
|
|
3246
|
+
...pythonLlmFiles.map((file) => file.path),
|
|
3247
|
+
...actionSurfaceFiles.map((file) => file.path),
|
|
3248
|
+
...evaluationSurfaceFiles.map((file) => file.path)
|
|
3249
|
+
]);
|
|
3250
|
+
const integrationSet = new Set(integrations);
|
|
3251
|
+
if (actionSurfaceFiles.length > 0) integrationSet.add("Model-facing tool registration");
|
|
3252
|
+
if (hasEvaluationSurface) integrationSet.add("Declared LLM evaluation surface");
|
|
3253
|
+
return CejelLlmPackResultSchema.parse({
|
|
3254
|
+
packId: CEJEL_LLM_PACK_ID,
|
|
3255
|
+
packVersion: CEJEL_LLM_PACK_VERSION,
|
|
3256
|
+
status,
|
|
3257
|
+
findings,
|
|
3258
|
+
ruleResults,
|
|
3259
|
+
coverage: {
|
|
3260
|
+
supportedLanguages: ["JavaScript/TypeScript", "Python"],
|
|
3261
|
+
sourceFilesConsidered: sourceFiles.length,
|
|
3262
|
+
sourceFilesWithLlmIndicators: indicatorPaths.size,
|
|
3263
|
+
detectedIntegrations: [...integrationSet].sort(),
|
|
3264
|
+
enabledRuleIds: [...CEJEL_LLM_ENABLED_RULE_IDS],
|
|
3265
|
+
deferredRuleIds: CEJEL_LLM_RULE_IDS.filter(
|
|
3266
|
+
(ruleId) => !ENABLED_RULE_ID_SET.has(ruleId)
|
|
3267
|
+
),
|
|
3268
|
+
limitations: [
|
|
3269
|
+
"Alpha coverage is limited to fixture-backed JavaScript, TypeScript, and Python source patterns in files up to 1 MB.",
|
|
3270
|
+
"Rules use bounded, observable source patterns and do not claim whole-program data-flow analysis.",
|
|
3271
|
+
"Action-governance and evaluation-hygiene rules require one of the explicitly supported complete local JavaScript, TypeScript, or Python evidence paths.",
|
|
3272
|
+
"No model-quality, factuality, prompt-injection-resistance, or general hallucination rate is measured."
|
|
3273
|
+
]
|
|
3274
|
+
},
|
|
3275
|
+
notes: applicable ? "Static LLM application-integrity alpha; findings require calibration before public release." : "No supported LLM application surface was detected; LLM controls were not scored."
|
|
3276
|
+
});
|
|
3277
|
+
}
|
|
3278
|
+
|
|
3279
|
+
// src/packs/llm/artifact.ts
|
|
3280
|
+
import { createHash as createHash2 } from "crypto";
|
|
3281
|
+
var CEJEL_LLM_ARTIFACT_SCHEMA_VERSION = "cejel-free-llm-artifact-v1";
|
|
3282
|
+
var CEJEL_LLM_RULE_CONTRACT_VERSION = "cejel-free-llm-rules-v1-2026-07-22";
|
|
3283
|
+
var CEJEL_LLM_DETECTOR_VERSION = "free-llm-detector-alpha-1";
|
|
3284
|
+
var CEJEL_LLM_ATTESTATION_PREDICATE = "https://cejel.dev/attestations/free-llm/v1";
|
|
3285
|
+
var CejelLlmPackArtifactSchema = external_exports.object({
|
|
3286
|
+
schemaVersion: external_exports.literal(CEJEL_LLM_ARTIFACT_SCHEMA_VERSION),
|
|
3287
|
+
generatedAt: external_exports.string().datetime({ offset: true }),
|
|
3288
|
+
repo: external_exports.object({
|
|
3289
|
+
path: external_exports.string().min(1).max(700),
|
|
3290
|
+
headSha: external_exports.string().min(7).max(64).optional()
|
|
3291
|
+
}).strict(),
|
|
3292
|
+
baseReportSha256: external_exports.string().regex(/^[a-f0-9]{64}$/),
|
|
3293
|
+
inputSourceSha256: external_exports.string().regex(/^[a-f0-9]{64}$/),
|
|
3294
|
+
lineage: external_exports.object({
|
|
3295
|
+
ruleContractVersion: external_exports.literal(CEJEL_LLM_RULE_CONTRACT_VERSION),
|
|
3296
|
+
detectorVersion: external_exports.literal(CEJEL_LLM_DETECTOR_VERSION),
|
|
3297
|
+
toolVersion: external_exports.string().min(1).max(100),
|
|
3298
|
+
detectorSourceRevision: external_exports.string().min(1).max(160)
|
|
3299
|
+
}).strict(),
|
|
3300
|
+
result: CejelLlmPackResultSchema,
|
|
3301
|
+
assurance: external_exports.object({
|
|
3302
|
+
status: external_exports.literal("unsigned"),
|
|
3303
|
+
issuer: external_exports.literal("self-generated")
|
|
3304
|
+
}).strict(),
|
|
3305
|
+
claimBoundary: external_exports.literal(
|
|
3306
|
+
"Static application-integrity and evaluation-hygiene evidence; not a model hallucination rate or universal safety certificate."
|
|
3307
|
+
)
|
|
3308
|
+
}).strict();
|
|
3309
|
+
function createCejelLlmPackArtifact(result, options) {
|
|
3310
|
+
return CejelLlmPackArtifactSchema.parse({
|
|
3311
|
+
schemaVersion: CEJEL_LLM_ARTIFACT_SCHEMA_VERSION,
|
|
3312
|
+
generatedAt: options.generatedAt,
|
|
3313
|
+
repo: {
|
|
3314
|
+
path: options.repoPath,
|
|
3315
|
+
...options.headSha ? { headSha: options.headSha } : {}
|
|
3316
|
+
},
|
|
3317
|
+
baseReportSha256: options.baseReportSha256,
|
|
3318
|
+
inputSourceSha256: options.inputSourceSha256,
|
|
3319
|
+
lineage: {
|
|
3320
|
+
ruleContractVersion: CEJEL_LLM_RULE_CONTRACT_VERSION,
|
|
3321
|
+
detectorVersion: CEJEL_LLM_DETECTOR_VERSION,
|
|
3322
|
+
toolVersion: options.toolVersion,
|
|
3323
|
+
detectorSourceRevision: options.detectorSourceRevision ?? `local-or-unverified-build:@cejel/cejel@${options.toolVersion}`
|
|
3324
|
+
},
|
|
3325
|
+
result,
|
|
3326
|
+
assurance: { status: "unsigned", issuer: "self-generated" },
|
|
3327
|
+
claimBoundary: "Static application-integrity and evaluation-hygiene evidence; not a model hallucination rate or universal safety certificate."
|
|
3328
|
+
});
|
|
3329
|
+
}
|
|
3330
|
+
function serializeCejelLlmPackArtifact(artifact) {
|
|
3331
|
+
return `${JSON.stringify(CejelLlmPackArtifactSchema.parse(artifact), null, 2)}
|
|
3332
|
+
`;
|
|
3333
|
+
}
|
|
3334
|
+
var CejelLlmPackAttestationSchema = external_exports.object({
|
|
3335
|
+
_type: external_exports.literal("https://in-toto.io/Statement/v1"),
|
|
3336
|
+
subject: external_exports.tuple([
|
|
3337
|
+
external_exports.object({
|
|
3338
|
+
name: external_exports.literal("llm-report.json"),
|
|
3339
|
+
digest: external_exports.object({ sha256: external_exports.string().regex(/^[a-f0-9]{64}$/) }).strict()
|
|
3340
|
+
}).strict()
|
|
3341
|
+
]),
|
|
3342
|
+
predicateType: external_exports.literal(CEJEL_LLM_ATTESTATION_PREDICATE),
|
|
3343
|
+
predicate: external_exports.object({
|
|
3344
|
+
packId: external_exports.string().min(1),
|
|
3345
|
+
packVersion: external_exports.string().min(1),
|
|
3346
|
+
generatedAt: external_exports.string().datetime({ offset: true }),
|
|
3347
|
+
baseReportSha256: external_exports.string().regex(/^[a-f0-9]{64}$/),
|
|
3348
|
+
inputSourceSha256: external_exports.string().regex(/^[a-f0-9]{64}$/),
|
|
3349
|
+
ruleContractVersion: external_exports.literal(CEJEL_LLM_RULE_CONTRACT_VERSION),
|
|
3350
|
+
detectorVersion: external_exports.literal(CEJEL_LLM_DETECTOR_VERSION),
|
|
3351
|
+
toolVersion: external_exports.string().min(1).max(100),
|
|
3352
|
+
detectorSourceRevision: external_exports.string().min(1).max(160),
|
|
3353
|
+
assurance: external_exports.object({
|
|
3354
|
+
status: external_exports.literal("unsigned"),
|
|
3355
|
+
issuer: external_exports.literal("self-generated")
|
|
3356
|
+
}).strict()
|
|
3357
|
+
}).strict()
|
|
3358
|
+
}).strict();
|
|
3359
|
+
function createCejelLlmPackAttestation(artifact, serializedArtifact) {
|
|
3360
|
+
const parsed = CejelLlmPackArtifactSchema.parse(artifact);
|
|
3361
|
+
const serializedValue = CejelLlmPackArtifactSchema.parse(JSON.parse(serializedArtifact));
|
|
3362
|
+
if (JSON.stringify(serializedValue) !== JSON.stringify(parsed)) {
|
|
3363
|
+
throw new Error("Serialized LLM artifact bytes do not represent the supplied artifact.");
|
|
3364
|
+
}
|
|
3365
|
+
return CejelLlmPackAttestationSchema.parse({
|
|
3366
|
+
_type: "https://in-toto.io/Statement/v1",
|
|
3367
|
+
subject: [
|
|
3368
|
+
{
|
|
3369
|
+
name: "llm-report.json",
|
|
3370
|
+
digest: { sha256: createHash2("sha256").update(serializedArtifact).digest("hex") }
|
|
3371
|
+
}
|
|
3372
|
+
],
|
|
3373
|
+
predicateType: CEJEL_LLM_ATTESTATION_PREDICATE,
|
|
3374
|
+
predicate: {
|
|
3375
|
+
packId: parsed.result.packId,
|
|
3376
|
+
packVersion: parsed.result.packVersion,
|
|
3377
|
+
generatedAt: parsed.generatedAt,
|
|
3378
|
+
baseReportSha256: parsed.baseReportSha256,
|
|
3379
|
+
inputSourceSha256: parsed.inputSourceSha256,
|
|
3380
|
+
ruleContractVersion: parsed.lineage.ruleContractVersion,
|
|
3381
|
+
detectorVersion: parsed.lineage.detectorVersion,
|
|
3382
|
+
toolVersion: parsed.lineage.toolVersion,
|
|
3383
|
+
detectorSourceRevision: parsed.lineage.detectorSourceRevision,
|
|
3384
|
+
assurance: parsed.assurance
|
|
3385
|
+
}
|
|
3386
|
+
});
|
|
3387
|
+
}
|
|
3388
|
+
function verifyCejelLlmPackAttestationBinding(statement, artifact, serializedArtifact) {
|
|
3389
|
+
const parsedStatement = CejelLlmPackAttestationSchema.safeParse(statement);
|
|
3390
|
+
if (!parsedStatement.success) {
|
|
3391
|
+
return {
|
|
3392
|
+
valid: false,
|
|
3393
|
+
errors: parsedStatement.error.issues.map(
|
|
3394
|
+
(issue) => `attestation ${issue.path.join(".") || "<root>"}: ${issue.message}`
|
|
3395
|
+
)
|
|
3396
|
+
};
|
|
3397
|
+
}
|
|
3398
|
+
const parsedArtifact = CejelLlmPackArtifactSchema.parse(artifact);
|
|
3399
|
+
const value = parsedStatement.data;
|
|
3400
|
+
const errors = [];
|
|
3401
|
+
let serializedValue = null;
|
|
3402
|
+
try {
|
|
3403
|
+
serializedValue = CejelLlmPackArtifactSchema.parse(
|
|
3404
|
+
JSON.parse(Buffer.isBuffer(serializedArtifact) ? serializedArtifact.toString("utf8") : serializedArtifact)
|
|
3405
|
+
);
|
|
3406
|
+
} catch {
|
|
3407
|
+
errors.push("serialized llm-report.json is not the supplied valid artifact");
|
|
3408
|
+
}
|
|
3409
|
+
if (serializedValue && JSON.stringify(serializedValue) !== JSON.stringify(parsedArtifact)) {
|
|
3410
|
+
errors.push("serialized llm-report.json does not represent the supplied artifact");
|
|
3411
|
+
}
|
|
3412
|
+
const artifactSha256 = createHash2("sha256").update(serializedArtifact).digest("hex");
|
|
3413
|
+
if (value.subject[0].digest.sha256 !== artifactSha256) {
|
|
3414
|
+
errors.push("subject digest does not match llm-report.json");
|
|
3415
|
+
}
|
|
3416
|
+
if (value.predicate.packId !== parsedArtifact.result.packId) {
|
|
3417
|
+
errors.push("predicate pack id does not match llm-report.json");
|
|
3418
|
+
}
|
|
3419
|
+
if (value.predicate.packVersion !== parsedArtifact.result.packVersion) {
|
|
3420
|
+
errors.push("predicate pack version does not match llm-report.json");
|
|
3421
|
+
}
|
|
3422
|
+
if (value.predicate.generatedAt !== parsedArtifact.generatedAt) {
|
|
3423
|
+
errors.push("predicate generated timestamp does not match llm-report.json");
|
|
3424
|
+
}
|
|
3425
|
+
if (value.predicate.baseReportSha256 !== parsedArtifact.baseReportSha256) {
|
|
3426
|
+
errors.push("predicate base report digest does not match llm-report.json");
|
|
3427
|
+
}
|
|
3428
|
+
if (value.predicate.inputSourceSha256 !== parsedArtifact.inputSourceSha256) {
|
|
3429
|
+
errors.push("predicate input source digest does not match llm-report.json");
|
|
3430
|
+
}
|
|
3431
|
+
if (value.predicate.ruleContractVersion !== parsedArtifact.lineage.ruleContractVersion) {
|
|
3432
|
+
errors.push("predicate rule contract version does not match llm-report.json");
|
|
3433
|
+
}
|
|
3434
|
+
if (value.predicate.detectorVersion !== parsedArtifact.lineage.detectorVersion) {
|
|
3435
|
+
errors.push("predicate detector version does not match llm-report.json");
|
|
3436
|
+
}
|
|
3437
|
+
if (value.predicate.toolVersion !== parsedArtifact.lineage.toolVersion) {
|
|
3438
|
+
errors.push("predicate tool version does not match llm-report.json");
|
|
3439
|
+
}
|
|
3440
|
+
if (value.predicate.detectorSourceRevision !== parsedArtifact.lineage.detectorSourceRevision) {
|
|
3441
|
+
errors.push("predicate detector source revision does not match llm-report.json");
|
|
3442
|
+
}
|
|
3443
|
+
return { valid: errors.length === 0, errors };
|
|
3444
|
+
}
|
|
3445
|
+
|
|
3446
|
+
// src/packs/llm/render.ts
|
|
3447
|
+
function escapeHtml(value) {
|
|
3448
|
+
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
3449
|
+
}
|
|
3450
|
+
function renderCejelLlmPackHtml(artifact) {
|
|
3451
|
+
const { result } = artifact;
|
|
3452
|
+
const findings = result.findings.map(
|
|
3453
|
+
(finding4) => `<li><strong>${escapeHtml(finding4.ruleId)}</strong> <span>${escapeHtml(finding4.severity)} / ${escapeHtml(finding4.confidence)}</span><p>${escapeHtml(finding4.summary)}</p><code>${escapeHtml(finding4.evidence.path)}:${finding4.evidence.line}</code></li>`
|
|
3454
|
+
).join("\n");
|
|
3455
|
+
const limitations = result.coverage.limitations.map((limitation) => `<li>${escapeHtml(limitation)}</li>`).join("\n");
|
|
3456
|
+
return `<!doctype html>
|
|
3457
|
+
<html lang="en">
|
|
3458
|
+
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Cejel Free LLM Pack</title>
|
|
3459
|
+
<style>body{font:16px/1.5 system-ui,sans-serif;max-width:900px;margin:40px auto;padding:0 20px;color:#17202a}code{background:#f2f4f7;padding:2px 5px}li{margin:14px 0}.boundary{border-left:4px solid #b7791f;padding:10px 14px;background:#fffaf0}</style></head>
|
|
3460
|
+
<body><h1>Cejel Free LLM Pack</h1>
|
|
3461
|
+
<p><strong>Status:</strong> ${escapeHtml(result.status)} \xB7 <strong>Findings:</strong> ${result.findings.length}</p>
|
|
3462
|
+
<p><strong>Pack:</strong> ${escapeHtml(result.packVersion)} \xB7 <strong>Generated:</strong> ${escapeHtml(artifact.generatedAt)}</p>
|
|
3463
|
+
<p class="boundary">${escapeHtml(artifact.claimBoundary)}</p>
|
|
3464
|
+
<h2>Evidence-backed findings</h2>${findings ? `<ol>${findings}</ol>` : "<p>No rule evidence matched. This is not proof that every control exists.</p>"}
|
|
3465
|
+
<h2>Coverage limitations</h2><ul>${limitations}</ul>
|
|
3466
|
+
<p>Assurance: unsigned, self-generated. Source snapshot SHA-256: <code>${escapeHtml(artifact.inputSourceSha256)}</code>. Base report SHA-256: <code>${escapeHtml(artifact.baseReportSha256)}</code>.</p>
|
|
3467
|
+
</body></html>
|
|
3468
|
+
`;
|
|
3469
|
+
}
|
|
3470
|
+
function renderCejelLlmPackTerminal(artifact) {
|
|
3471
|
+
const { result } = artifact;
|
|
3472
|
+
const lines = [
|
|
3473
|
+
"Free LLM Pack",
|
|
3474
|
+
` Status: ${result.status}`,
|
|
3475
|
+
` Findings: ${result.findings.length}`,
|
|
3476
|
+
` Coverage: ${result.coverage.supportedLanguages.join(", ")}`,
|
|
3477
|
+
" Boundary: static application-integrity evidence; not a hallucination rate."
|
|
3478
|
+
];
|
|
3479
|
+
return `${lines.join("\n")}
|
|
3480
|
+
`;
|
|
3481
|
+
}
|
|
3482
|
+
|
|
20
3483
|
// src/terminal.ts
|
|
21
3484
|
function renderTerminalCertificate(summary) {
|
|
22
3485
|
const abstained = summary.verdict === "Insufficient source";
|
|
@@ -50,8 +3513,8 @@ function renderTerminalCertificate(summary) {
|
|
|
50
3513
|
lines.push("No evidence-backed findings.");
|
|
51
3514
|
} else {
|
|
52
3515
|
lines.push(`Top findings (${summary.findingCount} total):`);
|
|
53
|
-
for (const
|
|
54
|
-
lines.push(` [${
|
|
3516
|
+
for (const finding4 of summary.topFindings) {
|
|
3517
|
+
lines.push(` [${finding4.severity}] ${finding4.criterionId}: ${finding4.summary}`);
|
|
55
3518
|
}
|
|
56
3519
|
const remaining = summary.findingCount - summary.topFindings.length;
|
|
57
3520
|
if (remaining > 0) {
|
|
@@ -63,9 +3526,9 @@ function renderTerminalCertificate(summary) {
|
|
|
63
3526
|
"",
|
|
64
3527
|
`External findings (${summary.externalFindingCount} total, attributed to tool + criterion):`
|
|
65
3528
|
);
|
|
66
|
-
for (const
|
|
3529
|
+
for (const finding4 of summary.topExternalFindings) {
|
|
67
3530
|
lines.push(
|
|
68
|
-
` [${
|
|
3531
|
+
` [${finding4.severity}] ${finding4.label} \u2192 ${finding4.dimension}: ${finding4.ruleId} \u2014 ${finding4.message}${finding4.location ? ` (${finding4.location})` : ""}`
|
|
69
3532
|
);
|
|
70
3533
|
}
|
|
71
3534
|
const remaining = summary.externalFindingCount - summary.topExternalFindings.length;
|
|
@@ -102,6 +3565,12 @@ var CLI_FLAG_SPECS = [
|
|
|
102
3565
|
description: "fold in a SARIF/JSON scanner report (repeatable)",
|
|
103
3566
|
kind: "ingest"
|
|
104
3567
|
},
|
|
3568
|
+
{
|
|
3569
|
+
tokens: ["--pack"],
|
|
3570
|
+
value: "<llm>",
|
|
3571
|
+
description: "run an opt-in pack and write its separate evidence artifact (repeatable)",
|
|
3572
|
+
kind: "pack"
|
|
3573
|
+
},
|
|
105
3574
|
{
|
|
106
3575
|
tokens: ["--quiet"],
|
|
107
3576
|
description: "suppress the terminal summary",
|
|
@@ -126,7 +3595,7 @@ var CLI_FLAG_KIND_BY_TOKEN = new Map(
|
|
|
126
3595
|
);
|
|
127
3596
|
function readCliManifest() {
|
|
128
3597
|
return JSON.parse(
|
|
129
|
-
|
|
3598
|
+
readFileSync2(new URL("../package.json", import.meta.url), "utf8")
|
|
130
3599
|
);
|
|
131
3600
|
}
|
|
132
3601
|
function cliVersion() {
|
|
@@ -193,29 +3662,69 @@ async function runWitanFreeCli(args) {
|
|
|
193
3662
|
process.stdout.write(USAGE);
|
|
194
3663
|
return 0;
|
|
195
3664
|
}
|
|
3665
|
+
const initialLlmSnapshot = options.packs.includes("llm") ? snapshotCejelLlmPackInput(options.repoPath) : null;
|
|
196
3666
|
const { report, summary } = runCejelScan({
|
|
197
3667
|
repoPath: options.repoPath,
|
|
198
3668
|
ingestPatterns: options.ingestPatterns,
|
|
199
3669
|
warnOnEmptyIngestMatch: !options.quiet
|
|
200
3670
|
});
|
|
201
3671
|
const attestation = createWitanAttestation(report, { toolVersion: cliVersion() });
|
|
3672
|
+
const serializedReport = serializeWitanReport(report);
|
|
202
3673
|
mkdirSync(options.outDir, { recursive: true });
|
|
203
|
-
writeFileSync(
|
|
3674
|
+
writeFileSync(join2(options.outDir, "report.json"), serializedReport, "utf8");
|
|
204
3675
|
writeFileSync(
|
|
205
|
-
|
|
3676
|
+
join2(options.outDir, "attestation.json"),
|
|
206
3677
|
JSON.stringify(attestation, null, 2),
|
|
207
3678
|
"utf8"
|
|
208
3679
|
);
|
|
209
|
-
writeFileSync(
|
|
3680
|
+
writeFileSync(join2(options.outDir, "certificate.html"), renderWitanHtmlReport(report), "utf8");
|
|
210
3681
|
writeFileSync(
|
|
211
|
-
|
|
3682
|
+
join2(options.outDir, "badge.json"),
|
|
212
3683
|
JSON.stringify(renderWitanBadgeEndpoint(report), null, 2),
|
|
213
3684
|
"utf8"
|
|
214
3685
|
);
|
|
215
|
-
writeFileSync(
|
|
216
|
-
writeFileSync(
|
|
3686
|
+
writeFileSync(join2(options.outDir, "badge.svg"), renderWitanBadgeSvg(report), "utf8");
|
|
3687
|
+
writeFileSync(join2(options.outDir, "summary.json"), JSON.stringify(summary, null, 2), "utf8");
|
|
3688
|
+
let llmTerminal = "";
|
|
3689
|
+
if (options.packs.includes("llm")) {
|
|
3690
|
+
const llmSnapshot = snapshotCejelLlmPackInput(options.repoPath);
|
|
3691
|
+
if (!initialLlmSnapshot || initialLlmSnapshot.sourceSha256 !== llmSnapshot.sourceSha256) {
|
|
3692
|
+
throw new Error("Cejel: repository source changed between the base and LLM pack scans; rerun on a stable tree.");
|
|
3693
|
+
}
|
|
3694
|
+
const llmResult = collectCejelLlmPack(options.repoPath, llmSnapshot.repoFiles);
|
|
3695
|
+
const finalLlmSnapshot = snapshotCejelLlmPackInput(options.repoPath);
|
|
3696
|
+
if (finalLlmSnapshot.sourceSha256 !== llmSnapshot.sourceSha256) {
|
|
3697
|
+
throw new Error("Cejel: repository source changed during the LLM pack scan; rerun on a stable tree.");
|
|
3698
|
+
}
|
|
3699
|
+
const llmArtifact = createCejelLlmPackArtifact(llmResult, {
|
|
3700
|
+
generatedAt: report.generatedAt,
|
|
3701
|
+
repoPath: options.repoPath,
|
|
3702
|
+
...report.repo.headSha ? { headSha: report.repo.headSha } : {},
|
|
3703
|
+
baseReportSha256: createHash3("sha256").update(serializedReport).digest("hex"),
|
|
3704
|
+
inputSourceSha256: llmSnapshot.sourceSha256,
|
|
3705
|
+
toolVersion: cliVersion(),
|
|
3706
|
+
detectorSourceRevision: `local-or-unverified-build:@cejel/cejel@${cliVersion()}`
|
|
3707
|
+
});
|
|
3708
|
+
const serializedLlmArtifact = serializeCejelLlmPackArtifact(llmArtifact);
|
|
3709
|
+
const llmAttestation = createCejelLlmPackAttestation(llmArtifact, serializedLlmArtifact);
|
|
3710
|
+
writeFileSync(join2(options.outDir, "llm-report.json"), serializedLlmArtifact, "utf8");
|
|
3711
|
+
writeFileSync(
|
|
3712
|
+
join2(options.outDir, "llm-attestation.json"),
|
|
3713
|
+
`${JSON.stringify(llmAttestation, null, 2)}
|
|
3714
|
+
`,
|
|
3715
|
+
"utf8"
|
|
3716
|
+
);
|
|
3717
|
+
writeFileSync(
|
|
3718
|
+
join2(options.outDir, "llm-certificate.html"),
|
|
3719
|
+
renderCejelLlmPackHtml(llmArtifact),
|
|
3720
|
+
"utf8"
|
|
3721
|
+
);
|
|
3722
|
+
llmTerminal = renderCejelLlmPackTerminal(llmArtifact);
|
|
3723
|
+
}
|
|
217
3724
|
if (!options.quiet) {
|
|
218
3725
|
process.stdout.write(renderTerminalCertificate(summary));
|
|
3726
|
+
if (llmTerminal) process.stdout.write(`
|
|
3727
|
+
${llmTerminal}`);
|
|
219
3728
|
process.stdout.write(
|
|
220
3729
|
`
|
|
221
3730
|
Wrote:
|
|
@@ -224,7 +3733,10 @@ Wrote:
|
|
|
224
3733
|
${options.outDir}/certificate.html
|
|
225
3734
|
${options.outDir}/badge.json
|
|
226
3735
|
${options.outDir}/badge.svg
|
|
227
|
-
`
|
|
3736
|
+
${options.packs.includes("llm") ? ` ${options.outDir}/llm-report.json
|
|
3737
|
+
${options.outDir}/llm-attestation.json
|
|
3738
|
+
${options.outDir}/llm-certificate.html
|
|
3739
|
+
` : ""}`
|
|
228
3740
|
);
|
|
229
3741
|
}
|
|
230
3742
|
if (options.minScore != null) {
|
|
@@ -267,14 +3779,45 @@ function parseCliInvocation(args) {
|
|
|
267
3779
|
}
|
|
268
3780
|
return {
|
|
269
3781
|
command: "verify",
|
|
270
|
-
reportPath:
|
|
271
|
-
attestationPath:
|
|
3782
|
+
reportPath: resolve3(reportPath),
|
|
3783
|
+
attestationPath: resolve3(attestationPath)
|
|
272
3784
|
};
|
|
273
3785
|
}
|
|
274
3786
|
return { command: "scan", options: parseArgs(args) };
|
|
275
3787
|
}
|
|
276
3788
|
function runVerifyBinding(reportPath, attestationPath) {
|
|
277
3789
|
const reportArtifact = readJsonArtifact(reportPath, "report");
|
|
3790
|
+
const statement = readJsonArtifact(attestationPath, "attestation");
|
|
3791
|
+
if (isLlmPackAttestation(statement.value)) {
|
|
3792
|
+
const artifactResult = CejelLlmPackArtifactSchema.safeParse(reportArtifact.value);
|
|
3793
|
+
if (!artifactResult.success) {
|
|
3794
|
+
const members = artifactResult.error.issues.map(
|
|
3795
|
+
(issue) => `${issue.path.join(".") || "<root>"}: ${issue.message}`
|
|
3796
|
+
);
|
|
3797
|
+
process.stderr.write(
|
|
3798
|
+
`Cejel: Free LLM Pack artifact validation failed:
|
|
3799
|
+
${members.map((member) => ` - ${member}`).join("\n")}
|
|
3800
|
+
`
|
|
3801
|
+
);
|
|
3802
|
+
return 1;
|
|
3803
|
+
}
|
|
3804
|
+
const result2 = verifyCejelLlmPackAttestationBinding(
|
|
3805
|
+
statement.value,
|
|
3806
|
+
artifactResult.data,
|
|
3807
|
+
reportArtifact.contents
|
|
3808
|
+
);
|
|
3809
|
+
if (!result2.valid) {
|
|
3810
|
+
process.stderr.write(
|
|
3811
|
+
`Cejel: Free LLM Pack artifact/attestation binding verification failed:
|
|
3812
|
+
${result2.errors.map((error) => ` - ${error}`).join("\n")}
|
|
3813
|
+
`
|
|
3814
|
+
);
|
|
3815
|
+
return 1;
|
|
3816
|
+
}
|
|
3817
|
+
process.stdout.write("Cejel: Free LLM Pack artifact/attestation binding verified.\n");
|
|
3818
|
+
process.stdout.write("Cejel: signature and signer identity were not verified.\n");
|
|
3819
|
+
return 0;
|
|
3820
|
+
}
|
|
278
3821
|
const reportResult = WitanReportSchema.safeParse(reportArtifact.value);
|
|
279
3822
|
if (!reportResult.success) {
|
|
280
3823
|
const members = reportResult.error.issues.map(
|
|
@@ -287,8 +3830,7 @@ ${members.map((member) => ` - ${member}`).join("\n")}
|
|
|
287
3830
|
);
|
|
288
3831
|
return 1;
|
|
289
3832
|
}
|
|
290
|
-
const
|
|
291
|
-
const reportSha256 = createHash("sha256").update(reportArtifact.contents).digest("hex");
|
|
3833
|
+
const reportSha256 = createHash3("sha256").update(reportArtifact.contents).digest("hex");
|
|
292
3834
|
const result = verifyWitanAttestationBinding(statement.value, reportResult.data, {
|
|
293
3835
|
reportSha256
|
|
294
3836
|
});
|
|
@@ -304,10 +3846,13 @@ ${result.errors.map((error) => ` - ${error}`).join("\n")}
|
|
|
304
3846
|
process.stdout.write("Cejel: signature and signer identity were not verified.\n");
|
|
305
3847
|
return 0;
|
|
306
3848
|
}
|
|
3849
|
+
function isLlmPackAttestation(value) {
|
|
3850
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && value.predicateType === CEJEL_LLM_ATTESTATION_PREDICATE;
|
|
3851
|
+
}
|
|
307
3852
|
function readJsonArtifact(path, label) {
|
|
308
3853
|
let contents;
|
|
309
3854
|
try {
|
|
310
|
-
contents =
|
|
3855
|
+
contents = readFileSync2(path);
|
|
311
3856
|
} catch (error) {
|
|
312
3857
|
const message = error instanceof Error ? error.message : String(error);
|
|
313
3858
|
throw new Error(`Cejel: could not read ${label} file ${path}: ${message}`);
|
|
@@ -327,6 +3872,7 @@ function parseArgs(args) {
|
|
|
327
3872
|
let showHelp = false;
|
|
328
3873
|
let showVersion = false;
|
|
329
3874
|
const ingestPatterns = [];
|
|
3875
|
+
const packs = [];
|
|
330
3876
|
for (let index = 0; index < args.length; index += 1) {
|
|
331
3877
|
const arg = args[index];
|
|
332
3878
|
if (arg === "--" || arg === void 0) continue;
|
|
@@ -370,6 +3916,16 @@ function parseArgs(args) {
|
|
|
370
3916
|
index += 1;
|
|
371
3917
|
break;
|
|
372
3918
|
}
|
|
3919
|
+
case "pack": {
|
|
3920
|
+
const value = args[index + 1];
|
|
3921
|
+
if (!value) throw new Error("Missing value for --pack");
|
|
3922
|
+
if (value !== "llm") {
|
|
3923
|
+
throw new Error(`Unknown Cejel pack: ${value}. Supported packs: llm`);
|
|
3924
|
+
}
|
|
3925
|
+
if (!packs.includes(value)) packs.push(value);
|
|
3926
|
+
index += 1;
|
|
3927
|
+
break;
|
|
3928
|
+
}
|
|
373
3929
|
}
|
|
374
3930
|
continue;
|
|
375
3931
|
}
|
|
@@ -380,13 +3936,14 @@ function parseArgs(args) {
|
|
|
380
3936
|
}
|
|
381
3937
|
}
|
|
382
3938
|
return {
|
|
383
|
-
repoPath:
|
|
3939
|
+
repoPath: resolve3(repoPath ?? "."),
|
|
384
3940
|
outDir,
|
|
385
3941
|
...minScore != null ? { minScore } : {},
|
|
386
3942
|
quiet,
|
|
387
3943
|
showHelp,
|
|
388
3944
|
showVersion,
|
|
389
|
-
ingestPatterns
|
|
3945
|
+
ingestPatterns,
|
|
3946
|
+
packs
|
|
390
3947
|
};
|
|
391
3948
|
}
|
|
392
3949
|
function isEntryPoint() {
|