oh-my-knowledge 0.51.0 → 0.51.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/eval-core/dependency-checker.js +30 -18
- package/dist/observability/codex-exec-command.d.ts +7 -0
- package/dist/observability/codex-exec-command.js +134 -0
- package/dist/observability/codex-trace-adapter.js +16 -6
- package/dist/observability/trace-attribution.js +11 -107
- package/package.json +1 -1
|
@@ -25,7 +25,14 @@ const SYSTEM_ENV_VARS = new Set([
|
|
|
25
25
|
'TMPDIR', 'EDITOR', 'VISUAL', 'HOSTNAME', 'LOGNAME', 'DISPLAY',
|
|
26
26
|
'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_CACHE_HOME',
|
|
27
27
|
'NODE_ENV', 'NODE_PATH', 'NODE_OPTIONS', 'NPM_CONFIG_PREFIX',
|
|
28
|
+
// Agent Skills resolves this placeholder to the active skill directory.
|
|
29
|
+
// It is not a user-provided environment prerequisite.
|
|
30
|
+
'SKILL_ROOT',
|
|
28
31
|
]);
|
|
32
|
+
// A shell probe that explicitly tolerates failure is discovering the target
|
|
33
|
+
// project shape. Candidate paths on that line are not hard dependencies of the
|
|
34
|
+
// skill itself.
|
|
35
|
+
const OPTIONAL_PROBE_LINE_REGEX = /(?:2>\s*\/dev\/null|\|\|\s*(?:true|:)(?:\s|$))/;
|
|
29
36
|
function extractFromText(text) {
|
|
30
37
|
const tools = new Set();
|
|
31
38
|
const files = new Set();
|
|
@@ -42,25 +49,30 @@ function extractFromText(text) {
|
|
|
42
49
|
tools.add(cmd);
|
|
43
50
|
}
|
|
44
51
|
}
|
|
45
|
-
// File paths
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
if (path.startsWith('http') || path.startsWith('node_modules') || /^\d/.test(path))
|
|
52
|
+
// File paths. Keep line context so optional discovery commands do not turn
|
|
53
|
+
// every candidate project entry point into a fatal preflight requirement.
|
|
54
|
+
for (const line of text.split(/\r?\n/)) {
|
|
55
|
+
if (OPTIONAL_PROBE_LINE_REGEX.test(line))
|
|
50
56
|
continue;
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
57
|
+
for (const match of line.matchAll(FILE_PATH_REGEX)) {
|
|
58
|
+
const path = match[1];
|
|
59
|
+
// Skip paths that look like URLs, package names, or version strings
|
|
60
|
+
if (path.startsWith('http') || path.startsWith('node_modules') || /^\d/.test(path))
|
|
61
|
+
continue;
|
|
62
|
+
// Skip very short paths that are likely not real files
|
|
63
|
+
if (path.length < 5)
|
|
64
|
+
continue;
|
|
65
|
+
// Skip extension-mention patterns(`.d.ts` / `.tsx` 这种以点开头的"扩展名讨论"
|
|
66
|
+
// 不是真路径,SKILL.md 里"查看 .d.ts 文件"会被误识别)
|
|
67
|
+
if (path.startsWith('.'))
|
|
68
|
+
continue;
|
|
69
|
+
// Skip bare filenames without a directory segment(`index.ts` / `package.json`
|
|
70
|
+
// 这种通用文件名几乎都是示例性提及,真依赖会带路径段。要声明 bare 文件
|
|
71
|
+
// 走显式 requires)
|
|
72
|
+
if (!path.includes('/'))
|
|
73
|
+
continue;
|
|
74
|
+
files.add(path);
|
|
75
|
+
}
|
|
64
76
|
}
|
|
65
77
|
// Environment variables
|
|
66
78
|
for (const match of text.matchAll(ENV_VAR_REGEX)) {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** Static extraction for Codex Desktop's JavaScript exec bridge. */
|
|
2
|
+
/**
|
|
3
|
+
* Extract only literal `cmd` values from real `tools.exec_command(...)` calls.
|
|
4
|
+
* The bridge source is never evaluated, and examples inside strings/comments
|
|
5
|
+
* remain ignored.
|
|
6
|
+
*/
|
|
7
|
+
export declare function extractCodexExecCommands(source: string): string[];
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/** Static extraction for Codex Desktop's JavaScript exec bridge. */
|
|
2
|
+
function skipJsString(source, start) {
|
|
3
|
+
const quote = source[start];
|
|
4
|
+
let index = start + 1;
|
|
5
|
+
while (index < source.length) {
|
|
6
|
+
if (source[index] === '\\') {
|
|
7
|
+
index += 2;
|
|
8
|
+
continue;
|
|
9
|
+
}
|
|
10
|
+
if (source[index] === quote)
|
|
11
|
+
return index + 1;
|
|
12
|
+
index += 1;
|
|
13
|
+
}
|
|
14
|
+
return source.length;
|
|
15
|
+
}
|
|
16
|
+
function skipJsTrivia(source, start) {
|
|
17
|
+
let index = start;
|
|
18
|
+
while (index < source.length) {
|
|
19
|
+
if (/\s/.test(source[index])) {
|
|
20
|
+
index += 1;
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
if (source.startsWith('//', index)) {
|
|
24
|
+
const newline = source.indexOf('\n', index + 2);
|
|
25
|
+
return newline < 0 ? source.length : skipJsTrivia(source, newline + 1);
|
|
26
|
+
}
|
|
27
|
+
if (source.startsWith('/*', index)) {
|
|
28
|
+
const end = source.indexOf('*/', index + 2);
|
|
29
|
+
return end < 0 ? source.length : skipJsTrivia(source, end + 2);
|
|
30
|
+
}
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
return index;
|
|
34
|
+
}
|
|
35
|
+
function commandPropertyEnd(source, index) {
|
|
36
|
+
const bareKey = source.startsWith('cmd', index)
|
|
37
|
+
&& !/[\w$]/.test(source[index - 1] ?? '')
|
|
38
|
+
&& !/[\w$]/.test(source[index + 3] ?? '');
|
|
39
|
+
if (bareKey)
|
|
40
|
+
return index + 3;
|
|
41
|
+
const quote = source[index];
|
|
42
|
+
if (quote !== '"' && quote !== "'")
|
|
43
|
+
return null;
|
|
44
|
+
const end = skipJsString(source, index);
|
|
45
|
+
return source.slice(index + 1, end - 1) === 'cmd' ? end : null;
|
|
46
|
+
}
|
|
47
|
+
function commandLiteralValue(source, start, end) {
|
|
48
|
+
const literal = source.slice(start, end);
|
|
49
|
+
if (source[start] === '"') {
|
|
50
|
+
try {
|
|
51
|
+
const parsed = JSON.parse(literal);
|
|
52
|
+
if (typeof parsed === 'string')
|
|
53
|
+
return parsed;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
// Fall through to the lossless source slice for non-JSON JS escapes.
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return source.slice(start + 1, Math.max(start + 1, end - 1));
|
|
60
|
+
}
|
|
61
|
+
function extractExecCommandLiteral(source, callStart) {
|
|
62
|
+
let index = skipJsTrivia(source, callStart + 'tools.exec_command'.length);
|
|
63
|
+
if (source[index] !== '(')
|
|
64
|
+
return null;
|
|
65
|
+
index = skipJsTrivia(source, index + 1);
|
|
66
|
+
if (source[index] !== '{')
|
|
67
|
+
return null;
|
|
68
|
+
let depth = 1;
|
|
69
|
+
index += 1;
|
|
70
|
+
while (index < source.length && depth > 0) {
|
|
71
|
+
index = skipJsTrivia(source, index);
|
|
72
|
+
const char = source[index];
|
|
73
|
+
if (depth === 1) {
|
|
74
|
+
const keyEnd = commandPropertyEnd(source, index);
|
|
75
|
+
if (keyEnd !== null) {
|
|
76
|
+
let valueStart = skipJsTrivia(source, keyEnd);
|
|
77
|
+
if (source[valueStart] !== ':') {
|
|
78
|
+
index = keyEnd;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
valueStart = skipJsTrivia(source, valueStart + 1);
|
|
82
|
+
const quote = source[valueStart];
|
|
83
|
+
if (quote !== '"' && quote !== "'" && quote !== '`')
|
|
84
|
+
return null;
|
|
85
|
+
const end = skipJsString(source, valueStart);
|
|
86
|
+
return commandLiteralValue(source, valueStart, end);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (char === '"' || char === "'" || char === '`') {
|
|
90
|
+
index = skipJsString(source, index);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (char === '{') {
|
|
94
|
+
depth += 1;
|
|
95
|
+
index += 1;
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (char === '}') {
|
|
99
|
+
depth -= 1;
|
|
100
|
+
index += 1;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
index += 1;
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Extract only literal `cmd` values from real `tools.exec_command(...)` calls.
|
|
109
|
+
* The bridge source is never evaluated, and examples inside strings/comments
|
|
110
|
+
* remain ignored.
|
|
111
|
+
*/
|
|
112
|
+
export function extractCodexExecCommands(source) {
|
|
113
|
+
const commands = [];
|
|
114
|
+
let index = 0;
|
|
115
|
+
while (index < source.length) {
|
|
116
|
+
index = skipJsTrivia(source, index);
|
|
117
|
+
const char = source[index];
|
|
118
|
+
if (char === '"' || char === "'" || char === '`') {
|
|
119
|
+
index = skipJsString(source, index);
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
if (source.startsWith('tools.exec_command', index)
|
|
123
|
+
&& !/[\w$.]/.test(source[index - 1] ?? '')
|
|
124
|
+
&& !/[\w$]/.test(source[index + 'tools.exec_command'.length] ?? '')) {
|
|
125
|
+
const command = extractExecCommandLiteral(source, index);
|
|
126
|
+
if (command)
|
|
127
|
+
commands.push(command);
|
|
128
|
+
index += 'tools.exec_command'.length;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
index += 1;
|
|
132
|
+
}
|
|
133
|
+
return commands;
|
|
134
|
+
}
|
|
@@ -4,6 +4,7 @@ import { isToolResultFailureText } from './text-signals.js';
|
|
|
4
4
|
import { correlateTraceToolEvents, createTraceId, normalizeTraceTimestamp, traceTimestampBounds, } from './trace-ir.js';
|
|
5
5
|
import { nonNegativeMetric, optionalTokenCount, splitInclusiveInputTokens, tokenCount, } from '../shared/token-usage.js';
|
|
6
6
|
import { normalizeToolIdentity } from '../shared/tool-identity.js';
|
|
7
|
+
import { extractCodexExecCommands } from './codex-exec-command.js';
|
|
7
8
|
const CODEX_RESPONSE_ITEM_TYPES = new Set([
|
|
8
9
|
'message',
|
|
9
10
|
'reasoning',
|
|
@@ -242,8 +243,10 @@ function convertCodexRecords(rawRecords, runId) {
|
|
|
242
243
|
const explicitStatus = runtimeOutcome.present
|
|
243
244
|
? runtimeOutcome.status
|
|
244
245
|
: statusFromCodex(payloadStatus);
|
|
245
|
-
const
|
|
246
|
-
const
|
|
246
|
+
const bridgeFailure = codexToolOutputFailed(output);
|
|
247
|
+
const bridgeSuccess = !bridgeFailure && codexToolOutputSucceeded(output);
|
|
248
|
+
const inferredFailure = bridgeFailure || (!bridgeSuccess && isToolResultFailureText(output));
|
|
249
|
+
const inferredSuccess = bridgeSuccess;
|
|
247
250
|
const completedExternalCall = externalEnds.byOccurrence.has(mcpCallOccurrenceKey(callId, externalOccurrence));
|
|
248
251
|
const inferredStatus = inferredFailure
|
|
249
252
|
? 'failure'
|
|
@@ -609,6 +612,10 @@ function runtimeToolEndOutcome(end) {
|
|
|
609
612
|
}
|
|
610
613
|
function normalizeCodexTool(sourceName, rawInput, mcpEnd, sourceNamespace) {
|
|
611
614
|
const input = parseToolInput(rawInput);
|
|
615
|
+
const sourceInput = stringValue(input.input);
|
|
616
|
+
const execCommands = sourceName.toLowerCase() === 'exec' && sourceInput
|
|
617
|
+
? extractCodexExecCommands(sourceInput)
|
|
618
|
+
: [];
|
|
612
619
|
// Codex desktop's orchestration wrapper names its JavaScript command bridge
|
|
613
620
|
// `exec`. This mapping is source-specific: a generic tool named `exec` must not
|
|
614
621
|
// become shell execution outside the Codex adapter.
|
|
@@ -629,10 +636,13 @@ function normalizeCodexTool(sourceName, rawInput, mcpEnd, sourceNamespace) {
|
|
|
629
636
|
tool,
|
|
630
637
|
input: {
|
|
631
638
|
...input,
|
|
632
|
-
command:
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
639
|
+
command: execCommands.length > 0
|
|
640
|
+
? execCommands.join('\n')
|
|
641
|
+
: stringValue(input.command)
|
|
642
|
+
?? stringValue(input.cmd)
|
|
643
|
+
?? sourceInput
|
|
644
|
+
?? '',
|
|
645
|
+
...(execCommands.length > 0 ? { commands: execCommands } : {}),
|
|
636
646
|
},
|
|
637
647
|
};
|
|
638
648
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
/** Skill attribution rules for trace records. */
|
|
2
|
+
import { extractCodexExecCommands } from './codex-exec-command.js';
|
|
2
3
|
export function extractMarkdownLogSkill(text) {
|
|
3
4
|
const patterns = [
|
|
4
5
|
/\b(?:prefer|use|call|invoke)\s+`?([a-zA-Z0-9][\w.-]*)`?\s+skill\b/i,
|
|
@@ -192,108 +193,6 @@ function extractInstalledSkillReadRef(text) {
|
|
|
192
193
|
}
|
|
193
194
|
return null;
|
|
194
195
|
}
|
|
195
|
-
function skipJsString(source, start) {
|
|
196
|
-
const quote = source[start];
|
|
197
|
-
let index = start + 1;
|
|
198
|
-
while (index < source.length) {
|
|
199
|
-
if (source[index] === '\\') {
|
|
200
|
-
index += 2;
|
|
201
|
-
continue;
|
|
202
|
-
}
|
|
203
|
-
if (source[index] === quote)
|
|
204
|
-
return index + 1;
|
|
205
|
-
index += 1;
|
|
206
|
-
}
|
|
207
|
-
return source.length;
|
|
208
|
-
}
|
|
209
|
-
function skipJsTrivia(source, start) {
|
|
210
|
-
let index = start;
|
|
211
|
-
while (index < source.length) {
|
|
212
|
-
if (/\s/.test(source[index])) {
|
|
213
|
-
index += 1;
|
|
214
|
-
continue;
|
|
215
|
-
}
|
|
216
|
-
if (source.startsWith('//', index)) {
|
|
217
|
-
const newline = source.indexOf('\n', index + 2);
|
|
218
|
-
return newline < 0 ? source.length : skipJsTrivia(source, newline + 1);
|
|
219
|
-
}
|
|
220
|
-
if (source.startsWith('/*', index)) {
|
|
221
|
-
const end = source.indexOf('*/', index + 2);
|
|
222
|
-
return end < 0 ? source.length : skipJsTrivia(source, end + 2);
|
|
223
|
-
}
|
|
224
|
-
break;
|
|
225
|
-
}
|
|
226
|
-
return index;
|
|
227
|
-
}
|
|
228
|
-
function extractExecCommandLiteral(source, callStart) {
|
|
229
|
-
let index = skipJsTrivia(source, callStart + 'tools.exec_command'.length);
|
|
230
|
-
if (source[index] !== '(')
|
|
231
|
-
return null;
|
|
232
|
-
index = skipJsTrivia(source, index + 1);
|
|
233
|
-
if (source[index] !== '{')
|
|
234
|
-
return null;
|
|
235
|
-
let depth = 1;
|
|
236
|
-
index += 1;
|
|
237
|
-
while (index < source.length && depth > 0) {
|
|
238
|
-
index = skipJsTrivia(source, index);
|
|
239
|
-
const char = source[index];
|
|
240
|
-
if (char === '"' || char === "'" || char === '`') {
|
|
241
|
-
index = skipJsString(source, index);
|
|
242
|
-
continue;
|
|
243
|
-
}
|
|
244
|
-
if (char === '{') {
|
|
245
|
-
depth += 1;
|
|
246
|
-
index += 1;
|
|
247
|
-
continue;
|
|
248
|
-
}
|
|
249
|
-
if (char === '}') {
|
|
250
|
-
depth -= 1;
|
|
251
|
-
index += 1;
|
|
252
|
-
continue;
|
|
253
|
-
}
|
|
254
|
-
if (depth === 1
|
|
255
|
-
&& source.startsWith('cmd', index)
|
|
256
|
-
&& !/[\w$]/.test(source[index - 1] ?? '')
|
|
257
|
-
&& !/[\w$]/.test(source[index + 3] ?? '')) {
|
|
258
|
-
let valueStart = skipJsTrivia(source, index + 3);
|
|
259
|
-
if (source[valueStart] !== ':') {
|
|
260
|
-
index += 3;
|
|
261
|
-
continue;
|
|
262
|
-
}
|
|
263
|
-
valueStart = skipJsTrivia(source, valueStart + 1);
|
|
264
|
-
const quote = source[valueStart];
|
|
265
|
-
if (quote !== '"' && quote !== "'" && quote !== '`')
|
|
266
|
-
return null;
|
|
267
|
-
const end = skipJsString(source, valueStart);
|
|
268
|
-
return source.slice(valueStart + 1, Math.max(valueStart + 1, end - 1));
|
|
269
|
-
}
|
|
270
|
-
index += 1;
|
|
271
|
-
}
|
|
272
|
-
return null;
|
|
273
|
-
}
|
|
274
|
-
function extractExecCommandLiterals(source) {
|
|
275
|
-
const commands = [];
|
|
276
|
-
let index = 0;
|
|
277
|
-
while (index < source.length) {
|
|
278
|
-
index = skipJsTrivia(source, index);
|
|
279
|
-
const char = source[index];
|
|
280
|
-
if (char === '"' || char === "'" || char === '`') {
|
|
281
|
-
index = skipJsString(source, index);
|
|
282
|
-
continue;
|
|
283
|
-
}
|
|
284
|
-
if (source.startsWith('tools.exec_command', index)
|
|
285
|
-
&& !/[\w$.]/.test(source[index - 1] ?? '')
|
|
286
|
-
&& !/[\w$]/.test(source[index + 'tools.exec_command'.length] ?? '')) {
|
|
287
|
-
const command = extractExecCommandLiteral(source, index);
|
|
288
|
-
if (command)
|
|
289
|
-
commands.push(command);
|
|
290
|
-
index += 'tools.exec_command'.length;
|
|
291
|
-
continue;
|
|
292
|
-
}
|
|
293
|
-
index += 1;
|
|
294
|
-
}
|
|
295
|
-
return commands;
|
|
296
|
-
}
|
|
297
196
|
function splitShellCommandSegments(command) {
|
|
298
197
|
const segments = [];
|
|
299
198
|
let start = 0;
|
|
@@ -375,7 +274,7 @@ export function extractSkillReadFileRef(record) {
|
|
|
375
274
|
? []
|
|
376
275
|
: part.name === 'Bash'
|
|
377
276
|
? [rawCommand]
|
|
378
|
-
:
|
|
277
|
+
: extractCodexExecCommands(rawCommand);
|
|
379
278
|
for (const command of commands) {
|
|
380
279
|
const skillRef = extractShellSkillReadRef(command);
|
|
381
280
|
if (skillRef)
|
|
@@ -408,7 +307,7 @@ export function extractSkillScriptCommandRef(record) {
|
|
|
408
307
|
: part.input?.code ?? part.input?.command;
|
|
409
308
|
if (typeof rawCommand === 'string') {
|
|
410
309
|
texts.push(...(part.name?.toLowerCase() === 'exec'
|
|
411
|
-
?
|
|
310
|
+
? extractCodexExecCommands(rawCommand)
|
|
412
311
|
: [rawCommand]));
|
|
413
312
|
}
|
|
414
313
|
}
|
|
@@ -465,9 +364,14 @@ export function extractSkillReadFileRefFromEvent(event) {
|
|
|
465
364
|
|| sourceName === 'js'
|
|
466
365
|
|| event.tool.name === 'node_repl.js'
|
|
467
366
|
|| event.tool.name.toLowerCase() === 'js';
|
|
468
|
-
const
|
|
469
|
-
?
|
|
470
|
-
: [
|
|
367
|
+
const normalizedCommands = Array.isArray(event.input.commands)
|
|
368
|
+
? event.input.commands.filter((value) => typeof value === 'string')
|
|
369
|
+
: [];
|
|
370
|
+
const commands = normalizedCommands.length > 0
|
|
371
|
+
? normalizedCommands
|
|
372
|
+
: isOrchestrationWrapper
|
|
373
|
+
? extractCodexExecCommands(rawCommand)
|
|
374
|
+
: [rawCommand];
|
|
471
375
|
for (const command of commands) {
|
|
472
376
|
const skillRef = extractShellSkillReadRef(command);
|
|
473
377
|
if (skillRef)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "oh-my-knowledge",
|
|
3
|
-
"version": "0.51.
|
|
3
|
+
"version": "0.51.1",
|
|
4
4
|
"packageManager": "yarn@4.16.0",
|
|
5
5
|
"description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
|
|
6
6
|
"type": "module",
|