@cat-factory/kernel 0.292.0 → 0.292.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -8,10 +8,22 @@
|
|
|
8
8
|
* `{` whose balanced span doesn't parse (e.g. preamble prose like `I weighed [the auth
|
|
9
9
|
* flow] and concluded: {…}` — the `[the auth flow]` is not JSON) is skipped and the next
|
|
10
10
|
* bracket is tried, so the real object after the prose is still found rather than the
|
|
11
|
-
* whole extraction collapsing to `null`.
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
11
|
+
* whole extraction collapsing to `null`. Fenced code blocks (```` ```json … ``` ````) are
|
|
12
|
+
* searched first, in the order written: a model that fenced its reasoning before emitting the
|
|
13
|
+
* real object still gets read, and so does one whose payload sits in a later fence.
|
|
14
|
+
*
|
|
15
|
+
* A span that is valid JSON except for RAW control characters inside a string literal is
|
|
16
|
+
* REPAIRED rather than refused (see {@link escapeControlCharsInStrings}) — but only in a second
|
|
17
|
+
* pass, after every candidate in every source has been tried AS WRITTEN.
|
|
18
|
+
*
|
|
19
|
+
* That two-pass order is the whole safety property, and repairing inside the scan loop broke it:
|
|
20
|
+
* a repair makes an earlier span parse that would otherwise have been skipped, so an example
|
|
21
|
+
* shape or a prose aside ("Notes on the field: {"name": "the field<newline>I mean"}") starts
|
|
22
|
+
* SHADOWING the real verdict written after it. A verdict is the one thing that must not be
|
|
23
|
+
* guessed, and a model told to lay a field out over several lines writes exactly the
|
|
24
|
+
* malformation that used to make such a span skippable. So a strictly-parseable value anywhere
|
|
25
|
+
* in the reply outranks a repaired one, and the repair only ever recovers a reply that had no
|
|
26
|
+
* readable JSON at all.
|
|
15
27
|
*/
|
|
16
28
|
export declare function extractJson(text: string): unknown;
|
|
17
29
|
//# sourceMappingURL=llm-output.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-output.d.ts","sourceRoot":"","sources":["../../src/domain/llm-output.ts"],"names":[],"mappings":"AAIA
|
|
1
|
+
{"version":3,"file":"llm-output.d.ts","sourceRoot":"","sources":["../../src/domain/llm-output.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAYjD"}
|
|
@@ -11,32 +11,62 @@
|
|
|
11
11
|
* `{` whose balanced span doesn't parse (e.g. preamble prose like `I weighed [the auth
|
|
12
12
|
* flow] and concluded: {…}` — the `[the auth flow]` is not JSON) is skipped and the next
|
|
13
13
|
* bracket is tried, so the real object after the prose is still found rather than the
|
|
14
|
-
* whole extraction collapsing to `null`.
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
14
|
+
* whole extraction collapsing to `null`. Fenced code blocks (```` ```json … ``` ````) are
|
|
15
|
+
* searched first, in the order written: a model that fenced its reasoning before emitting the
|
|
16
|
+
* real object still gets read, and so does one whose payload sits in a later fence.
|
|
17
|
+
*
|
|
18
|
+
* A span that is valid JSON except for RAW control characters inside a string literal is
|
|
19
|
+
* REPAIRED rather than refused (see {@link escapeControlCharsInStrings}) — but only in a second
|
|
20
|
+
* pass, after every candidate in every source has been tried AS WRITTEN.
|
|
21
|
+
*
|
|
22
|
+
* That two-pass order is the whole safety property, and repairing inside the scan loop broke it:
|
|
23
|
+
* a repair makes an earlier span parse that would otherwise have been skipped, so an example
|
|
24
|
+
* shape or a prose aside ("Notes on the field: {"name": "the field<newline>I mean"}") starts
|
|
25
|
+
* SHADOWING the real verdict written after it. A verdict is the one thing that must not be
|
|
26
|
+
* guessed, and a model told to lay a field out over several lines writes exactly the
|
|
27
|
+
* malformation that used to make such a span skippable. So a strictly-parseable value anywhere
|
|
28
|
+
* in the reply outranks a repaired one, and the repair only ever recovers a reply that had no
|
|
29
|
+
* readable JSON at all.
|
|
18
30
|
*/
|
|
19
31
|
export function extractJson(text) {
|
|
20
32
|
const trimmed = text.trim();
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
33
|
+
// Fence bodies first (a deliberate delimiter of the payload), then the whole reply. The same
|
|
34
|
+
// ordered list is walked by both passes, so the fence preference holds within each.
|
|
35
|
+
const sources = [...fencedBodies(trimmed), trimmed];
|
|
36
|
+
for (const mode of PARSE_MODES) {
|
|
37
|
+
for (const source of sources) {
|
|
38
|
+
const value = extractJsonValue(source, mode);
|
|
39
|
+
if (value !== null)
|
|
40
|
+
return value;
|
|
41
|
+
}
|
|
26
42
|
}
|
|
27
|
-
return
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
/** Strict first, everywhere; repair only once nothing parsed as written. See {@link extractJson}. */
|
|
46
|
+
const PARSE_MODES = ['strict', 'repair'];
|
|
47
|
+
/**
|
|
48
|
+
* The body of every fenced block in the reply, in the order written.
|
|
49
|
+
*
|
|
50
|
+
* All of them, not just the first: the fence a payload sits in is not always the first one (a
|
|
51
|
+
* model fences its reasoning, or shows an example, before the object), and in the repair pass the
|
|
52
|
+
* ordering decides which candidate wins. A nested fence (a code block inside a string value) makes
|
|
53
|
+
* the split imperfect by construction — the closing ``` of the inner block ends the outer match —
|
|
54
|
+
* which is why the whole reply stays in the list as the final source.
|
|
55
|
+
*/
|
|
56
|
+
function fencedBodies(text) {
|
|
57
|
+
return [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/gi)].map((m) => m[1] ?? '');
|
|
28
58
|
}
|
|
29
59
|
/**
|
|
30
60
|
* Scan `candidate` for the first balanced JSON object/array that parses, skipping any
|
|
31
61
|
* earlier bracket whose balanced span is not valid JSON (e.g. a bracket inside prose).
|
|
32
62
|
*/
|
|
33
|
-
function extractJsonValue(candidate) {
|
|
63
|
+
function extractJsonValue(candidate, mode) {
|
|
34
64
|
for (let from = 0; from < candidate.length;) {
|
|
35
65
|
const rel = candidate.slice(from).search(/[[{]/);
|
|
36
66
|
if (rel === -1)
|
|
37
67
|
return null;
|
|
38
68
|
const start = from + rel;
|
|
39
|
-
const value = parseBalancedFrom(candidate, start);
|
|
69
|
+
const value = parseBalancedFrom(candidate, start, mode);
|
|
40
70
|
if (value !== null)
|
|
41
71
|
return value;
|
|
42
72
|
from = start + 1;
|
|
@@ -44,7 +74,7 @@ function extractJsonValue(candidate) {
|
|
|
44
74
|
return null;
|
|
45
75
|
}
|
|
46
76
|
/** Parse the balanced JSON value that starts at `candidate[start]`, or null if it doesn't parse. */
|
|
47
|
-
function parseBalancedFrom(candidate, start) {
|
|
77
|
+
function parseBalancedFrom(candidate, start, mode) {
|
|
48
78
|
const open = candidate[start];
|
|
49
79
|
const close = open === '{' ? '}' : ']';
|
|
50
80
|
let depth = 0;
|
|
@@ -67,16 +97,92 @@ function parseBalancedFrom(candidate, start) {
|
|
|
67
97
|
depth++;
|
|
68
98
|
else if (ch === close) {
|
|
69
99
|
depth--;
|
|
70
|
-
if (depth === 0)
|
|
71
|
-
|
|
72
|
-
return JSON.parse(candidate.slice(start, i + 1));
|
|
73
|
-
}
|
|
74
|
-
catch {
|
|
75
|
-
return null;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
100
|
+
if (depth === 0)
|
|
101
|
+
return parseSpan(candidate.slice(start, i + 1), mode);
|
|
78
102
|
}
|
|
79
103
|
}
|
|
80
104
|
return null;
|
|
81
105
|
}
|
|
106
|
+
/**
|
|
107
|
+
* Parse one balanced span: as written in the `strict` pass, with its string literals re-escaped
|
|
108
|
+
* in the `repair` one (see {@link escapeControlCharsInStrings}). Null when it doesn't parse, which
|
|
109
|
+
* sends the caller on to the next bracket rather than claiming a value nobody wrote.
|
|
110
|
+
*/
|
|
111
|
+
function parseSpan(span, mode) {
|
|
112
|
+
// A span carrying no raw control character cannot be repaired into anything a strict parse did
|
|
113
|
+
// not already refuse, so the rewrite is skipped rather than run over every bracket in a long
|
|
114
|
+
// reply (a tester quoting a code block puts hundreds of them in prose).
|
|
115
|
+
if (mode === 'repair' && !hasRawControlChar(span))
|
|
116
|
+
return null;
|
|
117
|
+
try {
|
|
118
|
+
return JSON.parse(mode === 'repair' ? escapeControlCharsInStrings(span) : span);
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
/** Whether `span` holds any raw control character: the cheap gate on attempting a repair. */
|
|
125
|
+
function hasRawControlChar(span) {
|
|
126
|
+
for (let i = 0; i < span.length; i++) {
|
|
127
|
+
if (span.charCodeAt(i) < 0x20)
|
|
128
|
+
return true;
|
|
129
|
+
}
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
/** The control characters JSON gives a short escape; the rest go to `\uXXXX`. */
|
|
133
|
+
const CONTROL_ESCAPES = {
|
|
134
|
+
'\n': '\\n',
|
|
135
|
+
'\r': '\\r',
|
|
136
|
+
'\t': '\\t',
|
|
137
|
+
'\b': '\\b',
|
|
138
|
+
'\f': '\\f',
|
|
139
|
+
};
|
|
140
|
+
/**
|
|
141
|
+
* Re-escape raw control characters that sit INSIDE a JSON string literal.
|
|
142
|
+
*
|
|
143
|
+
* `JSON.parse` rejects a literal newline in a string, and that is the malformation a model
|
|
144
|
+
* reliably produces once a field is asked for as several lines (a review verdict laid out as
|
|
145
|
+
* blocks, a rationale with a bullet list): it writes the layout it was told to write and drops
|
|
146
|
+
* the `\n` escape. The reply is otherwise the verdict, so refusing it loses a whole review to a
|
|
147
|
+
* quoting slip. Only characters inside a string are rewritten, so the structural whitespace
|
|
148
|
+
* between tokens keeps its meaning and a genuinely broken reply still fails to parse.
|
|
149
|
+
*
|
|
150
|
+
* Copied in SLICES around each rewritten character rather than one character at a time: this runs
|
|
151
|
+
* over a whole span per failing candidate bracket, and a reply that quotes a code block has many.
|
|
152
|
+
* A span needing no rewrite therefore costs one pass and no allocation.
|
|
153
|
+
*/
|
|
154
|
+
function escapeControlCharsInStrings(json) {
|
|
155
|
+
let out = '';
|
|
156
|
+
let copiedTo = 0;
|
|
157
|
+
let inString = false;
|
|
158
|
+
let escaped = false;
|
|
159
|
+
for (let i = 0; i < json.length; i++) {
|
|
160
|
+
const ch = json[i];
|
|
161
|
+
if (!inString) {
|
|
162
|
+
if (ch === '"')
|
|
163
|
+
inString = true;
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
if (escaped) {
|
|
167
|
+
escaped = false;
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
if (ch === '\\') {
|
|
171
|
+
escaped = true;
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
if (ch === '"') {
|
|
175
|
+
inString = false;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
// Only a control character needs rewriting; everything else (a surrogate pair included) is
|
|
179
|
+
// carried by the slices, untouched.
|
|
180
|
+
if (json.charCodeAt(i) >= 0x20)
|
|
181
|
+
continue;
|
|
182
|
+
const escape = CONTROL_ESCAPES[ch] ?? `\\u${json.charCodeAt(i).toString(16).padStart(4, '0')}`;
|
|
183
|
+
out += json.slice(copiedTo, i) + escape;
|
|
184
|
+
copiedTo = i + 1;
|
|
185
|
+
}
|
|
186
|
+
return out + json.slice(copiedTo);
|
|
187
|
+
}
|
|
82
188
|
//# sourceMappingURL=llm-output.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-output.js","sourceRoot":"","sources":["../../src/domain/llm-output.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,sFAAsF;AACtF,kFAAkF;AAElF
|
|
1
|
+
{"version":3,"file":"llm-output.js","sourceRoot":"","sources":["../../src/domain/llm-output.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,sFAAsF;AACtF,kFAAkF;AAElF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAC3B,6FAA6F;IAC7F,oFAAoF;IACpF,MAAM,OAAO,GAAG,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAA;IACnD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAC5C,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;QAClC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,qGAAqG;AACrG,MAAM,WAAW,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAU,CAAA;AAGjD;;;;;;;;GAQG;AACH,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,gCAAgC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;AACpF,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,SAAiB,EAAE,IAAe;IAC1D,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC;QAC5C,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAChD,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAA;QAC3B,MAAM,KAAK,GAAG,IAAI,GAAG,GAAG,CAAA;QACxB,MAAM,KAAK,GAAG,iBAAiB,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;QACvD,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAA;QAChC,IAAI,GAAG,KAAK,GAAG,CAAC,CAAA;IAClB,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,oGAAoG;AACpG,SAAS,iBAAiB,CAAC,SAAiB,EAAE,KAAa,EAAE,IAAe;IAC1E,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;IAC7B,MAAM,KAAK,GAAG,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAA;IACtC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,QAAQ,GAAG,KAAK,CAAA;IACpB,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;QACvB,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,OAAO;gBAAE,OAAO,GAAG,KAAK,CAAA;iBACvB,IAAI,EAAE,KAAK,IAAI;gBAAE,OAAO,GAAG,IAAI,CAAA;iBAC/B,IAAI,EAAE,KAAK,GAAG;gBAAE,QAAQ,GAAG,KAAK,CAAA;YACrC,SAAQ;QACV,CAAC;QACD,IAAI,EAAE,KAAK,GAAG;YAAE,QAAQ,GAAG,IAAI,CAAA;aAC1B,IAAI,EAAE,KAAK,IAAI;YAAE,KAAK,EAAE,CAAA;aACxB,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;YACtB,KAAK,EAAE,CAAA;YACP,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QACxE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAY,EAAE,IAAe;IAC9C,+FAA+F;IAC/F,6FAA6F;IAC7F,wEAAwE;IACxE,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAC9D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACjF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,6FAA6F;AAC7F,SAAS,iBAAiB,CAAC,IAAY;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI;YAAE,OAAO,IAAI,CAAA;IAC5C,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,iFAAiF;AACjF,MAAM,eAAe,GAA2B;IAC9C,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;CACZ,CAAA;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,2BAA2B,CAAC,IAAY;IAC/C,IAAI,GAAG,GAAG,EAAE,CAAA;IACZ,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,QAAQ,GAAG,KAAK,CAAA;IACpB,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAE,CAAA;QACnB,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,EAAE,KAAK,GAAG;gBAAE,QAAQ,GAAG,IAAI,CAAA;YAC/B,SAAQ;QACV,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,GAAG,KAAK,CAAA;YACf,SAAQ;QACV,CAAC;QACD,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,OAAO,GAAG,IAAI,CAAA;YACd,SAAQ;QACV,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,QAAQ,GAAG,KAAK,CAAA;YAChB,SAAQ;QACV,CAAC;QACD,2FAA2F;QAC3F,oCAAoC;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI;YAAE,SAAQ;QACxC,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,CAAC,IAAI,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAA;QAC9F,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,MAAM,CAAA;QACvC,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAA;IAClB,CAAC;IACD,OAAO,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;AACnC,CAAC"}
|