@memberjunction/global 6.1.0-edge.3 → 6.1.0-edge.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ClassFactory.d.ts +13 -0
- package/dist/ClassFactory.d.ts.map +1 -1
- package/dist/ClassFactory.js +70 -7
- package/dist/ClassFactory.js.map +1 -1
- package/dist/ClassUtils.d.ts +15 -0
- package/dist/ClassUtils.d.ts.map +1 -1
- package/dist/ClassUtils.js +26 -0
- package/dist/ClassUtils.js.map +1 -1
- package/dist/__tests__/ClassFactory.collision.test.d.ts +2 -0
- package/dist/__tests__/ClassFactory.collision.test.d.ts.map +1 -0
- package/dist/__tests__/ClassFactory.collision.test.js +136 -0
- package/dist/__tests__/ClassFactory.collision.test.js.map +1 -0
- package/dist/__tests__/ClassFactory.test.js +40 -0
- package/dist/__tests__/ClassFactory.test.js.map +1 -1
- package/dist/__tests__/util.jsonRepair.test.d.ts +2 -0
- package/dist/__tests__/util.jsonRepair.test.d.ts.map +1 -0
- package/dist/__tests__/util.jsonRepair.test.js +112 -0
- package/dist/__tests__/util.jsonRepair.test.js.map +1 -0
- package/dist/__tests__/util.test.js +62 -1
- package/dist/__tests__/util.test.js.map +1 -1
- package/dist/util.d.ts +90 -5
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +260 -27
- package/dist/util.js.map +1 -1
- package/package.json +1 -1
package/dist/util.d.ts
CHANGED
|
@@ -5,11 +5,6 @@
|
|
|
5
5
|
export interface GlobalObjectStore {
|
|
6
6
|
[key: string]: any;
|
|
7
7
|
}
|
|
8
|
-
/**
|
|
9
|
-
* The Global Object Store is a place to store global objects that need to be shared across the application. Depending on the execution environment, this could be the window object in a browser, or the global object in a node environment, or something else in other contexts. The key here is that in some cases static variables are not truly shared
|
|
10
|
-
* because it is possible that a given class might have copies of its code in multiple paths in a deployed application. This approach ensures that no matter how many code copies might exist, there is only one instance of the object in question by using the Global Object Store.
|
|
11
|
-
* @returns
|
|
12
|
-
*/
|
|
13
8
|
export declare function GetGlobalObjectStore(): GlobalObjectStore | null;
|
|
14
9
|
/**
|
|
15
10
|
* This utility function will copy all scalar and array properties from an object to a new object and return the new object.
|
|
@@ -120,6 +115,67 @@ export declare function CleanAndParseJSON<T = any>(inputString: string | null, l
|
|
|
120
115
|
* // Returns: '{\n "extracted": true\n}'
|
|
121
116
|
*/
|
|
122
117
|
export declare function CleanJSON(inputString: string | null): string | null;
|
|
118
|
+
/**
|
|
119
|
+
* Outcome of a {@link RepairJSONEscaping} attempt.
|
|
120
|
+
*/
|
|
121
|
+
export interface JSONEscapingRepairResult {
|
|
122
|
+
/** True only when the input parsed after repair. */
|
|
123
|
+
repaired: boolean;
|
|
124
|
+
/** The repaired JSON text, present only when `repaired` is true. */
|
|
125
|
+
text?: string;
|
|
126
|
+
/** The parsed value, present only when `repaired` is true. */
|
|
127
|
+
value?: any;
|
|
128
|
+
/** Zero-based offsets that were escaped, in the order they were fixed. */
|
|
129
|
+
repairedOffsets: number[];
|
|
130
|
+
/** Why the repair stopped, when `repaired` is false. */
|
|
131
|
+
reason?: string;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Deterministically repairs the single most common way an LLM breaks otherwise-valid JSON:
|
|
135
|
+
* a double quote or raw control character left unescaped inside a string value.
|
|
136
|
+
*
|
|
137
|
+
* ## Why this exists
|
|
138
|
+
*
|
|
139
|
+
* Models embed rich markdown in string fields — mermaid diagrams, HTML mockups, code samples —
|
|
140
|
+
* and reliably escape most of it. A single missed quote inside a 25KB response invalidates the
|
|
141
|
+
* whole document. Nothing else in the repair chain recovers that: JSON5's leniency covers trailing
|
|
142
|
+
* commas, comments and unquoted keys, but an unescaped `"` terminates a string in JSON5 exactly as
|
|
143
|
+
* it does in JSON. The remaining fallback is an LLM round-trip on the full payload, which is slow,
|
|
144
|
+
* costly, and itself unreliable at that size.
|
|
145
|
+
*
|
|
146
|
+
* ## How it works
|
|
147
|
+
*
|
|
148
|
+
* Purely error-driven, one character per pass:
|
|
149
|
+
*
|
|
150
|
+
* 1. `JSON.parse` the text and read the failure offset from the thrown error.
|
|
151
|
+
* 2. Scan backwards from that offset for the character that ended the string early.
|
|
152
|
+
* 3. Escape that one character.
|
|
153
|
+
* 4. Re-parse. Repeat until it parses or a stopping condition trips.
|
|
154
|
+
*
|
|
155
|
+
* Every pass is validated by a real parse, so this never "pattern matches" its way to a wrong
|
|
156
|
+
* answer the way a global regex rewrite would. It converges in one pass per offending character
|
|
157
|
+
* (about two per quoted mermaid label) and gives up rather than guessing when it cannot make
|
|
158
|
+
* progress.
|
|
159
|
+
*
|
|
160
|
+
* ## Safety
|
|
161
|
+
*
|
|
162
|
+
* This can, in principle, produce valid-but-wrong JSON: escaping a quote that legitimately ended a
|
|
163
|
+
* string would merge two pieces of structure. Three properties keep that in check — it only runs
|
|
164
|
+
* after a parse has already failed (so a correct document is never touched), it only ever *adds*
|
|
165
|
+
* escapes and never deletes or reorders content, and it reports every offset it changed so callers
|
|
166
|
+
* can log, audit, or schema-check the result before trusting it. Callers holding an expected shape
|
|
167
|
+
* should validate against it; a wrong guess almost always fails shape validation.
|
|
168
|
+
*
|
|
169
|
+
* @param inputString - Raw model output, expected to be a JSON envelope
|
|
170
|
+
* @param maxRepairs - Maximum characters to escape before giving up
|
|
171
|
+
* @returns The repair outcome; `repaired` is false when the input could not be recovered
|
|
172
|
+
*
|
|
173
|
+
* @example
|
|
174
|
+
* // A mermaid relationship label whose quotes were not escaped
|
|
175
|
+
* RepairJSONEscaping('{"doc":"```mermaid\\nerDiagram\\n A ||--o{ B : "has items"\\n```"}')
|
|
176
|
+
* // => { repaired: true, repairedOffsets: [...], value: { doc: '...' } }
|
|
177
|
+
*/
|
|
178
|
+
export declare function RepairJSONEscaping(inputString: string | null, maxRepairs?: number): JSONEscapingRepairResult;
|
|
123
179
|
/**
|
|
124
180
|
* This function takes in a string that may contain JavaScript code in a markdown code block and returns the JavaScript code without the code block.
|
|
125
181
|
* @param javaScriptCode
|
|
@@ -354,6 +410,35 @@ export declare function ParseJSONRecursive(obj: any, options?: ParseJSONOptions)
|
|
|
354
410
|
* @returns The escaped HTML string.
|
|
355
411
|
*/
|
|
356
412
|
export declare function EscapeHTML(text: string): string;
|
|
413
|
+
/**
|
|
414
|
+
* Safely escapes a string literal for use inside single-quoted SQL statements, clauses, or `ExtraFilter` predicates.
|
|
415
|
+
*
|
|
416
|
+
* Replaces each single quote with doubled single quotes (`''`) per ANSI SQL standard — the escaping
|
|
417
|
+
* mechanism supported by both SQL Server and PostgreSQL — and removes null bytes (`\0`), which cannot
|
|
418
|
+
* appear in any legitimate value and invite parser-level surprises when left in a predicate.
|
|
419
|
+
*
|
|
420
|
+
* **This escapes string literals and nothing else.** Three cases it does NOT cover:
|
|
421
|
+
*
|
|
422
|
+
* - **`LIKE` patterns** — `%`, `_` and `[` remain live wildcards after quote doubling, so a user
|
|
423
|
+
* searching for `%` still matches every row. A LIKE value must additionally escape those
|
|
424
|
+
* metacharacters and pair the clause with `ESCAPE '\'`. See `escapeLikeValue()` in
|
|
425
|
+
* `@memberjunction/core` (`generic/runQuerySQLFilterImplementations.ts`) or
|
|
426
|
+
* `GenericDatabaseProvider.escapeLikeTerm()`.
|
|
427
|
+
* - **Identifier names** (table/column/schema) — those require bracket or double-quote quoting, and
|
|
428
|
+
* are handled by SchemaEngine's `ValidateIdentifier()`.
|
|
429
|
+
* - **Values that must not be missing** — `null`/`undefined` map to `''`, so a predicate built from a
|
|
430
|
+
* missing value silently becomes `Field = ''` rather than throwing. Validate before interpolating
|
|
431
|
+
* when absence is a bug.
|
|
432
|
+
*
|
|
433
|
+
* @param value - The raw string value to escape. If null or undefined, returns empty string.
|
|
434
|
+
* @returns Safely escaped string value (without surrounding quotes).
|
|
435
|
+
*
|
|
436
|
+
* @example
|
|
437
|
+
* ```typescript
|
|
438
|
+
* const filter = `Email = '${EscapeSQLString(userEmail)}'`;
|
|
439
|
+
* ```
|
|
440
|
+
*/
|
|
441
|
+
export declare function EscapeSQLString(value: string | null | undefined): string;
|
|
357
442
|
/**
|
|
358
443
|
* The format a piece of text appears to be authored in, as classified by
|
|
359
444
|
* {@link detectRichTextFormat}.
|
package/dist/util.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAE9B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACtB;
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAE9B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACtB;AASD,wBAAgB,oBAAoB,IAAI,iBAAiB,GAAG,IAAI,CA6B/D;AAGD;;;;;;;;;;;;;;;;;;;GAmBG;AACH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAE9E;AAED,wBAAgB,oBAAoB,CAAC,CAAC,SAAS,MAAM,EACjD,KAAK,EAAE,CAAC,EACR,yBAAyB,GAAE,OAAe,EAC1C,QAAQ,GAAE,MAAW,GACtB,OAAO,CAAC,CAAC,CAAC,CAkHZ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,wBAAgB,iBAAiB,CAAC,CAAC,GAAG,GAAG,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI,EAAE,SAAS,GAAE,OAAe,GAAG,CAAC,GAAG,IAAI,CAS3G;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CA2InE;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACrC,oDAAoD;IACpD,QAAQ,EAAE,OAAO,CAAC;IAClB,oEAAoE;IACpE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8DAA8D;IAE9D,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,0EAA0E;IAC1E,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAKD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACH,wBAAgB,kBAAkB,CAC9B,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,UAAU,GAAE,MAAkC,GAC/C,wBAAwB,CA0C1B;AAiGD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM,CAe9D;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,wBAAgB,aAAa,CAAC,CAAC,GAAG,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,GAAE,OAAe,GAAG,CAAC,GAAG,IAAI,CAY/F;AAED;;;;;GAKG;AACH,wBAAgB,+BAA+B,CAAC,YAAY,EAAE,SAAS,GAAG,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAalH;AAOD;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAChC,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gJAAgJ;IAChJ,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,sEAAsE;IACtE,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;CACpC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,MAAM,CAoBlF;AAuTD;;;;;EAKE;AACF,wBAAgB,4BAA4B,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAc9D;AAGD;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAMjD;AAuED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAEtE;AAuCD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAG;IAAE,yBAAyB,CAAC,EAAE,OAAO,CAAC;IAAC,oBAAoB,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,MAAM,CAwCnJ;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;IACjD,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,oBAAoB,CAAC,EAAE,OAAO,CAAA;IAC9B,wBAAwB,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,MAAM,CAuBhD;AAID;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,OAAO,GAAG,MAAM,CAerG;AAGD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAalD;AAGD;;;GAGG;AACH,wBAAgB,MAAM,IAAI,MAAM,CAE/B;AAGD;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,GAAE,OAAc,GAAG,MAAM,EAAE,CA4CvG;AASD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,uEAAuE;IACvE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oHAAoH;IACpH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,iEAAiE;IACjE,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAmBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,GAAE,gBAAqB,GAAG,GAAG,CAUhF;AA2DD;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQ/C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAGxE;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,MAAM,GAAG,OAAO,CAAC;AAE3D,kFAAkF;AAClF,eAAO,MAAM,6BAA6B,MAAM,CAAC;AA8BjD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,oBAAoB,CAChC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAChC,aAAa,GAAE,MAAsC,GACtD,cAAc,CAchB;AA2BD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CA2B9F;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,OAAO,CAYrE;AAkFD;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAIlF"}
|
package/dist/util.js
CHANGED
|
@@ -5,34 +5,39 @@ import _ from 'lodash';
|
|
|
5
5
|
* because it is possible that a given class might have copies of its code in multiple paths in a deployed application. This approach ensures that no matter how many code copies might exist, there is only one instance of the object in question by using the Global Object Store.
|
|
6
6
|
* @returns
|
|
7
7
|
*/
|
|
8
|
+
let __globalObjectStore = undefined;
|
|
8
9
|
export function GetGlobalObjectStore() {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Memoised, and probed with `typeof`.
|
|
12
|
+
*
|
|
13
|
+
* The previous shape was `if (window)` inside a try/catch. In Node, `window` is an
|
|
14
|
+
* UNDECLARED identifier, so that line THROWS a ReferenceError on every single call and the
|
|
15
|
+
* catch falls through to `global` — correct answer, pathological path. This function sits
|
|
16
|
+
* under ClassFactory and BaseEngine hot paths, and building + unwinding that exception was
|
|
17
|
+
* measured at several percent of a busy server process. `typeof window` is the one probe
|
|
18
|
+
* that is legal on an undeclared identifier, and the environment does not change after
|
|
19
|
+
* startup, so the answer is computed once.
|
|
20
|
+
*
|
|
21
|
+
* The `&& window` is not redundant with the `typeof` guard: it keeps the one input where the
|
|
22
|
+
* two differ. `typeof null === 'object'`, so a DECLARED-but-null `window` — which SSR shims
|
|
23
|
+
* in the wild do produce — would pass a bare `typeof` check and get memoised as the store,
|
|
24
|
+
* where `if (window)` had correctly fallen through to `global`. Nothing in this repo does
|
|
25
|
+
* that today; the guard is here because MJ is embedded by callers whose environment we do
|
|
26
|
+
* not control, and because memoising the wrong answer is unrecoverable for the process.
|
|
27
|
+
*/
|
|
28
|
+
if (__globalObjectStore !== undefined)
|
|
29
|
+
return __globalObjectStore;
|
|
30
|
+
if (typeof window !== 'undefined' && window) {
|
|
31
|
+
__globalObjectStore = window;
|
|
32
|
+
}
|
|
33
|
+
else if (typeof global !== 'undefined') {
|
|
34
|
+
__globalObjectStore = global;
|
|
21
35
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
if (global)
|
|
26
|
-
return global;
|
|
27
|
-
else
|
|
28
|
-
return null; // won't get here typically because attempting to access the global object will throw an exception if it doesn't exist
|
|
29
|
-
}
|
|
30
|
-
catch (e) {
|
|
31
|
-
// if we get here, we don't have a global object either, so we're not running in a browser or node, so we're probably running in a unit test
|
|
32
|
-
// in that case, we don't have a provider saved, return null, we need to be either in node or a browser
|
|
33
|
-
return null;
|
|
34
|
-
}
|
|
36
|
+
else {
|
|
37
|
+
// neither browser nor node (e.g. an exotic test sandbox) — callers already handle null
|
|
38
|
+
__globalObjectStore = null;
|
|
35
39
|
}
|
|
40
|
+
return __globalObjectStore;
|
|
36
41
|
}
|
|
37
42
|
/**
|
|
38
43
|
* This utility function will copy all scalar and array properties from an object to a new object and return the new object.
|
|
@@ -329,8 +334,23 @@ export function CleanJSON(inputString) {
|
|
|
329
334
|
// This regex looks for ``` (including when the ` is escaped like \`)
|
|
330
335
|
// optionally followed by js or javascript (case-insensitive), then captures until the closing ```
|
|
331
336
|
const markdownRegex = /(?:```|\\`\\`\\`)(?:json|JSON)?\s*([\s\S]*?)(?:```|\\`\\`\\`)/gi;
|
|
337
|
+
// Fence extraction is for responses that WRAP their JSON in a fence, or bury it in prose.
|
|
338
|
+
// It must not run on something that is already shaped like a JSON envelope: a fence found
|
|
339
|
+
// inside such a string belongs to a string VALUE, not to the response, and extracting it
|
|
340
|
+
// throws the whole document away.
|
|
341
|
+
//
|
|
342
|
+
// This was not hypothetical. An agent response embedded a mermaid diagram in a markdown
|
|
343
|
+
// string field and left one quote in the diagram unescaped. The top-level parse failed on
|
|
344
|
+
// that quote, control reached here, the regex matched the ```mermaid fence inside the string
|
|
345
|
+
// value, and CleanJSON returned the diagram's contents — discarding a 28KB response and
|
|
346
|
+
// reporting "Unexpected token 'm'". The real defect (one quote, at a known offset) never
|
|
347
|
+
// surfaced, so neither the JSON5 stage nor the AI repair stage that followed had any chance.
|
|
348
|
+
//
|
|
349
|
+
// A genuinely fence-wrapped response starts with the fence, and a prose-buried one starts
|
|
350
|
+
// with prose, so neither is affected by this guard.
|
|
351
|
+
const looksLikeJSONEnvelope = processedString.startsWith('{') || processedString.startsWith('[');
|
|
332
352
|
// Check if the input contains Markdown code fences for JavaScript
|
|
333
|
-
const matches = Array.from(processedString.matchAll(markdownRegex));
|
|
353
|
+
const matches = looksLikeJSONEnvelope ? [] : Array.from(processedString.matchAll(markdownRegex));
|
|
334
354
|
if (matches.length > 0) {
|
|
335
355
|
// If there are matches, concatenate all captured groups (in case there are multiple code blocks)
|
|
336
356
|
const extracted = matches.map(match => match[1].trim()).join('\n');
|
|
@@ -366,9 +386,189 @@ export function CleanJSON(inputString) {
|
|
|
366
386
|
catch (error) {
|
|
367
387
|
// that was our last attempt and it failed so we need
|
|
368
388
|
// to throw an exception here with the orignal exception
|
|
369
|
-
|
|
389
|
+
//
|
|
390
|
+
// `cause` carries the untouched top-level parse error. The message is a summary that
|
|
391
|
+
// has been mangled by every transform above it; callers that need to act on the actual
|
|
392
|
+
// syntax error (a repair pass needing the failure offset, for instance) must not have
|
|
393
|
+
// to scrape it back out of this string.
|
|
394
|
+
throw new Error(`Failed to find a path to CleanJSON\n\n${originalException?.message}`, {
|
|
395
|
+
cause: originalException ?? undefined
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
/** Upper bound on repair passes. Each pass fixes exactly one character. */
|
|
401
|
+
const MAX_JSON_ESCAPING_REPAIRS = 40;
|
|
402
|
+
/**
|
|
403
|
+
* Deterministically repairs the single most common way an LLM breaks otherwise-valid JSON:
|
|
404
|
+
* a double quote or raw control character left unescaped inside a string value.
|
|
405
|
+
*
|
|
406
|
+
* ## Why this exists
|
|
407
|
+
*
|
|
408
|
+
* Models embed rich markdown in string fields — mermaid diagrams, HTML mockups, code samples —
|
|
409
|
+
* and reliably escape most of it. A single missed quote inside a 25KB response invalidates the
|
|
410
|
+
* whole document. Nothing else in the repair chain recovers that: JSON5's leniency covers trailing
|
|
411
|
+
* commas, comments and unquoted keys, but an unescaped `"` terminates a string in JSON5 exactly as
|
|
412
|
+
* it does in JSON. The remaining fallback is an LLM round-trip on the full payload, which is slow,
|
|
413
|
+
* costly, and itself unreliable at that size.
|
|
414
|
+
*
|
|
415
|
+
* ## How it works
|
|
416
|
+
*
|
|
417
|
+
* Purely error-driven, one character per pass:
|
|
418
|
+
*
|
|
419
|
+
* 1. `JSON.parse` the text and read the failure offset from the thrown error.
|
|
420
|
+
* 2. Scan backwards from that offset for the character that ended the string early.
|
|
421
|
+
* 3. Escape that one character.
|
|
422
|
+
* 4. Re-parse. Repeat until it parses or a stopping condition trips.
|
|
423
|
+
*
|
|
424
|
+
* Every pass is validated by a real parse, so this never "pattern matches" its way to a wrong
|
|
425
|
+
* answer the way a global regex rewrite would. It converges in one pass per offending character
|
|
426
|
+
* (about two per quoted mermaid label) and gives up rather than guessing when it cannot make
|
|
427
|
+
* progress.
|
|
428
|
+
*
|
|
429
|
+
* ## Safety
|
|
430
|
+
*
|
|
431
|
+
* This can, in principle, produce valid-but-wrong JSON: escaping a quote that legitimately ended a
|
|
432
|
+
* string would merge two pieces of structure. Three properties keep that in check — it only runs
|
|
433
|
+
* after a parse has already failed (so a correct document is never touched), it only ever *adds*
|
|
434
|
+
* escapes and never deletes or reorders content, and it reports every offset it changed so callers
|
|
435
|
+
* can log, audit, or schema-check the result before trusting it. Callers holding an expected shape
|
|
436
|
+
* should validate against it; a wrong guess almost always fails shape validation.
|
|
437
|
+
*
|
|
438
|
+
* @param inputString - Raw model output, expected to be a JSON envelope
|
|
439
|
+
* @param maxRepairs - Maximum characters to escape before giving up
|
|
440
|
+
* @returns The repair outcome; `repaired` is false when the input could not be recovered
|
|
441
|
+
*
|
|
442
|
+
* @example
|
|
443
|
+
* // A mermaid relationship label whose quotes were not escaped
|
|
444
|
+
* RepairJSONEscaping('{"doc":"```mermaid\\nerDiagram\\n A ||--o{ B : "has items"\\n```"}')
|
|
445
|
+
* // => { repaired: true, repairedOffsets: [...], value: { doc: '...' } }
|
|
446
|
+
*/
|
|
447
|
+
export function RepairJSONEscaping(inputString, maxRepairs = MAX_JSON_ESCAPING_REPAIRS) {
|
|
448
|
+
if (!inputString) {
|
|
449
|
+
return { repaired: false, repairedOffsets: [], reason: 'Input was empty.' };
|
|
450
|
+
}
|
|
451
|
+
let text = inputString;
|
|
452
|
+
const repairedOffsets = [];
|
|
453
|
+
for (let pass = 0; pass <= maxRepairs; pass++) {
|
|
454
|
+
try {
|
|
455
|
+
const value = JSON.parse(text);
|
|
456
|
+
return { repaired: pass > 0, text, value, repairedOffsets };
|
|
457
|
+
}
|
|
458
|
+
catch (error) {
|
|
459
|
+
if (pass === maxRepairs) {
|
|
460
|
+
return {
|
|
461
|
+
repaired: false,
|
|
462
|
+
repairedOffsets,
|
|
463
|
+
reason: `Still unparseable after ${maxRepairs} repairs.`
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
467
|
+
const offset = extractJSONErrorOffset(message);
|
|
468
|
+
if (offset === null) {
|
|
469
|
+
return { repaired: false, repairedOffsets, reason: `No failure offset in: ${message}` };
|
|
470
|
+
}
|
|
471
|
+
const target = findUnescapedStringTerminator(text, offset);
|
|
472
|
+
if (target === null) {
|
|
473
|
+
return {
|
|
474
|
+
repaired: false,
|
|
475
|
+
repairedOffsets,
|
|
476
|
+
reason: `No repairable character near offset ${offset}: ${message}`
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
text = text.slice(0, target) + escapeJSONStringChar(text[target]) + text.slice(target + 1);
|
|
480
|
+
repairedOffsets.push(target);
|
|
370
481
|
}
|
|
371
482
|
}
|
|
483
|
+
return { repaired: false, repairedOffsets, reason: 'Repair loop exhausted.' };
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Reads the character offset out of a `JSON.parse` error message.
|
|
487
|
+
*
|
|
488
|
+
* Engines word these differently ("...at position 23011", "...at position 23011 (line 21 column
|
|
489
|
+
* 512)"), so this matches the offset itself rather than any one phrasing.
|
|
490
|
+
*
|
|
491
|
+
* @param errorMessage - Message thrown by JSON.parse
|
|
492
|
+
* @returns The offset, or null when the message carries none
|
|
493
|
+
*/
|
|
494
|
+
function extractJSONErrorOffset(errorMessage) {
|
|
495
|
+
const match = /position (\d+)/i.exec(errorMessage);
|
|
496
|
+
if (!match) {
|
|
497
|
+
return null;
|
|
498
|
+
}
|
|
499
|
+
const offset = Number.parseInt(match[1], 10);
|
|
500
|
+
return Number.isFinite(offset) ? offset : null;
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Given the offset where parsing failed, finds the character that ended a string value early.
|
|
504
|
+
*
|
|
505
|
+
* The parser reports where it found something unexpected, which is at or just after the character
|
|
506
|
+
* that actually broke the document. Scanning backwards from there finds the closing quote the model
|
|
507
|
+
* should have escaped — or a raw control character, which is illegal inside a JSON string and is
|
|
508
|
+
* reported at its own offset.
|
|
509
|
+
*
|
|
510
|
+
* @param text - The JSON text being repaired
|
|
511
|
+
* @param errorOffset - Offset reported by JSON.parse
|
|
512
|
+
* @returns Offset of the character to escape, or null when nothing nearby is repairable
|
|
513
|
+
*/
|
|
514
|
+
function findUnescapedStringTerminator(text, errorOffset) {
|
|
515
|
+
// A raw control character is illegal inside a string and is flagged where it sits.
|
|
516
|
+
const atOffset = text[errorOffset];
|
|
517
|
+
if (atOffset !== undefined && isJSONControlCharacter(atOffset)) {
|
|
518
|
+
return errorOffset;
|
|
519
|
+
}
|
|
520
|
+
// Otherwise walk back to the quote that closed the string ahead of time. The scan is bounded:
|
|
521
|
+
// the culprit is adjacent to the failure by construction, and an unbounded walk could reach
|
|
522
|
+
// back into unrelated, correctly-formed structure.
|
|
523
|
+
const lowestOffset = Math.max(0, errorOffset - 64);
|
|
524
|
+
for (let i = Math.min(errorOffset, text.length - 1); i >= lowestOffset; i--) {
|
|
525
|
+
const char = text[i];
|
|
526
|
+
if (isJSONControlCharacter(char)) {
|
|
527
|
+
return i;
|
|
528
|
+
}
|
|
529
|
+
if (char === '"' && !isEscaped(text, i)) {
|
|
530
|
+
return i;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
return null;
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* True when the character must be escaped to appear inside a JSON string literal.
|
|
537
|
+
*
|
|
538
|
+
* @param char - Character to test
|
|
539
|
+
*/
|
|
540
|
+
function isJSONControlCharacter(char) {
|
|
541
|
+
return char < ' ';
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* True when the character at `index` is escaped, accounting for runs of backslashes — in `\\"` the
|
|
545
|
+
* quote is NOT escaped, because the two backslashes escape each other.
|
|
546
|
+
*
|
|
547
|
+
* @param text - The text being inspected
|
|
548
|
+
* @param index - Index of the character in question
|
|
549
|
+
*/
|
|
550
|
+
function isEscaped(text, index) {
|
|
551
|
+
let backslashes = 0;
|
|
552
|
+
for (let i = index - 1; i >= 0 && text[i] === '\\'; i--) {
|
|
553
|
+
backslashes++;
|
|
554
|
+
}
|
|
555
|
+
return backslashes % 2 === 1;
|
|
556
|
+
}
|
|
557
|
+
/**
|
|
558
|
+
* Escapes one character for inclusion in a JSON string literal.
|
|
559
|
+
*
|
|
560
|
+
* @param char - Character to escape
|
|
561
|
+
*/
|
|
562
|
+
function escapeJSONStringChar(char) {
|
|
563
|
+
switch (char) {
|
|
564
|
+
case '"': return '\\"';
|
|
565
|
+
case '\n': return '\\n';
|
|
566
|
+
case '\r': return '\\r';
|
|
567
|
+
case '\t': return '\\t';
|
|
568
|
+
case '\b': return '\\b';
|
|
569
|
+
case '\f': return '\\f';
|
|
570
|
+
default: return '\\u' + char.charCodeAt(0).toString(16).padStart(4, '0');
|
|
571
|
+
}
|
|
372
572
|
}
|
|
373
573
|
/**
|
|
374
574
|
* This function takes in a string that may contain JavaScript code in a markdown code block and returns the JavaScript code without the code block.
|
|
@@ -1225,6 +1425,39 @@ export function EscapeHTML(text) {
|
|
|
1225
1425
|
.replace(/"/g, '"')
|
|
1226
1426
|
.replace(/'/g, ''');
|
|
1227
1427
|
}
|
|
1428
|
+
/**
|
|
1429
|
+
* Safely escapes a string literal for use inside single-quoted SQL statements, clauses, or `ExtraFilter` predicates.
|
|
1430
|
+
*
|
|
1431
|
+
* Replaces each single quote with doubled single quotes (`''`) per ANSI SQL standard — the escaping
|
|
1432
|
+
* mechanism supported by both SQL Server and PostgreSQL — and removes null bytes (`\0`), which cannot
|
|
1433
|
+
* appear in any legitimate value and invite parser-level surprises when left in a predicate.
|
|
1434
|
+
*
|
|
1435
|
+
* **This escapes string literals and nothing else.** Three cases it does NOT cover:
|
|
1436
|
+
*
|
|
1437
|
+
* - **`LIKE` patterns** — `%`, `_` and `[` remain live wildcards after quote doubling, so a user
|
|
1438
|
+
* searching for `%` still matches every row. A LIKE value must additionally escape those
|
|
1439
|
+
* metacharacters and pair the clause with `ESCAPE '\'`. See `escapeLikeValue()` in
|
|
1440
|
+
* `@memberjunction/core` (`generic/runQuerySQLFilterImplementations.ts`) or
|
|
1441
|
+
* `GenericDatabaseProvider.escapeLikeTerm()`.
|
|
1442
|
+
* - **Identifier names** (table/column/schema) — those require bracket or double-quote quoting, and
|
|
1443
|
+
* are handled by SchemaEngine's `ValidateIdentifier()`.
|
|
1444
|
+
* - **Values that must not be missing** — `null`/`undefined` map to `''`, so a predicate built from a
|
|
1445
|
+
* missing value silently becomes `Field = ''` rather than throwing. Validate before interpolating
|
|
1446
|
+
* when absence is a bug.
|
|
1447
|
+
*
|
|
1448
|
+
* @param value - The raw string value to escape. If null or undefined, returns empty string.
|
|
1449
|
+
* @returns Safely escaped string value (without surrounding quotes).
|
|
1450
|
+
*
|
|
1451
|
+
* @example
|
|
1452
|
+
* ```typescript
|
|
1453
|
+
* const filter = `Email = '${EscapeSQLString(userEmail)}'`;
|
|
1454
|
+
* ```
|
|
1455
|
+
*/
|
|
1456
|
+
export function EscapeSQLString(value) {
|
|
1457
|
+
if (value == null)
|
|
1458
|
+
return '';
|
|
1459
|
+
return String(value).replace(/\0/g, '').replace(/'/g, "''");
|
|
1460
|
+
}
|
|
1228
1461
|
/** Default number of leading characters {@link detectRichTextFormat} inspects. */
|
|
1229
1462
|
export const DEFAULT_RICH_TEXT_SCAN_LENGTH = 500;
|
|
1230
1463
|
/**
|