@oxyhq/core 10.0.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/.tsbuildinfo +1 -1
- package/dist/cjs/index.js +8 -2
- package/dist/cjs/utils/textNormalization.js +168 -0
- package/dist/esm/.tsbuildinfo +1 -1
- package/dist/esm/index.js +4 -0
- package/dist/esm/utils/textNormalization.js +164 -0
- package/dist/types/.tsbuildinfo +1 -1
- package/dist/types/index.d.ts +1 -0
- package/dist/types/models/interfaces.d.ts +6 -0
- package/dist/types/utils/textNormalization.d.ts +100 -0
- package/package.json +3 -3
- package/src/index.ts +8 -0
- package/src/models/interfaces.ts +6 -0
- package/src/utils/__tests__/textNormalization.test.ts +196 -0
- package/src/utils/textNormalization.ts +173 -0
package/dist/types/index.d.ts
CHANGED
|
@@ -75,6 +75,7 @@ export type { PaginationParams, ApiResponse, ErrorResponse, } from './utils/apiU
|
|
|
75
75
|
export { ErrorCodes, createApiError, handleHttpError, validateRequiredFields, } from './utils/errorUtils';
|
|
76
76
|
export { retryAsync } from './utils/asyncUtils';
|
|
77
77
|
export { EMAIL_REGEX, USERNAME_REGEX, PASSWORD_REGEX, isValidEmail, isValidUsername, isValidPassword, isValidDisplayName, isRequiredString, isRequiredNumber, isRequiredBoolean, isValidArray, isValidObject, isValidUUID, isValidURL, isValidDate, isValidFileSize, isValidFileType, sanitizeString, sanitizeHTML, isValidObjectId, validateAndSanitizeUserInput, } from './utils/validationUtils';
|
|
78
|
+
export { normalizeInlineText, normalizeMultilineText, } from './utils/textNormalization';
|
|
78
79
|
export { logger, LogLevel, logAuth, logApi, logSession, logUser, logDevice, logPayment, logPerformance, } from './utils/loggerUtils';
|
|
79
80
|
export type { LogContext } from './utils/loggerUtils';
|
|
80
81
|
export { updateAvatarVisibility } from './utils/avatarUtils';
|
|
@@ -93,6 +93,12 @@ export interface User {
|
|
|
93
93
|
*/
|
|
94
94
|
name: UserNameResponse;
|
|
95
95
|
bio?: string;
|
|
96
|
+
/**
|
|
97
|
+
* Longer public profile text. Emitted alongside `bio` by every user DTO the
|
|
98
|
+
* API produces (single profile, `getUsersByIds`, follower/following/mutual
|
|
99
|
+
* lists, profile search) — the two are separate fields on the User document.
|
|
100
|
+
*/
|
|
101
|
+
description?: string;
|
|
96
102
|
phone?: string;
|
|
97
103
|
address?: string;
|
|
98
104
|
birthday?: string;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical text normalization for the Oxy ecosystem.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS
|
|
5
|
+
* ---------------
|
|
6
|
+
* Third-party text (federated actor display names, spoiler/CW text, image alt
|
|
7
|
+
* text, bios, `<title>` / `og:site_name` of scraped remote pages, …) arrives
|
|
8
|
+
* with the whitespace the remote author's markup happened to contain. A real
|
|
9
|
+
* bug: a remote page served
|
|
10
|
+
*
|
|
11
|
+
* <title>
|
|
12
|
+
* Mi título
|
|
13
|
+
* </title>
|
|
14
|
+
*
|
|
15
|
+
* and the extracted string — newlines and indentation included — was stored
|
|
16
|
+
* verbatim. Clients render text with React Native Web `Text`, which maps to
|
|
17
|
+
* CSS `white-space: pre-wrap`: unlike HTML, newlines and repeated spaces are
|
|
18
|
+
* NOT collapsed at render time, so the user saw a blank line and a six-space
|
|
19
|
+
* indent inside the link preview.
|
|
20
|
+
*
|
|
21
|
+
* Storage-time normalization is therefore the ONLY place this can be fixed:
|
|
22
|
+
* every renderer downstream is faithful by design.
|
|
23
|
+
*
|
|
24
|
+
* WHICH HELPER DO I USE?
|
|
25
|
+
* ----------------------
|
|
26
|
+
* - {@link normalizeInlineText} — the value is conceptually ONE LINE, and a
|
|
27
|
+
* line break in it is always an accident of the source markup: page titles,
|
|
28
|
+
* `siteName`, display names, image `alt`, handles, profile field labels.
|
|
29
|
+
* Every line break becomes a space.
|
|
30
|
+
*
|
|
31
|
+
* - {@link normalizeMultilineText} — the value is a BODY whose line breaks are
|
|
32
|
+
* the author's own paragraphs and must survive: post text, bios/summaries.
|
|
33
|
+
* Line breaks are preserved (capped at one blank line); only the surrounding
|
|
34
|
+
* whitespace noise is cleaned.
|
|
35
|
+
*
|
|
36
|
+
* Using the multiline helper on a title would keep the newlines and reproduce
|
|
37
|
+
* the original bug; using the inline helper on a post body would flatten the
|
|
38
|
+
* author's paragraphs into a single run-on line. Pick deliberately.
|
|
39
|
+
*
|
|
40
|
+
* Neither helper truncates, strips markup, or enforces a character policy —
|
|
41
|
+
* those are separate, product-specific concerns (see `isValidDisplayName` for
|
|
42
|
+
* the display-name character policy). These functions do exactly one thing:
|
|
43
|
+
* normalize whitespace and Unicode form.
|
|
44
|
+
*/
|
|
45
|
+
/**
|
|
46
|
+
* Normalize a SINGLE-LINE text value: page/link-preview titles, `siteName`,
|
|
47
|
+
* display names, image alt text, handles, profile field labels.
|
|
48
|
+
*
|
|
49
|
+
* A line break in such a value is never meaningful — it is an artifact of the
|
|
50
|
+
* markup the value was extracted from (`<title>\n Título\n</title>`) — and it
|
|
51
|
+
* survives into the UI because RN Web renders `Text` with `white-space:
|
|
52
|
+
* pre-wrap`. So ALL whitespace, line breaks included, collapses to one space.
|
|
53
|
+
*
|
|
54
|
+
* 1. NFC-normalize, so visually identical strings store and compare
|
|
55
|
+
* identically (a decomposed `e`+◌́ recomposes into `é`).
|
|
56
|
+
* 2. Collapse every run of whitespace — spaces, tabs, `\n`, `\r`, and Unicode
|
|
57
|
+
* spaces such as NBSP — to a single plain space.
|
|
58
|
+
* 3. Trim both ends.
|
|
59
|
+
*
|
|
60
|
+
* Length is NOT capped: a maximum length is a product rule of the specific
|
|
61
|
+
* field (see `MAX_DISPLAY_NAME_LENGTH`), not a property of text normalization.
|
|
62
|
+
* Markup is NOT stripped: run the caller's sanitizer first if the source can
|
|
63
|
+
* contain HTML.
|
|
64
|
+
*
|
|
65
|
+
* A value that is empty or whitespace-only returns `''`. Callers decide whether
|
|
66
|
+
* that means "omit the field" (`|| undefined`) or "store an empty string".
|
|
67
|
+
*
|
|
68
|
+
* Idempotent: `f(f(x)) === f(x)`.
|
|
69
|
+
*/
|
|
70
|
+
export declare function normalizeInlineText(value: string): string;
|
|
71
|
+
/**
|
|
72
|
+
* Normalize a MULTILINE text BODY: post text, bios, summaries — anywhere the
|
|
73
|
+
* line breaks are the author's paragraphs and must be preserved.
|
|
74
|
+
*
|
|
75
|
+
* Cleans the whitespace noise around those paragraphs without destroying them:
|
|
76
|
+
*
|
|
77
|
+
* 1. NFC-normalize (same rationale as {@link normalizeInlineText}).
|
|
78
|
+
* 2. Unify every line-break form (CRLF, lone CR, U+2028, U+2029) to `\n`.
|
|
79
|
+
* 3. Collapse runs of HORIZONTAL whitespace (spaces, tabs, NBSP and friends)
|
|
80
|
+
* to a single space. Line breaks are untouched.
|
|
81
|
+
* 4. Strip the horizontal whitespace at the END of each line.
|
|
82
|
+
* 5. Collapse three or more line breaks to exactly one blank line (`\n\n`).
|
|
83
|
+
* 6. Trim both ends.
|
|
84
|
+
*
|
|
85
|
+
* STEP 4 MUST PRECEDE STEP 5 — this is the whole point of the function. A
|
|
86
|
+
* "blank" line that actually contains spaces (`"a\n \n \nb"`) breaks the
|
|
87
|
+
* run of `\n` characters, so a bare `\n{3,}` collapse (step 5 alone) never sees
|
|
88
|
+
* it and the extra blank lines survive into the UI. That is exactly the bug in
|
|
89
|
+
* federated post bodies. Removing the trailing horizontal whitespace first
|
|
90
|
+
* turns those lines into real, empty lines, which step 5 then collapses.
|
|
91
|
+
*
|
|
92
|
+
* A single space at the START of a line is preserved: only RUNS of horizontal
|
|
93
|
+
* whitespace collapse, and an indent is not trailing whitespace, so a one-space
|
|
94
|
+
* indent is treated as the author's and left alone.
|
|
95
|
+
*
|
|
96
|
+
* A value that is empty or whitespace-only returns `''`.
|
|
97
|
+
*
|
|
98
|
+
* Idempotent: `f(f(x)) === f(x)`.
|
|
99
|
+
*/
|
|
100
|
+
export declare function normalizeMultilineText(value: string): string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@oxyhq/core",
|
|
3
|
-
"version": "10.
|
|
3
|
+
"version": "10.1.0",
|
|
4
4
|
"description": "OxyHQ SDK Foundation — API client, authentication, cryptographic identity, and shared utilities",
|
|
5
5
|
"main": "dist/cjs/index.js",
|
|
6
6
|
"module": "dist/esm/index.js",
|
|
@@ -94,8 +94,8 @@
|
|
|
94
94
|
}
|
|
95
95
|
},
|
|
96
96
|
"dependencies": {
|
|
97
|
-
"@oxyhq/contracts": "
|
|
98
|
-
"@oxyhq/protocol": "
|
|
97
|
+
"@oxyhq/contracts": "workspace:^",
|
|
98
|
+
"@oxyhq/protocol": "workspace:^",
|
|
99
99
|
"bip39": "^3.1.0",
|
|
100
100
|
"buffer": "^6.0.3",
|
|
101
101
|
"elliptic": "^6.6.1",
|
package/src/index.ts
CHANGED
|
@@ -466,6 +466,14 @@ export {
|
|
|
466
466
|
validateAndSanitizeUserInput,
|
|
467
467
|
} from './utils/validationUtils';
|
|
468
468
|
|
|
469
|
+
// ---------------------------------------------------------------------------
|
|
470
|
+
// Text normalization
|
|
471
|
+
// ---------------------------------------------------------------------------
|
|
472
|
+
export {
|
|
473
|
+
normalizeInlineText,
|
|
474
|
+
normalizeMultilineText,
|
|
475
|
+
} from './utils/textNormalization';
|
|
476
|
+
|
|
469
477
|
// ---------------------------------------------------------------------------
|
|
470
478
|
// Logging
|
|
471
479
|
// ---------------------------------------------------------------------------
|
package/src/models/interfaces.ts
CHANGED
|
@@ -103,6 +103,12 @@ export interface User {
|
|
|
103
103
|
*/
|
|
104
104
|
name: UserNameResponse;
|
|
105
105
|
bio?: string;
|
|
106
|
+
/**
|
|
107
|
+
* Longer public profile text. Emitted alongside `bio` by every user DTO the
|
|
108
|
+
* API produces (single profile, `getUsersByIds`, follower/following/mutual
|
|
109
|
+
* lists, profile search) — the two are separate fields on the User document.
|
|
110
|
+
*/
|
|
111
|
+
description?: string;
|
|
106
112
|
phone?: string;
|
|
107
113
|
address?: string;
|
|
108
114
|
birthday?: string;
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { normalizeInlineText, normalizeMultilineText } from '../textNormalization';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Whitespace characters are built from their code points rather than pasted as
|
|
5
|
+
* literals: an NBSP or a U+2028 in the source would be invisible to a reviewer
|
|
6
|
+
* and trivially "fixed" by an editor, silently gutting the test it anchors.
|
|
7
|
+
*/
|
|
8
|
+
const NBSP = String.fromCharCode(0x00a0); // no-break space
|
|
9
|
+
const EN_QUAD = String.fromCharCode(0x2000); // U+2000 space block
|
|
10
|
+
const IDEOGRAPHIC_SPACE = String.fromCharCode(0x3000); // CJK full-width space
|
|
11
|
+
const NARROW_NBSP = String.fromCharCode(0x202f);
|
|
12
|
+
const ZWNBSP = String.fromCharCode(0xfeff); // BOM when leading
|
|
13
|
+
const LINE_SEPARATOR = String.fromCharCode(0x2028);
|
|
14
|
+
const PARAGRAPH_SEPARATOR = String.fromCharCode(0x2029);
|
|
15
|
+
|
|
16
|
+
/** "é" as a base letter plus a combining acute accent (NFD form). */
|
|
17
|
+
const DECOMPOSED_E_ACUTE = `e${String.fromCharCode(0x0301)}`;
|
|
18
|
+
/** The precomposed "é" (NFC form) the two above must normalize into. */
|
|
19
|
+
const COMPOSED_E_ACUTE = String.fromCharCode(0x00e9);
|
|
20
|
+
|
|
21
|
+
describe('normalizeInlineText', () => {
|
|
22
|
+
it('flattens the indented multi-line <title> that caused the original bug', () => {
|
|
23
|
+
expect(normalizeInlineText('\n Mi título\n ')).toBe('Mi título');
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('collapses every whitespace form to a single space', () => {
|
|
27
|
+
expect(normalizeInlineText('a\tb\nc\r\nd e')).toBe('a b c d e');
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('collapses Unicode spaces (NBSP, U+2000 block, ideographic, narrow NBSP)', () => {
|
|
31
|
+
expect(
|
|
32
|
+
normalizeInlineText(`a${NBSP}b${EN_QUAD}c${IDEOGRAPHIC_SPACE}d${NARROW_NBSP}e`)
|
|
33
|
+
).toBe('a b c d e');
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('collapses the Unicode line and paragraph separators', () => {
|
|
37
|
+
expect(normalizeInlineText(`a${LINE_SEPARATOR}b${PARAGRAPH_SEPARATOR}c`)).toBe('a b c');
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('strips a leading BOM / zero-width no-break space', () => {
|
|
41
|
+
expect(normalizeInlineText(`${ZWNBSP}Título`)).toBe('Título');
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('handles CRLF line endings', () => {
|
|
45
|
+
expect(normalizeInlineText('News\r\n\r\nToday')).toBe('News Today');
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('returns an empty string for empty and whitespace-only input', () => {
|
|
49
|
+
expect(normalizeInlineText('')).toBe('');
|
|
50
|
+
expect(normalizeInlineText(' ')).toBe('');
|
|
51
|
+
expect(normalizeInlineText(`\n\t ${NBSP}\r\n`)).toBe('');
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('NFC-normalizes so decomposed accents are stored composed', () => {
|
|
55
|
+
const normalized = normalizeInlineText(`Ren${DECOMPOSED_E_ACUTE}e`);
|
|
56
|
+
expect(normalized).toBe(`Ren${COMPOSED_E_ACUTE}e`);
|
|
57
|
+
expect(normalized).toHaveLength(5);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('leaves already-clean text untouched', () => {
|
|
61
|
+
expect(normalizeInlineText('Hacker News')).toBe('Hacker News');
|
|
62
|
+
expect(normalizeInlineText('Título en español — con guion')).toBe(
|
|
63
|
+
'Título en español — con guion'
|
|
64
|
+
);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('does not truncate long values (length caps are a product rule)', () => {
|
|
68
|
+
const long = 'a'.repeat(500);
|
|
69
|
+
expect(normalizeInlineText(long)).toHaveLength(500);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it('does not strip markup or punctuation (that is the sanitizer\'s job)', () => {
|
|
73
|
+
expect(normalizeInlineText(' <b>Bold</b> & "quoted" ')).toBe('<b>Bold</b> & "quoted"');
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('is idempotent', () => {
|
|
77
|
+
const inputs = [
|
|
78
|
+
'\n Mi título\n ',
|
|
79
|
+
`a${NBSP}${NBSP}b`,
|
|
80
|
+
'Hacker News',
|
|
81
|
+
'',
|
|
82
|
+
' ',
|
|
83
|
+
`Ren${DECOMPOSED_E_ACUTE}e`,
|
|
84
|
+
];
|
|
85
|
+
for (const input of inputs) {
|
|
86
|
+
const once = normalizeInlineText(input);
|
|
87
|
+
expect(normalizeInlineText(once)).toBe(once);
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
describe('normalizeMultilineText', () => {
|
|
93
|
+
it('preserves the author\'s paragraphs', () => {
|
|
94
|
+
const body = 'First paragraph.\n\nSecond paragraph.\nSame paragraph, next line.';
|
|
95
|
+
expect(normalizeMultilineText(body)).toBe(body);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('collapses a blank line that contains spaces — the ordering bug', () => {
|
|
99
|
+
// The "blank" lines hold spaces, so they break the run of \n characters: a
|
|
100
|
+
// bare /\n{3,}/ collapse never sees them. Trailing horizontal whitespace
|
|
101
|
+
// must be stripped FIRST.
|
|
102
|
+
expect(normalizeMultilineText('a\n \n \nb')).toBe('a\n\nb');
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('collapses a blank line that contains tabs and NBSP', () => {
|
|
106
|
+
expect(normalizeMultilineText(`a\n\t\n${NBSP}${NBSP}\nb`)).toBe('a\n\nb');
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('strips trailing horizontal whitespace from every line', () => {
|
|
110
|
+
expect(normalizeMultilineText('one \ntwo\t\nthree ')).toBe('one\ntwo\nthree');
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('collapses three or more newlines to a single blank line', () => {
|
|
114
|
+
expect(normalizeMultilineText('a\n\n\n\n\nb')).toBe('a\n\nb');
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('keeps exactly one blank line as-is', () => {
|
|
118
|
+
expect(normalizeMultilineText('a\n\nb')).toBe('a\n\nb');
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it('collapses runs of horizontal whitespace to one space, keeping newlines', () => {
|
|
122
|
+
expect(normalizeMultilineText('hello world\nsecond\t\tline')).toBe(
|
|
123
|
+
'hello world\nsecond line'
|
|
124
|
+
);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it('preserves a single-space indent (it is the author\'s, not markup noise)', () => {
|
|
128
|
+
expect(normalizeMultilineText('a\n b')).toBe('a\n b');
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('normalizes CRLF and lone CR to \\n', () => {
|
|
132
|
+
expect(normalizeMultilineText('a\r\nb\rc')).toBe('a\nb\nc');
|
|
133
|
+
expect(normalizeMultilineText('a\r\n\r\n\r\n\r\nb')).toBe('a\n\nb');
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('normalizes the Unicode line and paragraph separators to \\n', () => {
|
|
137
|
+
expect(normalizeMultilineText(`a${LINE_SEPARATOR}b${PARAGRAPH_SEPARATOR}c`)).toBe('a\nb\nc');
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it('collapses Unicode spaces without touching line structure', () => {
|
|
141
|
+
expect(normalizeMultilineText(`a${NBSP}${NBSP}b\nc${IDEOGRAPHIC_SPACE}d`)).toBe('a b\nc d');
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it('trims both ends, including leading and trailing blank lines', () => {
|
|
145
|
+
expect(normalizeMultilineText('\n\n Hello\n\nWorld \n\n ')).toBe('Hello\n\nWorld');
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it('returns an empty string for empty and whitespace-only input', () => {
|
|
149
|
+
expect(normalizeMultilineText('')).toBe('');
|
|
150
|
+
expect(normalizeMultilineText(' ')).toBe('');
|
|
151
|
+
expect(normalizeMultilineText(`\n\n \t${NBSP}\r\n \n`)).toBe('');
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it('NFC-normalizes so decomposed accents are stored composed', () => {
|
|
155
|
+
expect(normalizeMultilineText(`Caf${DECOMPOSED_E_ACUTE}\n\nabierto`)).toBe(
|
|
156
|
+
`Caf${COMPOSED_E_ACUTE}\n\nabierto`
|
|
157
|
+
);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it('leaves an already-clean body untouched', () => {
|
|
161
|
+
const body = 'Line one\nLine two\n\nNew paragraph.';
|
|
162
|
+
expect(normalizeMultilineText(body)).toBe(body);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it('cleans a realistic federated post body without losing its paragraphs', () => {
|
|
166
|
+
const federated = ' Hola a todos.\r\n \r\n\r\nEsto es una prueba.\t\r\nFin. ';
|
|
167
|
+
expect(normalizeMultilineText(federated)).toBe(
|
|
168
|
+
'Hola a todos.\n\nEsto es una prueba.\nFin.'
|
|
169
|
+
);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it('is idempotent', () => {
|
|
173
|
+
const inputs = [
|
|
174
|
+
'a\n \n \nb',
|
|
175
|
+
'a\r\n\r\n\r\nb',
|
|
176
|
+
'First paragraph.\n\nSecond paragraph.',
|
|
177
|
+
`a${NBSP}b`,
|
|
178
|
+
'a\n b',
|
|
179
|
+
'',
|
|
180
|
+
' ',
|
|
181
|
+
`Caf${DECOMPOSED_E_ACUTE}`,
|
|
182
|
+
];
|
|
183
|
+
for (const input of inputs) {
|
|
184
|
+
const once = normalizeMultilineText(input);
|
|
185
|
+
expect(normalizeMultilineText(once)).toBe(once);
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
describe('choosing between the two helpers', () => {
|
|
191
|
+
it('inline flattens what multiline preserves', () => {
|
|
192
|
+
const body = 'Title\n\nSubtitle';
|
|
193
|
+
expect(normalizeInlineText(body)).toBe('Title Subtitle');
|
|
194
|
+
expect(normalizeMultilineText(body)).toBe('Title\n\nSubtitle');
|
|
195
|
+
});
|
|
196
|
+
});
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical text normalization for the Oxy ecosystem.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS
|
|
5
|
+
* ---------------
|
|
6
|
+
* Third-party text (federated actor display names, spoiler/CW text, image alt
|
|
7
|
+
* text, bios, `<title>` / `og:site_name` of scraped remote pages, …) arrives
|
|
8
|
+
* with the whitespace the remote author's markup happened to contain. A real
|
|
9
|
+
* bug: a remote page served
|
|
10
|
+
*
|
|
11
|
+
* <title>
|
|
12
|
+
* Mi título
|
|
13
|
+
* </title>
|
|
14
|
+
*
|
|
15
|
+
* and the extracted string — newlines and indentation included — was stored
|
|
16
|
+
* verbatim. Clients render text with React Native Web `Text`, which maps to
|
|
17
|
+
* CSS `white-space: pre-wrap`: unlike HTML, newlines and repeated spaces are
|
|
18
|
+
* NOT collapsed at render time, so the user saw a blank line and a six-space
|
|
19
|
+
* indent inside the link preview.
|
|
20
|
+
*
|
|
21
|
+
* Storage-time normalization is therefore the ONLY place this can be fixed:
|
|
22
|
+
* every renderer downstream is faithful by design.
|
|
23
|
+
*
|
|
24
|
+
* WHICH HELPER DO I USE?
|
|
25
|
+
* ----------------------
|
|
26
|
+
* - {@link normalizeInlineText} — the value is conceptually ONE LINE, and a
|
|
27
|
+
* line break in it is always an accident of the source markup: page titles,
|
|
28
|
+
* `siteName`, display names, image `alt`, handles, profile field labels.
|
|
29
|
+
* Every line break becomes a space.
|
|
30
|
+
*
|
|
31
|
+
* - {@link normalizeMultilineText} — the value is a BODY whose line breaks are
|
|
32
|
+
* the author's own paragraphs and must survive: post text, bios/summaries.
|
|
33
|
+
* Line breaks are preserved (capped at one blank line); only the surrounding
|
|
34
|
+
* whitespace noise is cleaned.
|
|
35
|
+
*
|
|
36
|
+
* Using the multiline helper on a title would keep the newlines and reproduce
|
|
37
|
+
* the original bug; using the inline helper on a post body would flatten the
|
|
38
|
+
* author's paragraphs into a single run-on line. Pick deliberately.
|
|
39
|
+
*
|
|
40
|
+
* Neither helper truncates, strips markup, or enforces a character policy —
|
|
41
|
+
* those are separate, product-specific concerns (see `isValidDisplayName` for
|
|
42
|
+
* the display-name character policy). These functions do exactly one thing:
|
|
43
|
+
* normalize whitespace and Unicode form.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Characters that make a value ineligible for the zero-work fast path, and the
|
|
48
|
+
* whitespace shapes that a normalized INLINE value can never contain.
|
|
49
|
+
*
|
|
50
|
+
* A value that matches nothing here is, by construction, already normalized:
|
|
51
|
+
* it holds only printable ASCII (which is NFC-stable, so `normalize('NFC')`
|
|
52
|
+
* would be a no-op), its only whitespace is the plain space, it has no leading
|
|
53
|
+
* or trailing space, and no run of two spaces. Returning it untouched skips
|
|
54
|
+
* three string allocations — worth it because the common case in the feed
|
|
55
|
+
* hydration hot path is text that is already clean.
|
|
56
|
+
*
|
|
57
|
+
* Non-global (safe for repeated `.test()`; a global regex is stateful).
|
|
58
|
+
*/
|
|
59
|
+
const INLINE_NEEDS_NORMALIZATION = /[^\x20-\x7E]|^ | $| {2}/;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Same idea as {@link INLINE_NEEDS_NORMALIZATION}, for MULTILINE values: `\n`
|
|
63
|
+
* joins the printable-ASCII fast-path alphabet, and the additional shapes a
|
|
64
|
+
* normalized body can never contain are a space before a line break (trailing
|
|
65
|
+
* horizontal whitespace) and a run of three line breaks (more than one blank
|
|
66
|
+
* line). A single space AFTER a line break is legal — a one-space indent is
|
|
67
|
+
* the author's, and normalization deliberately preserves it.
|
|
68
|
+
*/
|
|
69
|
+
const MULTILINE_NEEDS_NORMALIZATION = /[^\x20-\x7E\n]|^[ \n]|[ \n]$| {2}| \n|\n{3}/;
|
|
70
|
+
|
|
71
|
+
/** Any run of whitespace, including tabs, line breaks and Unicode spaces. */
|
|
72
|
+
const ANY_WHITESPACE_RUN = /\s+/g;
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Every line-break form, unified to `\n` before the multiline passes run:
|
|
76
|
+
* CRLF (Windows), a lone CR (classic Mac / stray `\r`), and the Unicode LINE
|
|
77
|
+
* SEPARATOR (U+2028) / PARAGRAPH SEPARATOR (U+2029), which are mandatory breaks
|
|
78
|
+
* in Unicode and a well-known hazard in JSON/JS payloads. CRLF must be matched
|
|
79
|
+
* before the lone `\r` alternative or it would yield two breaks. The separators
|
|
80
|
+
* are matched by Unicode property (`\p{Zl}` = U+2028, `\p{Zp}` = U+2029) rather
|
|
81
|
+
* than as literals: a literal U+2028/U+2029 is a LineTerminator and cannot appear
|
|
82
|
+
* inside a regex literal at all.
|
|
83
|
+
*/
|
|
84
|
+
const LINE_BREAK_FORMS = /\r\n|\r|\p{Zl}|\p{Zp}/gu;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* A run of HORIZONTAL whitespace: any whitespace that is not a line break.
|
|
88
|
+
* `[^\S\n]` is "whitespace, minus `\n`" — it covers the tab, the vertical tab,
|
|
89
|
+
* the form feed and every Unicode space separator (NBSP U+00A0, the U+2000
|
|
90
|
+
* block, the ideographic space U+3000, the zero-width no-break space U+FEFF).
|
|
91
|
+
* Applied AFTER {@link LINE_BREAK_FORMS}, so no line break can hide in it.
|
|
92
|
+
*/
|
|
93
|
+
const HORIZONTAL_WHITESPACE_RUN = /[^\S\n]+/g;
|
|
94
|
+
|
|
95
|
+
/** Horizontal whitespace at the end of a line — the blank-line spoiler. */
|
|
96
|
+
const TRAILING_HORIZONTAL_WHITESPACE = / +\n/g;
|
|
97
|
+
|
|
98
|
+
/** Three or more line breaks: more than one blank line between paragraphs. */
|
|
99
|
+
const EXCESS_BLANK_LINES = /\n{3,}/g;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Normalize a SINGLE-LINE text value: page/link-preview titles, `siteName`,
|
|
103
|
+
* display names, image alt text, handles, profile field labels.
|
|
104
|
+
*
|
|
105
|
+
* A line break in such a value is never meaningful — it is an artifact of the
|
|
106
|
+
* markup the value was extracted from (`<title>\n Título\n</title>`) — and it
|
|
107
|
+
* survives into the UI because RN Web renders `Text` with `white-space:
|
|
108
|
+
* pre-wrap`. So ALL whitespace, line breaks included, collapses to one space.
|
|
109
|
+
*
|
|
110
|
+
* 1. NFC-normalize, so visually identical strings store and compare
|
|
111
|
+
* identically (a decomposed `e`+◌́ recomposes into `é`).
|
|
112
|
+
* 2. Collapse every run of whitespace — spaces, tabs, `\n`, `\r`, and Unicode
|
|
113
|
+
* spaces such as NBSP — to a single plain space.
|
|
114
|
+
* 3. Trim both ends.
|
|
115
|
+
*
|
|
116
|
+
* Length is NOT capped: a maximum length is a product rule of the specific
|
|
117
|
+
* field (see `MAX_DISPLAY_NAME_LENGTH`), not a property of text normalization.
|
|
118
|
+
* Markup is NOT stripped: run the caller's sanitizer first if the source can
|
|
119
|
+
* contain HTML.
|
|
120
|
+
*
|
|
121
|
+
* A value that is empty or whitespace-only returns `''`. Callers decide whether
|
|
122
|
+
* that means "omit the field" (`|| undefined`) or "store an empty string".
|
|
123
|
+
*
|
|
124
|
+
* Idempotent: `f(f(x)) === f(x)`.
|
|
125
|
+
*/
|
|
126
|
+
export function normalizeInlineText(value: string): string {
|
|
127
|
+
if (!INLINE_NEEDS_NORMALIZATION.test(value)) {
|
|
128
|
+
return value;
|
|
129
|
+
}
|
|
130
|
+
return value.normalize('NFC').replace(ANY_WHITESPACE_RUN, ' ').trim();
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Normalize a MULTILINE text BODY: post text, bios, summaries — anywhere the
|
|
135
|
+
* line breaks are the author's paragraphs and must be preserved.
|
|
136
|
+
*
|
|
137
|
+
* Cleans the whitespace noise around those paragraphs without destroying them:
|
|
138
|
+
*
|
|
139
|
+
* 1. NFC-normalize (same rationale as {@link normalizeInlineText}).
|
|
140
|
+
* 2. Unify every line-break form (CRLF, lone CR, U+2028, U+2029) to `\n`.
|
|
141
|
+
* 3. Collapse runs of HORIZONTAL whitespace (spaces, tabs, NBSP and friends)
|
|
142
|
+
* to a single space. Line breaks are untouched.
|
|
143
|
+
* 4. Strip the horizontal whitespace at the END of each line.
|
|
144
|
+
* 5. Collapse three or more line breaks to exactly one blank line (`\n\n`).
|
|
145
|
+
* 6. Trim both ends.
|
|
146
|
+
*
|
|
147
|
+
* STEP 4 MUST PRECEDE STEP 5 — this is the whole point of the function. A
|
|
148
|
+
* "blank" line that actually contains spaces (`"a\n \n \nb"`) breaks the
|
|
149
|
+
* run of `\n` characters, so a bare `\n{3,}` collapse (step 5 alone) never sees
|
|
150
|
+
* it and the extra blank lines survive into the UI. That is exactly the bug in
|
|
151
|
+
* federated post bodies. Removing the trailing horizontal whitespace first
|
|
152
|
+
* turns those lines into real, empty lines, which step 5 then collapses.
|
|
153
|
+
*
|
|
154
|
+
* A single space at the START of a line is preserved: only RUNS of horizontal
|
|
155
|
+
* whitespace collapse, and an indent is not trailing whitespace, so a one-space
|
|
156
|
+
* indent is treated as the author's and left alone.
|
|
157
|
+
*
|
|
158
|
+
* A value that is empty or whitespace-only returns `''`.
|
|
159
|
+
*
|
|
160
|
+
* Idempotent: `f(f(x)) === f(x)`.
|
|
161
|
+
*/
|
|
162
|
+
export function normalizeMultilineText(value: string): string {
|
|
163
|
+
if (!MULTILINE_NEEDS_NORMALIZATION.test(value)) {
|
|
164
|
+
return value;
|
|
165
|
+
}
|
|
166
|
+
return value
|
|
167
|
+
.normalize('NFC')
|
|
168
|
+
.replace(LINE_BREAK_FORMS, '\n')
|
|
169
|
+
.replace(HORIZONTAL_WHITESPACE_RUN, ' ')
|
|
170
|
+
.replace(TRAILING_HORIZONTAL_WHITESPACE, '\n')
|
|
171
|
+
.replace(EXCESS_BLANK_LINES, '\n\n')
|
|
172
|
+
.trim();
|
|
173
|
+
}
|