@pipeworx/mcp-edge-case-museum 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -12
- package/bin/cli.js +55 -11
- package/lib/index.js +333 -0
- package/package.json +17 -16
- package/server.json +1 -1
- package/tools.json +98 -0
- package/src/bake.mjs +0 -112
- package/src/ecm-index-data.ts +0 -15
- package/src/exhibits/datetime/add-24-hours-vs-add-1-day-across-dst.json +0 -51
- package/src/exhibits/datetime/add-365-days-to-feb-29-not-next-feb-29.json +0 -49
- package/src/exhibits/datetime/age-calculator-feb29-birthday-crash-vs-silent.json +0 -52
- package/src/exhibits/datetime/century-leap-year-1900-not-leap.json +0 -68
- package/src/exhibits/datetime/datetime-utcnow-naive-deprecated.json +0 -39
- package/src/exhibits/datetime/dst-fall-back-ambiguous-fold.json +0 -51
- package/src/exhibits/datetime/dst-spring-forward-nonexistent-time.json +0 -57
- package/src/exhibits/datetime/fractional-utc-offsets-kathmandu-kolkata.json +0 -50
- package/src/exhibits/datetime/iso-week-year-vs-calendar-year.json +0 -48
- package/src/exhibits/datetime/js-date-field-overflow-rollover.json +0 -49
- package/src/exhibits/datetime/leap-second-rejected-vs-silently-invalid.json +0 -57
- package/src/exhibits/datetime/lord-howe-30-minute-dst.json +0 -47
- package/src/exhibits/datetime/midnight-that-doesnt-exist-sao-paulo.json +0 -53
- package/src/exhibits/datetime/python-naive-aware-comparison-typeerror.json +0 -46
- package/src/exhibits/datetime/samoa-skipped-day-2011.json +0 -50
- package/src/exhibits/datetime/southern-hemisphere-dst-reversed.json +0 -50
- package/src/exhibits/datetime/two-digit-year-century-pivot-mismatch.json +0 -49
- package/src/exhibits/datetime/year-2038-int32-overflow.json +0 -52
- package/src/exhibits/filenames/absolute-second-argument-resets-join.json +0 -62
- package/src/exhibits/filenames/bidi-override-extension-spoof.json +0 -58
- package/src/exhibits/filenames/case-insensitive-filesystem-pure-path-mismatch.json +0 -74
- package/src/exhibits/filenames/dotfile-has-no-extension.json +0 -61
- package/src/exhibits/filenames/extension-trap-double-extension.json +0 -62
- package/src/exhibits/filenames/filename-length-limits-bytes-not-chars.json +0 -66
- package/src/exhibits/filenames/leading-dash-filename-parsed-as-flag.json +0 -67
- package/src/exhibits/filenames/nfc-nfd-filename-lookup-mismatch.json +0 -68
- package/src/exhibits/filenames/ntfs-alternate-data-stream-colon.json +0 -59
- package/src/exhibits/filenames/ntpath-splitdrive-unc-paths.json +0 -58
- package/src/exhibits/filenames/nul-byte-in-filename-raises-at-open.json +0 -60
- package/src/exhibits/filenames/path-join-escapes-the-base-directory.json +0 -63
- package/src/exhibits/filenames/windows-backslash-traversal-ntpath.json +0 -62
- package/src/exhibits/filenames/windows-reserved-device-names.json +0 -73
- package/src/exhibits/filenames/windows-trailing-dot-space-stripped.json +0 -77
- package/src/exhibits/float-money/decimal-from-float-vs-string.json +0 -66
- package/src/exhibits/float-money/float-anchor-0-1-plus-0-2.json +0 -67
- package/src/exhibits/float-money/intl-number-format-de-de-roundtrip-misparsed.json +0 -62
- package/src/exhibits/float-money/iso4217-minor-units-not-always-two.json +0 -78
- package/src/exhibits/float-money/math-abs-negative-zero.json +0 -51
- package/src/exhibits/float-money/max-safe-integer-plus-two.json +0 -62
- package/src/exhibits/float-money/naive-loop-sum-order-dependence.json +0 -71
- package/src/exhibits/float-money/nan-breaks-equality-and-max-min.json +0 -70
- package/src/exhibits/float-money/negative-zero-balance-formatting.json +0 -74
- package/src/exhibits/float-money/number-epsilon-false-positive-small-magnitude.json +0 -52
- package/src/exhibits/float-money/parsefloat-comma-thousands-separator.json +0 -66
- package/src/exhibits/float-money/python-banker-rounding-vs-js-round-half-up.json +0 -65
- package/src/exhibits/float-money/python-round-2-675-float-imprecision.json +0 -64
- package/src/exhibits/float-money/splitting-100-dollars-three-ways-leaves-a-cent.json +0 -63
- package/src/exhibits/float-money/sum-ten-dimes-not-a-dollar.json +0 -77
- package/src/exhibits/float-money/tofixed-1-005-rounds-down-not-up.json +0 -63
- package/src/exhibits/float-money/tofixed-2-345-inconsistent-rounding.json +0 -64
- package/src/exhibits/integers/array-index-above-2-32-2-not-an-index.json +0 -42
- package/src/exhibits/integers/bigint-number-mixing-typeerror.json +0 -49
- package/src/exhibits/integers/bitwise-32-bit-truncation.json +0 -56
- package/src/exhibits/integers/bool-is-int-subclass-dict-key-collision.json +0 -58
- package/src/exhibits/integers/floor-div-mod-sign-python-vs-js.json +0 -48
- package/src/exhibits/integers/json-parse-loses-id-precision.json +0 -58
- package/src/exhibits/integers/max-safe-integer-plus-one-equals-plus-two.json +0 -58
- package/src/exhibits/integers/number-empty-and-whitespace-string-equals-zero.json +0 -55
- package/src/exhibits/integers/parseint-null-radix36-digit-soup.json +0 -57
- package/src/exhibits/integers/parseint-number-argument-scientific-notation.json +0 -48
- package/src/exhibits/integers/parseint-stops-at-first-invalid-char-vs-number.json +0 -58
- package/src/exhibits/integers/python-int-accepts-fullwidth-unicode-digits.json +0 -56
- package/src/exhibits/integers/python-int-accepts-underscores-and-whitespace.json +0 -48
- package/src/exhibits/integers/python-unbounded-int-vs-fixed-width-wrap.json +0 -49
- package/src/exhibits/names-addresses/address-with-no-street-number.json +0 -54
- package/src/exhibits/names-addresses/ascii-regex-rejects-accented-and-hyphenated-names.json +0 -55
- package/src/exhibits/names-addresses/house-number-suffix-collapsed-by-digit-extraction.json +0 -55
- package/src/exhibits/names-addresses/mononym-fails-first-last-required.json +0 -56
- package/src/exhibits/names-addresses/naive-lowercase-breaks-greek-final-sigma.json +0 -64
- package/src/exhibits/names-addresses/null-surname-coerced-to-null-value.json +0 -58
- package/src/exhibits/names-addresses/single-letter-surname-fails-min-length.json +0 -55
- package/src/exhibits/names-addresses/sort-by-surname-particle-convention-mismatch.json +0 -57
- package/src/exhibits/names-addresses/spanish-double-surname-treated-as-single.json +0 -55
- package/src/exhibits/names-addresses/split-surname-on-last-space-loses-particle.json +0 -59
- package/src/exhibits/names-addresses/title-case-mangles-mixed-case-and-apostrophes.json +0 -60
- package/src/exhibits/names-addresses/uk-canada-postal-code-assumed-numeric.json +0 -55
- package/src/exhibits/names-addresses/us-zip-code-leading-zero-lost-as-integer.json +0 -58
- package/src/exhibits/names-addresses/varchar-truncation-splits-name-mid-grapheme.json +0 -63
- package/src/exhibits/parsing/csv-crlf-trailing-blank-line.json +0 -87
- package/src/exhibits/parsing/csv-doubled-quote-escaping.json +0 -70
- package/src/exhibits/parsing/csv-formula-injection-excel.json +0 -71
- package/src/exhibits/parsing/csv-quoted-field-with-comma.json +0 -70
- package/src/exhibits/parsing/csv-quoted-field-with-newline.json +0 -70
- package/src/exhibits/parsing/csv-semicolon-delimited-european-excel.json +0 -70
- package/src/exhibits/parsing/csv-utf8-bom-first-header.json +0 -74
- package/src/exhibits/parsing/email-regex-rejects-valid-apostrophe-address.json +0 -68
- package/src/exhibits/parsing/json-big-integer-precision-loss.json +0 -65
- package/src/exhibits/parsing/json-duplicate-keys-last-wins.json +0 -64
- package/src/exhibits/parsing/json-lone-surrogate-roundtrip.json +0 -58
- package/src/exhibits/parsing/json-nan-infinity-not-valid-json.json +0 -54
- package/src/exhibits/parsing/url-backslash-treated-as-slash-whatwg.json +0 -60
- package/src/exhibits/parsing/url-urlparse-never-raises-on-garbage.json +0 -60
- package/src/exhibits/parsing/url-userinfo-trusted-looking-host.json +0 -60
- package/src/exhibits/sorting/accented-letters-sort-after-z.json +0 -63
- package/src/exhibits/sorting/byte-order-caps-before-lowercase.json +0 -66
- package/src/exhibits/sorting/case-insensitive-sort-lower-vs-casefold.json +0 -56
- package/src/exhibits/sorting/comparator-returns-boolean-js.json +0 -49
- package/src/exhibits/sorting/js-array-sort-stringifies-numbers.json +0 -68
- package/src/exhibits/sorting/locale-collation-de-vs-sv.json +0 -54
- package/src/exhibits/sorting/localecompare-numeric-option.json +0 -50
- package/src/exhibits/sorting/nan-in-sort.json +0 -59
- package/src/exhibits/sorting/natural-sort-breaks-lexicographic.json +0 -63
- package/src/exhibits/sorting/postgres-collation-documented.json +0 -61
- package/src/exhibits/sorting/python-cannot-compare-str-int.json +0 -70
- package/src/exhibits/sorting/reverse-sort-vs-reverse-after-sort-ties.json +0 -48
- package/src/exhibits/sorting/sorting-mixed-none-null.json +0 -56
- package/src/exhibits/sorting/typed-array-sort-is-numeric-by-default.json +0 -48
- package/src/exhibits/unicode/arabic-indic-digit-parses.json +0 -73
- package/src/exhibits/unicode/bidi-override-filename-spoof.json +0 -62
- package/src/exhibits/unicode/cyrillic-homoglyph-paypal-spoof.json +0 -67
- package/src/exhibits/unicode/fullwidth-bypasses-keyword-filter.json +0 -61
- package/src/exhibits/unicode/german-eszett-case-fold.json +0 -72
- package/src/exhibits/unicode/grapheme-vs-codepoint-length.json +0 -57
- package/src/exhibits/unicode/greek-final-sigma-casefold.json +0 -54
- package/src/exhibits/unicode/js-word-regex-ascii-only.json +0 -64
- package/src/exhibits/unicode/nfc-nfd-same-name-two-accounts.json +0 -70
- package/src/exhibits/unicode/nfkc-compatibility-folding.json +0 -63
- package/src/exhibits/unicode/truncation-orphans-combining-mark.json +0 -62
- package/src/exhibits/unicode/truncation-splits-surrogate-pair.json +0 -65
- package/src/exhibits/unicode/turkish-dotless-i-case-mapping.json +0 -72
- package/src/exhibits/unicode/zalgo-combining-mark-stacking.json +0 -57
- package/src/exhibits/unicode/zero-width-space-username.json +0 -62
- package/src/harness.mjs +0 -331
- package/src/index.ts +0 -686
- package/src/server.ts +0 -49
- package/src/verified.json +0 -977
- package/tsconfig.json +0 -18
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "csv-formula-injection-excel",
|
|
3
|
-
"title": "A CSV cell that parses as a harmless string can still execute as a formula the moment someone opens it in Excel",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "If a CSV parser reads a cell's value back as the exact literal string that was written, the value is safe wherever the file ends up.",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "name,amount\\n\"=cmd|' /C calc'!A0\",100\\n",
|
|
8
|
-
"description": "User-controlled text (a name, a comment, an address) exported into a CSV/XLSX that will be opened by a human in Excel, Google Sheets, or LibreOffice Calc - any cell value beginning with `=`, `+`, `-`, or `@` is a candidate formula to a spreadsheet application."
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "A CSV writer/reader round-trips the attacker's string perfectly and faithfully - the field comes back exactly as `=cmd|' /C calc'!A0` - so nothing in the CSV layer looks wrong. But a spreadsheet application opening that same file treats any cell starting with `=`, `+`, `-`, or `@` as a formula to evaluate, which in older Excel versions could shell out via DDE (`cmd|...!A0`) or exfiltrate data to a remote URL through functions like `IMPORTXML` in Sheets - entirely outside what CSV parsing itself can see or prevent.",
|
|
11
|
-
"why": "CSV has no concept of formulas; a cell is just delimited text. The danger is downstream: spreadsheet applications overload CSV as an import format for their own formula language, and that interpretation triggers on specific leading characters regardless of what the data \"means\" to your application. OWASP catalogs this as CSV Injection - the fix lives entirely on the writer side (how you emit values a spreadsheet will later open), because the CSV format and a conformant CSV parser are both working exactly as specified.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"python",
|
|
14
|
-
"javascript",
|
|
15
|
-
"any CSV exported for a human to open in a spreadsheet application"
|
|
16
|
-
],
|
|
17
|
-
"repros": [
|
|
18
|
-
{
|
|
19
|
-
"language": "python",
|
|
20
|
-
"code": "import csv, io\n\ndef sanitize(value):\n if isinstance(value, str) and value and value[0] in (\"=\", \"+\", \"-\", \"@\"):\n return \"'\" + value\n return value\n\npayload = '=cmd|\\' /C calc\\'!A0'\nraw = io.StringIO()\ncsv.writer(raw).writerow([\"name\", \"amount\"])\ncsv.writer(raw).writerow([payload, \"100\"])\nprint(repr(raw.getvalue()))\nparsed = list(csv.reader(io.StringIO(raw.getvalue())))\nprint(parsed[1][0] == payload)\n\nsafe = io.StringIO()\ncsv.writer(safe).writerow([sanitize(payload), \"100\"])\nprint(repr(safe.getvalue()))\n",
|
|
21
|
-
"expected_output": "\"name,amount\\r\\n=cmd|' /C calc'!A0,100\\r\\n\"\nTrue\n\"'=cmd|' /C calc'!A0,100\\r\\n\"",
|
|
22
|
-
"verified_with": "Python 3.14.2"
|
|
23
|
-
},
|
|
24
|
-
{
|
|
25
|
-
"language": "javascript",
|
|
26
|
-
"code": "function sanitize(value) {\n if (typeof value === \"string\" && value.length && \"=+-@\".includes(value[0])) return \"'\" + value;\n return value;\n}\nconst payload = \"=cmd|' /C calc'!A0\";\nconsole.log(sanitize(payload));\nconsole.log(sanitize(\"100\"));\nconsole.log(sanitize(\"Smith\"));\n",
|
|
27
|
-
"expected_output": "'=cmd|' /C calc'!A0\n100\nSmith",
|
|
28
|
-
"verified_with": "Node.js v22.22.2"
|
|
29
|
-
}
|
|
30
|
-
],
|
|
31
|
-
"fix": "Prefix any user-controlled cell value that starts with `=`, `+`, `-`, or `@` (and, in some guidance, tab or carriage return) with a single quote or another neutralizing character before writing it to a file a human will open in a spreadsheet. Parsing the CSV correctly is necessary but not sufficient - the sanitization has to happen on write, aimed at the spreadsheet application's behavior, not the CSV format's.",
|
|
32
|
-
"related": [
|
|
33
|
-
"csv-doubled-quote-escaping"
|
|
34
|
-
],
|
|
35
|
-
"sources": [
|
|
36
|
-
{
|
|
37
|
-
"title": "OWASP: CSV Injection",
|
|
38
|
-
"url": "https://owasp.org/www-community/attacks/CSV_Injection"
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
"title": "Python docs: csv - CSV File Reading and Writing",
|
|
42
|
-
"url": "https://docs.python.org/3/library/csv.html"
|
|
43
|
-
}
|
|
44
|
-
],
|
|
45
|
-
"tags": [
|
|
46
|
-
"csv",
|
|
47
|
-
"upload",
|
|
48
|
-
"parsing",
|
|
49
|
-
"security",
|
|
50
|
-
"validation"
|
|
51
|
-
],
|
|
52
|
-
"test_cases": [
|
|
53
|
-
{
|
|
54
|
-
"inputs": [
|
|
55
|
-
"name,amount\n\"=cmd|' /C calc'!A0\",100\n"
|
|
56
|
-
],
|
|
57
|
-
"relation": "returns",
|
|
58
|
-
"expected": [
|
|
59
|
-
[
|
|
60
|
-
"name",
|
|
61
|
-
"amount"
|
|
62
|
-
],
|
|
63
|
-
[
|
|
64
|
-
"=cmd|' /C calc'!A0",
|
|
65
|
-
"100"
|
|
66
|
-
]
|
|
67
|
-
],
|
|
68
|
-
"note": "A CSV-upload parser must return the cell exactly as written - the defence belongs on the export side (prefix values starting with =, +, -, @ with a single quote before writing a file a human will open in a spreadsheet), not in silently altering parsed data."
|
|
69
|
-
}
|
|
70
|
-
]
|
|
71
|
-
}
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "csv-quoted-field-with-comma",
|
|
3
|
-
"title": "line.split(\",\") turns one quoted name into two broken columns",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "Splitting a CSV line on \",\" gives you the row's fields.",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "name,amount\\n\"Smith, John\",100\\n",
|
|
8
|
-
"description": "An uploaded CSV where a free-text field - a person's name written \"Last, First\", a street address, a product description - itself contains a comma, which RFC 4180 quoting exists specifically to protect."
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "`line.split(\",\")` on the data row produces three fields - `'\"Smith'`, `' John\"'`, `'100'` - instead of the two the row actually has, shifting every later naive-split column over by one.",
|
|
11
|
-
"why": "A comma inside a quoted field is data, not a delimiter - RFC 4180 (and every real CSV dialect) says a field wrapped in double quotes can contain commas, newlines, or anything else, and the quotes are what tells a parser to stop treating that comma as a separator. `str.split(\",\")` has no concept of quoting; it is a dumb character split, so it breaks the field at the comma regardless of the surrounding quotes. A real CSV parser (Python's `csv` module) tracks quote state and only splits on commas outside quotes.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"javascript",
|
|
14
|
-
"any naive split(\",\") CSV parser"
|
|
15
|
-
],
|
|
16
|
-
"repros": [
|
|
17
|
-
{
|
|
18
|
-
"language": "python",
|
|
19
|
-
"code": "import csv, io\ntext = 'name,amount\\n\"Smith, John\",100\\n'\nnaive = [line.split(\",\") for line in text.splitlines()]\nprint(naive)\nproper = list(csv.reader(io.StringIO(text)))\nprint(proper)\n",
|
|
20
|
-
"expected_output": "[['name', 'amount'], ['\"Smith', ' John\"', '100']]\n[['name', 'amount'], ['Smith, John', '100']]",
|
|
21
|
-
"verified_with": "Python 3.14.2"
|
|
22
|
-
},
|
|
23
|
-
{
|
|
24
|
-
"language": "javascript",
|
|
25
|
-
"code": "const text = 'name,amount\\n\"Smith, John\",100\\n';\nconst naive = text.trim().split(\"\\n\").map(line => line.split(\",\"));\nconsole.log(JSON.stringify(naive));\n",
|
|
26
|
-
"expected_output": "[[\"name\",\"amount\"],[\"\\\"Smith\",\" John\\\"\",\"100\"]]",
|
|
27
|
-
"verified_with": "Node.js v22.22.2"
|
|
28
|
-
}
|
|
29
|
-
],
|
|
30
|
-
"fix": "Never parse CSV with split(\",\"). Use a real CSV parser - Python's stdlib `csv` module, or a maintained library in JS (there is no CSV parser in the JS standard library) - that understands RFC 4180 quoting.",
|
|
31
|
-
"related": [
|
|
32
|
-
"csv-quoted-field-with-newline",
|
|
33
|
-
"csv-doubled-quote-escaping"
|
|
34
|
-
],
|
|
35
|
-
"sources": [
|
|
36
|
-
{
|
|
37
|
-
"title": "RFC 4180: Common Format and MIME Type for Comma-Separated Values (CSV) Files",
|
|
38
|
-
"url": "https://www.rfc-editor.org/rfc/rfc4180"
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
"title": "Python docs: csv - CSV File Reading and Writing",
|
|
42
|
-
"url": "https://docs.python.org/3/library/csv.html"
|
|
43
|
-
}
|
|
44
|
-
],
|
|
45
|
-
"tags": [
|
|
46
|
-
"csv",
|
|
47
|
-
"upload",
|
|
48
|
-
"parsing",
|
|
49
|
-
"validation"
|
|
50
|
-
],
|
|
51
|
-
"test_cases": [
|
|
52
|
-
{
|
|
53
|
-
"inputs": [
|
|
54
|
-
"name,amount\n\"Smith, John\",100\n"
|
|
55
|
-
],
|
|
56
|
-
"relation": "returns",
|
|
57
|
-
"expected": [
|
|
58
|
-
[
|
|
59
|
-
"name",
|
|
60
|
-
"amount"
|
|
61
|
-
],
|
|
62
|
-
[
|
|
63
|
-
"Smith, John",
|
|
64
|
-
"100"
|
|
65
|
-
]
|
|
66
|
-
],
|
|
67
|
-
"note": "A CSV-parsing function given this text must return two rows of two fields each, with the comma inside the quoted name preserved as part of that field's value."
|
|
68
|
-
}
|
|
69
|
-
]
|
|
70
|
-
}
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "csv-quoted-field-with-newline",
|
|
3
|
-
"title": "Splitting a CSV file into lines first breaks a row that has a quoted newline in it",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "Each line of a CSV file is one row, so splitting the file on newlines before parsing fields is safe.",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "name,note\\n\"Smith\",\"Line1\\nLine2\"\\n",
|
|
8
|
-
"description": "A CSV export of a notes or address field that contains an embedded newline (a multi-line note, a two-line mailing address) wrapped in quotes, as RFC 4180 requires."
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "Splitting the raw text into lines first (`text.splitlines()` in Python, `text.split(\"\\n\")` in JS) turns one logical two-field row into three separate \"lines\" - the quoted field's embedded newline is treated as a row boundary before the quoting is ever examined, so no amount of comma-aware parsing afterward can put the row back together.",
|
|
11
|
-
"why": "RFC 4180 allows a quoted field to contain literal newline characters as part of its value - a row boundary is only the newline that appears outside a quoted field. A real CSV parser (Python's `csv` module) reads character by character and tracks whether it is currently inside a quote, so it does not treat a newline as a row separator while inside quotes. Pre-splitting the text into lines with `splitlines()`/`split(\"\\n\")` throws that context away before the parser ever runs, so the damage is already done.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"javascript",
|
|
14
|
-
"any CSV handling that splits into lines before parsing fields"
|
|
15
|
-
],
|
|
16
|
-
"repros": [
|
|
17
|
-
{
|
|
18
|
-
"language": "python",
|
|
19
|
-
"code": "import csv, io\ntext = 'name,note\\n\"Smith\",\"Line1\\nLine2\"\\n'\nprint(text.splitlines())\nnaive = [line.split(\",\") for line in text.splitlines()]\nprint(naive)\nproper = list(csv.reader(io.StringIO(text)))\nprint(proper)\n",
|
|
20
|
-
"expected_output": "['name,note', '\"Smith\",\"Line1', 'Line2\"']\n[['name', 'note'], ['\"Smith\"', '\"Line1'], ['Line2\"']]\n[['name', 'note'], ['Smith', 'Line1\\nLine2']]",
|
|
21
|
-
"verified_with": "Python 3.14.2"
|
|
22
|
-
},
|
|
23
|
-
{
|
|
24
|
-
"language": "javascript",
|
|
25
|
-
"code": "const text = 'name,note\\n\"Smith\",\"Line1\\nLine2\"\\n';\nconst naiveLines = text.split(\"\\n\");\nconsole.log(JSON.stringify(naiveLines));\n",
|
|
26
|
-
"expected_output": "[\"name,note\",\"\\\"Smith\\\",\\\"Line1\",\"Line2\\\"\",\"\"]",
|
|
27
|
-
"verified_with": "Node.js v22.22.2"
|
|
28
|
-
}
|
|
29
|
-
],
|
|
30
|
-
"fix": "Never split a CSV file into lines before parsing fields. Hand the whole text (or a stream) to a real quote-aware CSV parser - Python's `csv.reader`, or a maintained CSV library in JS - and let it decide where rows actually end.",
|
|
31
|
-
"related": [
|
|
32
|
-
"csv-quoted-field-with-comma",
|
|
33
|
-
"csv-doubled-quote-escaping"
|
|
34
|
-
],
|
|
35
|
-
"sources": [
|
|
36
|
-
{
|
|
37
|
-
"title": "RFC 4180: Common Format and MIME Type for Comma-Separated Values (CSV) Files",
|
|
38
|
-
"url": "https://www.rfc-editor.org/rfc/rfc4180"
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
"title": "Python docs: csv - CSV File Reading and Writing",
|
|
42
|
-
"url": "https://docs.python.org/3/library/csv.html"
|
|
43
|
-
}
|
|
44
|
-
],
|
|
45
|
-
"tags": [
|
|
46
|
-
"csv",
|
|
47
|
-
"upload",
|
|
48
|
-
"parsing",
|
|
49
|
-
"validation"
|
|
50
|
-
],
|
|
51
|
-
"test_cases": [
|
|
52
|
-
{
|
|
53
|
-
"inputs": [
|
|
54
|
-
"name,note\n\"Smith\",\"Line1\nLine2\"\n"
|
|
55
|
-
],
|
|
56
|
-
"relation": "returns",
|
|
57
|
-
"expected": [
|
|
58
|
-
[
|
|
59
|
-
"name",
|
|
60
|
-
"note"
|
|
61
|
-
],
|
|
62
|
-
[
|
|
63
|
-
"Smith",
|
|
64
|
-
"Line1\nLine2"
|
|
65
|
-
]
|
|
66
|
-
],
|
|
67
|
-
"note": "A CSV-parsing function given this text must return two rows, with the embedded newline preserved inside the second field's value rather than splitting into three rows."
|
|
68
|
-
}
|
|
69
|
-
]
|
|
70
|
-
}
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "csv-semicolon-delimited-european-excel",
|
|
3
|
-
"title": "A \"CSV\" from a European Excel is delimited with semicolons, and a comma-delimiter parser reads the whole line as one field",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "A file with a .csv extension is comma-delimited.",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "name;age\\nAda;30\\n",
|
|
8
|
-
"description": "A CSV exported by Excel configured for a locale (most of continental Europe) where the decimal comma is the default decimal separator - Excel then uses a semicolon as the list/field separator instead of a comma, to avoid colliding with decimal commas in numeric fields, while still naming the file .csv."
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "Parsing this file with a comma-delimited reader produces one single-column row per line - `['name;age']`, `['Ada;30']` - instead of the two columns the file actually has; every downstream column lookup by index or name silently gets the wrong (and only) field.",
|
|
11
|
-
"why": "\".csv\" names a family of delimited-text formats, not a guarantee of which character is the delimiter. When Excel on Windows saves as CSV it uses the operating system's regional \"list separator\" setting, and in locales whose decimal separator is a comma (most of continental Europe) that setting defaults to a semicolon, so numbers like 3,5 don't collide with the field separator. The file still gets the same \".csv\" extension. A parser that hardcodes `delimiter=\",\"` has no way to detect this from the extension alone.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"python",
|
|
14
|
-
"javascript"
|
|
15
|
-
],
|
|
16
|
-
"repros": [
|
|
17
|
-
{
|
|
18
|
-
"language": "python",
|
|
19
|
-
"code": "import csv, io\ntext = 'name;age\\nAda;30\\n'\nprint(list(csv.reader(io.StringIO(text))))\nprint(list(csv.reader(io.StringIO(text), delimiter=\";\")))\n",
|
|
20
|
-
"expected_output": "[['name;age'], ['Ada;30']]\n[['name', 'age'], ['Ada', '30']]",
|
|
21
|
-
"verified_with": "Python 3.14.2"
|
|
22
|
-
},
|
|
23
|
-
{
|
|
24
|
-
"language": "javascript",
|
|
25
|
-
"code": "const text = \"name;age\\nAda;30\\n\";\nconst commaDelimited = text.trim().split(\"\\n\").map(line => line.split(\",\"));\nconst semicolonDelimited = text.trim().split(\"\\n\").map(line => line.split(\";\"));\nconsole.log(JSON.stringify(commaDelimited));\nconsole.log(JSON.stringify(semicolonDelimited));\n",
|
|
26
|
-
"expected_output": "[[\"name;age\"],[\"Ada;30\"]]\n[[\"name\",\"age\"],[\"Ada\",\"30\"]]",
|
|
27
|
-
"verified_with": "Node.js v22.22.2"
|
|
28
|
-
}
|
|
29
|
-
],
|
|
30
|
-
"fix": "Sniff the delimiter instead of hardcoding it - Python's `csv.Sniffer().sniff(sample)` guesses it from a sample of the file, or check the first line for the most frequent of `,`, `;`, and `\\t` before choosing a reader. Let users confirm the detected delimiter on upload rather than failing silently.",
|
|
31
|
-
"related": [
|
|
32
|
-
"csv-utf8-bom-first-header"
|
|
33
|
-
],
|
|
34
|
-
"sources": [
|
|
35
|
-
{
|
|
36
|
-
"title": "Python docs: csv.Sniffer",
|
|
37
|
-
"url": "https://docs.python.org/3/library/csv.html#csv.Sniffer"
|
|
38
|
-
},
|
|
39
|
-
{
|
|
40
|
-
"title": "RFC 4180: Common Format and MIME Type for Comma-Separated Values (CSV) Files",
|
|
41
|
-
"url": "https://www.rfc-editor.org/rfc/rfc4180"
|
|
42
|
-
}
|
|
43
|
-
],
|
|
44
|
-
"tags": [
|
|
45
|
-
"csv",
|
|
46
|
-
"upload",
|
|
47
|
-
"parsing",
|
|
48
|
-
"i18n",
|
|
49
|
-
"validation"
|
|
50
|
-
],
|
|
51
|
-
"test_cases": [
|
|
52
|
-
{
|
|
53
|
-
"inputs": [
|
|
54
|
-
"name;age\nAda;30\n"
|
|
55
|
-
],
|
|
56
|
-
"relation": "returns",
|
|
57
|
-
"expected": [
|
|
58
|
-
[
|
|
59
|
-
"name",
|
|
60
|
-
"age"
|
|
61
|
-
],
|
|
62
|
-
[
|
|
63
|
-
"Ada",
|
|
64
|
-
"30"
|
|
65
|
-
]
|
|
66
|
-
],
|
|
67
|
-
"note": "A CSV-parsing function that auto-detects the delimiter should return two columns for this semicolon-delimited file, not one column containing the whole line."
|
|
68
|
-
}
|
|
69
|
-
]
|
|
70
|
-
}
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "csv-utf8-bom-first-header",
|
|
3
|
-
"title": "A CSV header named \"name\" comes back as \"\\ufeffname\" and every lookup by column name fails",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "Decoding an uploaded CSV file as UTF-8 and reading its first header exactly recovers the column name the file's first line shows on screen.",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "bytes: EF BB BF 6e 61 6d 65 2c 61 67 65 0a 41 64 61 2c 33 30 0a (\"\\ufeffname,age\\nAda,30\\n\")",
|
|
8
|
-
"description": "A CSV saved from Excel with the \"CSV UTF-8 (Comma delimited)\" format (and by many other Windows tools), which writes a UTF-8 byte-order mark at the very start of the file - invisible in a spreadsheet and in most text editors, but present as real bytes. (Excel's plain \"CSV (Comma delimited)\" format writes no BOM, and uses the system code page rather than UTF-8.)"
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "Decoding the file's bytes as plain `\"utf-8\"` leaves the BOM as a literal U+FEFF character glued onto the first header, so the first column parses as `\"\\ufeffname\"` instead of `\"name\"` - a lookup like `row[\"name\"]` throws a KeyError even though the column is clearly named \"name\" in every spreadsheet viewer.",
|
|
11
|
-
"why": "A UTF-8 BOM is three specific bytes (EF BB BF) that some encoders prepend to mark a file as UTF-8; it decodes to the single code point U+FEFF. Decoding with Python's `\"utf-8\"` codec or Node's `Buffer.toString(\"utf8\")` passes that code point through unchanged as the first character of the text. Python's `\"utf-8-sig\"` codec, and JavaScript's `TextDecoder(\"utf-8\")` (whose spec-default `ignoreBOM: false` means the BOM *is* processed/stripped, despite the confusing name), both detect and strip a leading BOM - but only if you use them instead of the plain decoder.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"python",
|
|
14
|
-
"javascript"
|
|
15
|
-
],
|
|
16
|
-
"repros": [
|
|
17
|
-
{
|
|
18
|
-
"language": "python",
|
|
19
|
-
"code": "raw = '\\ufeffname,age\\nAda,30\\n'.encode(\"utf-8\")\nimport csv, io\nplain = raw.decode(\"utf-8\")\nsig = raw.decode(\"utf-8-sig\")\nprint(list(csv.reader(io.StringIO(plain)))[0])\nprint(list(csv.reader(io.StringIO(sig)))[0])\n",
|
|
20
|
-
"expected_output": "['\\ufeffname', 'age']\n['name', 'age']",
|
|
21
|
-
"verified_with": "Python 3.14.2"
|
|
22
|
-
},
|
|
23
|
-
{
|
|
24
|
-
"language": "javascript",
|
|
25
|
-
"code": "const bytes = Buffer.from(\"\\ufeffname,age\\nAda,30\\n\", \"utf8\");\nconst plain = bytes.toString(\"utf8\");\nconst stripped = new TextDecoder(\"utf-8\").decode(bytes);\nconsole.log(JSON.stringify(plain.split(\",\")[0]));\nconsole.log(JSON.stringify(stripped.split(\",\")[0]));\n",
|
|
26
|
-
"expected_output": "\"name\"\n\"name\"",
|
|
27
|
-
"verified_with": "Node.js v22.22.2"
|
|
28
|
-
}
|
|
29
|
-
],
|
|
30
|
-
"fix": "Decode uploaded CSV/text files with `encoding=\"utf-8-sig\"` in Python (it strips a BOM if present and is a no-op if absent), or use `TextDecoder(\"utf-8\")` (not a raw `Buffer.toString`) in Node, which strips the BOM by default. Alternatively, explicitly strip a leading `\\ufeff` from the decoded text before parsing.",
|
|
31
|
-
"related": [
|
|
32
|
-
"csv-crlf-trailing-blank-line"
|
|
33
|
-
],
|
|
34
|
-
"sources": [
|
|
35
|
-
{
|
|
36
|
-
"title": "Python docs: codecs - encodings and Unicode (utf-8-sig)",
|
|
37
|
-
"url": "https://docs.python.org/3/library/codecs.html#encodings-and-unicode"
|
|
38
|
-
},
|
|
39
|
-
{
|
|
40
|
-
"title": "WHATWG Encoding Standard: UTF-8 decoder (BOM handling)",
|
|
41
|
-
"url": "https://encoding.spec.whatwg.org/#utf-8-decoder"
|
|
42
|
-
},
|
|
43
|
-
{
|
|
44
|
-
"title": "MDN: TextDecoder() constructor (ignoreBOM option)",
|
|
45
|
-
"url": "https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/TextDecoder"
|
|
46
|
-
}
|
|
47
|
-
],
|
|
48
|
-
"tags": [
|
|
49
|
-
"csv",
|
|
50
|
-
"upload",
|
|
51
|
-
"parsing",
|
|
52
|
-
"validation",
|
|
53
|
-
"i18n"
|
|
54
|
-
],
|
|
55
|
-
"test_cases": [
|
|
56
|
-
{
|
|
57
|
-
"inputs": [
|
|
58
|
-
"name,age\nAda,30\n"
|
|
59
|
-
],
|
|
60
|
-
"relation": "returns",
|
|
61
|
-
"expected": [
|
|
62
|
-
[
|
|
63
|
-
"name",
|
|
64
|
-
"age"
|
|
65
|
-
],
|
|
66
|
-
[
|
|
67
|
-
"Ada",
|
|
68
|
-
"30"
|
|
69
|
-
]
|
|
70
|
-
],
|
|
71
|
-
"note": "A CSV-parsing function given text starting with a BOM character should strip it, so the header reads exactly \"name\" rather than \"\\ufeffname\"."
|
|
72
|
-
}
|
|
73
|
-
]
|
|
74
|
-
}
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "email-regex-rejects-valid-apostrophe-address",
|
|
3
|
-
"title": "A common signup-form email regex rejects o'brien@example.com but accepts user+tag@example.com",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "A regex like ^[\\w.+-]+@[\\w-]+\\.[a-zA-Z]{2,}$ is a reasonable, slightly-strict stand-in for \"is this a valid email address\".",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "o'brien@example.com",
|
|
8
|
-
"description": "A signup, checkout, or newsletter form's email field, validated client- or server-side with a hand-written regex, used by a person whose email address's local part (the part before @) contains a punctuation character the regex author didn't anticipate - most commonly an apostrophe in a surname."
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "`/^[\\w.+-]+@[\\w-]+\\.[a-zA-Z]{2,}$/` rejects `o'brien@example.com` (valid, real-world mail address) while accepting `user+tag@example.com` (plus-addressing, also valid) - the regex's author remembered to allow `+` for Gmail-style tagging but not `'`, so a real customer's real address bounces at the form while a less common but equally valid syntax sails through.",
|
|
11
|
-
"why": "RFC 5322 section 3.2.3 defines the characters allowed in an unquoted local-part (`atext`) as letters, digits and `! # $ % & ' * + - / = ? ^ _ ` { | } ~`, so the apostrophe is exactly as legal as `+` (a dot is also allowed, but only between such characters). A regex built from `\\w` (letters, digits, underscore) plus a hand-picked handful of extras will inevitably miss some of that list, and the apostrophe is a costly one to miss because it appears in real surnames such as O'Brien and D'Angelo, so the regex rejects real customers rather than malformed input.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"python",
|
|
14
|
-
"javascript",
|
|
15
|
-
"any email validator built from a hand-picked character class instead of RFC 5322's atext"
|
|
16
|
-
],
|
|
17
|
-
"repros": [
|
|
18
|
-
{
|
|
19
|
-
"language": "javascript",
|
|
20
|
-
"code": "const common = /^[\\w.+-]+@[\\w-]+\\.[a-zA-Z]{2,}$/;\nconsole.log(common.test(\"obrien@example.com\"));\nconsole.log(common.test(\"o'brien@example.com\"));\nconsole.log(common.test(\"user+tag@example.com\"));\n",
|
|
21
|
-
"expected_output": "true\nfalse\ntrue",
|
|
22
|
-
"verified_with": "Node.js v22.22.2"
|
|
23
|
-
},
|
|
24
|
-
{
|
|
25
|
-
"language": "python",
|
|
26
|
-
"code": "import re\ncommon = re.compile(r\"^[\\w.+-]+@[\\w-]+\\.[a-zA-Z]{2,}$\")\nprint(bool(common.match(\"obrien@example.com\")))\nprint(bool(common.match(\"o'brien@example.com\")))\nprint(bool(common.match(\"user+tag@example.com\")))\n",
|
|
27
|
-
"expected_output": "True\nFalse\nTrue",
|
|
28
|
-
"verified_with": "Python 3.14.2"
|
|
29
|
-
}
|
|
30
|
-
],
|
|
31
|
-
"fix": "Don't hand-roll an email-syntax regex. Validate only loosely for form UX (e.g. \"contains exactly one @, with non-whitespace on both sides\") and rely on the actual downstream proof that matters - whether sending a confirmation email to the address succeeds - rather than a regex meant to approximate RFC 5322's full grammar.",
|
|
32
|
-
"related": [],
|
|
33
|
-
"sources": [
|
|
34
|
-
{
|
|
35
|
-
"title": "RFC 5322: Internet Message Format, Section 3.2.3 (Atom / atext)",
|
|
36
|
-
"url": "https://www.rfc-editor.org/rfc/rfc5322#section-3.2.3"
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
"title": "RFC 5321: Simple Mail Transfer Protocol, Section 4.1.2 (Mailbox local-part)",
|
|
40
|
-
"url": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2"
|
|
41
|
-
}
|
|
42
|
-
],
|
|
43
|
-
"tags": [
|
|
44
|
-
"email",
|
|
45
|
-
"validation",
|
|
46
|
-
"forms",
|
|
47
|
-
"parsing",
|
|
48
|
-
"names"
|
|
49
|
-
],
|
|
50
|
-
"test_cases": [
|
|
51
|
-
{
|
|
52
|
-
"inputs": [
|
|
53
|
-
"o'brien@example.com"
|
|
54
|
-
],
|
|
55
|
-
"relation": "returns",
|
|
56
|
-
"expected": true,
|
|
57
|
-
"note": "their_is_valid_email(address) should accept this RFC-5322-legal address rather than rejecting it for containing an apostrophe."
|
|
58
|
-
},
|
|
59
|
-
{
|
|
60
|
-
"inputs": [
|
|
61
|
-
"user+tag@example.com"
|
|
62
|
-
],
|
|
63
|
-
"relation": "returns",
|
|
64
|
-
"expected": true,
|
|
65
|
-
"note": "Plus-addressing is also legal atext and should be accepted."
|
|
66
|
-
}
|
|
67
|
-
]
|
|
68
|
-
}
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "json-big-integer-precision-loss",
|
|
3
|
-
"title": "JSON.parse silently changes a 17-digit integer ID to a different number",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "Parsing a JSON number that looks like an integer gives back that exact integer, as long as the JSON text itself is well-formed.",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "{\"id\": 9007199254740993}",
|
|
8
|
-
"description": "A large numeric ID - a Snowflake-style ID, a 64-bit database primary key, a big serial number - serialized as a JSON number (not a string) by a backend and consumed by a JavaScript client."
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "`JSON.parse('{\"id\": 9007199254740993}').id` returns `9007199254740992` - one less than the actual ID - and re-serializing it with `JSON.stringify` writes back the wrong, rounded value. Python's `json.loads` on the identical text keeps the exact integer.",
|
|
11
|
-
"why": "JavaScript's JSON implementation stores every number as the same IEEE-754 double that all JS Numbers use, and doubles can only represent integers exactly up to 2**53 (9,007,199,254,740,992); 9007199254740993 is one past that boundary and rounds to the nearest representable double, which happens to be the even number below it. Python's `json` module parses an integer-looking JSON number into a native arbitrary-precision `int`, which has no such ceiling, so it round-trips exactly.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"javascript"
|
|
14
|
-
],
|
|
15
|
-
"repros": [
|
|
16
|
-
{
|
|
17
|
-
"language": "javascript",
|
|
18
|
-
"code": "const o = JSON.parse('{\"id\": 9007199254740993}');\nconsole.log(o.id);\nconsole.log(o.id === 9007199254740993);\nconsole.log(JSON.stringify(o));\n",
|
|
19
|
-
"expected_output": "9007199254740992\ntrue\n{\"id\":9007199254740992}",
|
|
20
|
-
"verified_with": "Node.js v22.22.2"
|
|
21
|
-
},
|
|
22
|
-
{
|
|
23
|
-
"language": "python",
|
|
24
|
-
"code": "import json\no = json.loads('{\"id\": 9007199254740993}')\nprint(o[\"id\"])\nprint(o[\"id\"] == 9007199254740993)\n",
|
|
25
|
-
"expected_output": "9007199254740993\nTrue",
|
|
26
|
-
"verified_with": "Python 3.14.2"
|
|
27
|
-
}
|
|
28
|
-
],
|
|
29
|
-
"fix": "Serialize any integer that might exceed 2**53 as a JSON string, not a JSON number, and parse it on the JS side with `BigInt(str)` rather than `JSON.parse`'s default Number conversion. Many JSON APIs for 64-bit IDs (e.g. Twitter/X's `id_str`) already do this for exactly this reason.",
|
|
30
|
-
"related": [
|
|
31
|
-
"max-safe-integer-plus-two"
|
|
32
|
-
],
|
|
33
|
-
"sources": [
|
|
34
|
-
{
|
|
35
|
-
"title": "MDN: Number.MAX_SAFE_INTEGER",
|
|
36
|
-
"url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER"
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
"title": "MDN: JSON.parse()",
|
|
40
|
-
"url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse"
|
|
41
|
-
},
|
|
42
|
-
{
|
|
43
|
-
"title": "Python docs: json - JSON encoder and decoder",
|
|
44
|
-
"url": "https://docs.python.org/3/library/json.html"
|
|
45
|
-
}
|
|
46
|
-
],
|
|
47
|
-
"tags": [
|
|
48
|
-
"json",
|
|
49
|
-
"parsing",
|
|
50
|
-
"id",
|
|
51
|
-
"validation"
|
|
52
|
-
],
|
|
53
|
-
"test_cases": [
|
|
54
|
-
{
|
|
55
|
-
"inputs": [
|
|
56
|
-
"{\"id\": 9007199254740993}"
|
|
57
|
-
],
|
|
58
|
-
"relation": "returns",
|
|
59
|
-
"expected": {
|
|
60
|
-
"id": 9007199254740993
|
|
61
|
-
},
|
|
62
|
-
"note": "their_parse(json_text) must preserve this exact integer id - if it goes through JS's Number type anywhere in the pipeline, it will silently become 9007199254740992."
|
|
63
|
-
}
|
|
64
|
-
]
|
|
65
|
-
}
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "json-duplicate-keys-last-wins",
|
|
3
|
-
"title": "A JSON object with the same key twice parses fine - and silently keeps only the last one",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "A JSON parser rejects (or at least warns about) an object literal that repeats a key, the way a careful reviewer would.",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "{\"a\":1,\"a\":2}",
|
|
8
|
-
"description": "A JSON payload assembled by string concatenation, a buggy serializer, or a merge of two partial objects that happens to repeat a key - e.g. a webhook payload built from two code paths that both set `\"status\"`."
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "Both Python's `json.loads` and JavaScript's `JSON.parse` accept `{\"a\":1,\"a\":2}` without error and return `{\"a\": 2}` - the first value is discarded with no warning, exception, or trace that the input ever had a conflict.",
|
|
11
|
-
"why": "The JSON grammar (RFC 8259 section 4) does not forbid duplicate names in an object: names SHOULD be unique, and when they aren't, the RFC says the behaviour of receiving software is unpredictable. The two languages then pin it down themselves. ECMA-262 specifies that `JSON.parse` overwrites a lexically earlier value with a later one for the same key, so last-wins is guaranteed in every conforming JavaScript engine. Python's `json` module documents the same choice: it does not raise and keeps only the last name/value pair. Other parsers in other languages may keep the first value or reject the document, which is how two services can read different values from the same payload.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"python",
|
|
14
|
-
"javascript"
|
|
15
|
-
],
|
|
16
|
-
"repros": [
|
|
17
|
-
{
|
|
18
|
-
"language": "python",
|
|
19
|
-
"code": "import json\nprint(json.loads('{\"a\":1,\"a\":2}'))\n",
|
|
20
|
-
"expected_output": "{'a': 2}",
|
|
21
|
-
"verified_with": "Python 3.14.2"
|
|
22
|
-
},
|
|
23
|
-
{
|
|
24
|
-
"language": "javascript",
|
|
25
|
-
"code": "console.log(JSON.parse('{\"a\":1,\"a\":2}'));\n",
|
|
26
|
-
"expected_output": "{ a: 2 }",
|
|
27
|
-
"verified_with": "Node.js v22.22.2"
|
|
28
|
-
}
|
|
29
|
-
],
|
|
30
|
-
"fix": "Detect duplicates explicitly where they matter. In Python, pass `object_pairs_hook` to `json.loads` and raise if a key repeats; in JavaScript, `JSON.parse` cannot report duplicates, so use a streaming/tokenizing parser that can, or reject them at the producer. Don't assume a successfully-parsed object had exactly the keys the sender wrote.",
|
|
31
|
-
"related": [],
|
|
32
|
-
"sources": [
|
|
33
|
-
{
|
|
34
|
-
"title": "RFC 8259: The JavaScript Object Notation (JSON) Data Interchange Format, Section 4",
|
|
35
|
-
"url": "https://www.rfc-editor.org/rfc/rfc8259#section-4"
|
|
36
|
-
},
|
|
37
|
-
{
|
|
38
|
-
"title": "Python docs: json - JSON encoder and decoder",
|
|
39
|
-
"url": "https://docs.python.org/3/library/json.html"
|
|
40
|
-
},
|
|
41
|
-
{
|
|
42
|
-
"title": "ECMA-262: JSON.parse",
|
|
43
|
-
"url": "https://tc39.es/ecma262/#sec-json.parse"
|
|
44
|
-
}
|
|
45
|
-
],
|
|
46
|
-
"tags": [
|
|
47
|
-
"json",
|
|
48
|
-
"parsing",
|
|
49
|
-
"validation",
|
|
50
|
-
"security"
|
|
51
|
-
],
|
|
52
|
-
"test_cases": [
|
|
53
|
-
{
|
|
54
|
-
"inputs": [
|
|
55
|
-
"{\"a\":1,\"a\":2}"
|
|
56
|
-
],
|
|
57
|
-
"relation": "returns",
|
|
58
|
-
"expected": {
|
|
59
|
-
"a": 2
|
|
60
|
-
},
|
|
61
|
-
"note": "their_parse(json_text) on an object with a repeated key returns the last value under standard parsers - if two upstream systems disagree about which value should win, catch this before trusting the parsed result."
|
|
62
|
-
}
|
|
63
|
-
]
|
|
64
|
-
}
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "json-lone-surrogate-roundtrip",
|
|
3
|
-
"title": "JSON.parse happily accepts a string with an unpaired surrogate - and converting it to UTF-8 corrupts or crashes later",
|
|
4
|
-
"category": "parsing",
|
|
5
|
-
"assumption": "If JSON.parse (or json.loads) accepts a string without error, that string is well-formed Unicode text that any later encoding step will handle safely.",
|
|
6
|
-
"input": {
|
|
7
|
-
"display": "\"\\ud800\" (a lone UTF-16 high surrogate, with no matching low surrogate)",
|
|
8
|
-
"description": "A string field corrupted upstream - a truncated emoji or CJK character cut exactly between its two surrogate halves, or data copied out of a UTF-16 source (e.g. a Windows filename or a Java/C# string) that already contained an unpaired surrogate - serialized into a JSON payload via a \\u escape."
|
|
9
|
-
},
|
|
10
|
-
"what_goes_wrong": "Both `JSON.parse` and Python's `json.loads` accept the escape `\\ud800` and hand back a string containing that single unpaired surrogate, with no error. The corruption surfaces later and differently in each language: encoding that Python string to UTF-8 raises `UnicodeEncodeError`, while in Node, `Buffer.from(s, \"utf8\")` silently replaces the lone surrogate with the U+FFFD replacement character - no exception, no warning, just a quietly different byte sequence than the one sent.",
|
|
11
|
-
"why": "JSON strings are defined over Unicode code points via `\\uXXXX` escapes, and RFC 8259 section 8.2 explicitly notes that the grammar allows an escape for a lone surrogate, leaving the behaviour of receiving software unpredictable - a parser that checks syntax is not required to check that the resulting text is well-formed Unicode. JavaScript strings are UTF-16 code unit sequences and can hold an unpaired surrogate without complaint, so `JSON.parse`/`JSON.stringify` round-trip it fine as a JS value. Python `str` objects are sequences of Unicode code points (including surrogates, which Python represents but which are not valid scalar values), so `json.loads` accepts the escape too - the failure only appears once something tries to encode that code point as actual UTF-8 bytes, because UTF-8 has no legal encoding for a surrogate on its own.",
|
|
12
|
-
"affects": [
|
|
13
|
-
"python",
|
|
14
|
-
"javascript"
|
|
15
|
-
],
|
|
16
|
-
"repros": [
|
|
17
|
-
{
|
|
18
|
-
"language": "python",
|
|
19
|
-
"code": "import json\ns = json.loads('\"\\\\ud800\"')\nprint(len(s), hex(ord(s)))\ntry:\n s.encode(\"utf-8\")\nexcept UnicodeEncodeError as e:\n print(\"UnicodeEncodeError:\", e)\n",
|
|
20
|
-
"expected_output": "1 0xd800\nUnicodeEncodeError: 'utf-8' codec can't encode character '\\ud800' in position 0: surrogates not allowed",
|
|
21
|
-
"verified_with": "Python 3.14.2"
|
|
22
|
-
},
|
|
23
|
-
{
|
|
24
|
-
"language": "javascript",
|
|
25
|
-
"code": "const s = JSON.parse('\"\\\\ud800\"');\nconsole.log(s.length, s.charCodeAt(0).toString(16));\nconsole.log(JSON.stringify(s));\nconst buf = Buffer.from(s, \"utf8\");\nconsole.log(buf.toString(\"hex\"), buf.length);\n",
|
|
26
|
-
"expected_output": "1 d800\n\"\\ud800\"\nefbfbd 3",
|
|
27
|
-
"verified_with": "Node.js v22.22.2"
|
|
28
|
-
}
|
|
29
|
-
],
|
|
30
|
-
"fix": "Check that decoded JSON strings are well-formed Unicode before re-encoding them: in Python, `s.encode(\"utf-8\")` (strict) raises on a lone surrogate; in JavaScript, `s.isWellFormed()` (Node 20+, ES2024) returns false for it, and `s.toWellFormed()` substitutes U+FFFD explicitly. Do this before `Buffer.from(s, \"utf8\")`, which substitutes silently.",
|
|
31
|
-
"related": [],
|
|
32
|
-
"sources": [
|
|
33
|
-
{
|
|
34
|
-
"title": "RFC 8259 section 8.2: Unicode characters (unpaired surrogates)",
|
|
35
|
-
"url": "https://www.rfc-editor.org/rfc/rfc8259#section-8.2"
|
|
36
|
-
},
|
|
37
|
-
{
|
|
38
|
-
"title": "Unicode Standard Annex #15 glossary: Unpaired Surrogate",
|
|
39
|
-
"url": "https://www.unicode.org/glossary/#unpaired_surrogate"
|
|
40
|
-
}
|
|
41
|
-
],
|
|
42
|
-
"tags": [
|
|
43
|
-
"json",
|
|
44
|
-
"parsing",
|
|
45
|
-
"validation",
|
|
46
|
-
"security",
|
|
47
|
-
"i18n"
|
|
48
|
-
],
|
|
49
|
-
"test_cases": [
|
|
50
|
-
{
|
|
51
|
-
"inputs": [
|
|
52
|
-
"\"\\ud800\""
|
|
53
|
-
],
|
|
54
|
-
"relation": "no_crash",
|
|
55
|
-
"note": "their_parse_json(text) may legitimately accept this string, but whatever encodes the result to UTF-8 downstream must handle (reject or substitute) an unpaired surrogate rather than silently corrupting it."
|
|
56
|
-
}
|
|
57
|
-
]
|
|
58
|
-
}
|