@wafertools/testdata-parser 0.11.1 → 0.13.0

package/README.md

@@ -116,8 +116,10 @@ interface CsvMapping {
   testnameCol?: string;       // for "tall" CSVs: column holding the test name per row
   testnumberCol?: string;     // for "tall" CSVs: column holding the test's real number per row
   testvalueCol?: string;      // for "tall" CSVs: column holding the test value per row
-  loLimitCol?: string;
+  loLimitCol?: string;        // for "tall" CSVs: test limits (STDF LO_LIMIT/HI_LIMIT) per row
   hiLimitCol?: string;
+  loSpecCol?: string;         // for "tall" CSVs: spec limits (STDF LO_SPEC/HI_SPEC) per row — a separate pair
+  hiSpecCol?: string;
   unitsCol?: string;
   passBins: number[];         // hbin/sbin values treated as a pass for pass/fail summary
 }
@@ -157,6 +159,7 @@ interface ParsedStdf {
 
 interface WaferData {
   waferId: string;
+  waferIdPlaceholder?: boolean; // true when the file gave the wafer no ID (waferId is W1, W2…)
   results: DieResult[];
   partCount?: number;
   goodCount?: number;
@@ -171,7 +174,8 @@ interface DieResult {
   hbin?: number;
   sbin?: number;
   siteNum?: number;
-  partId?: number;
+  partId?: string;     // STDF/ATDF PART_ID as text; data, not an identifier
+  supersedes?: 'partId' | 'position'; // the tester marked this record as replacing an earlier one
   testValues?: Record<string, number>;  // keyed by test number as a string
   testPass?: Record<string, boolean>;   // recorded verdicts, true = pass; see below
 }
@@ -181,6 +185,10 @@ interface TestDef {
   testType: string; // "P" (parametric) or "F" (functional)
   loLimit?: number;
   hiLimit?: number;
+  loSpec?: number;     // specification limits (STDF LO_SPEC/HI_SPEC), separate from the test limits
+  hiSpec?: number;
+  loLimitInclusive?: boolean; // false: a result equal to the low test limit fails (STDF PARM_FLG bit 6 clear, ATDF "L")
+  hiLimitInclusive?: boolean; // the same for the high test limit (PARM_FLG bit 7, ATDF "H"); absent = equal passes
   units?: string;
   order?: number; // display order, independent of the key — see below
 }
@@ -212,7 +220,14 @@ interface ParserWarning {
 
 type ParserWarningCode =
   | 'unpositioned-dies'         // dies with no X/Y — real data, not placeable
-  | 'soft-bin-mirrored'         // sbin was the 65535 sentinel; hbin mirrored in
+  | 'bin-invalid'               // a bin outside STDF's 0–32767, or a missing hard bin
+  | 'coordinate-invalid'        // an X/Y outside STDF's -32767..32767
+  | 'site-invalid'              // a site number outside STDF's 0–255 (flat formats)
+  | 'result-unusable'           // results the tester flagged unusable (value left out, verdict kept)
+  | 'records-not-read'          // records this parser does not read yet (MPR)
+  | 'wafer-end-missing'         // a wafer had no WRR; closed at the next wafer or end of file
+  | 'file-truncated'            // the file ends part-way through a record
+  | 'record-malformed'          // a PRR too short to hold its required fields
   | 'values-not-numeric'        // a mapped column held values that would not coerce
   | 'retests-assumed'           // repeated positions read as retests
   | 'wafer-split-by-column'     // one wafer per value of a mapped column
@@ -250,9 +265,10 @@ an empty array, which asserts that nothing passes.
 **`warnings` carries a stable `code`, prose, and a severity** — branch on the code, display
 the message, and never match on the prose. `severity: 'error'` means a number or a plot
 built from this result can mislead, because data was dropped or a value was substituted
-(`unpositioned-dies`, `soft-bin-mirrored`, `values-not-numeric`); `'warning'` means the
-parse made a documented interpretation you may want to change, and nothing was altered or
-lost. Nothing here is fatal — the parse succeeded. Surface them: a silently discarded
+(`unpositioned-dies`, `bin-invalid`, `coordinate-invalid`, `site-invalid`, `record-malformed`, `records-not-read`, `values-not-numeric`); `'warning'` means the
+parse applied a documented rule or interpretation — one you may want to change, or, like
+`result-unusable`, the spec's own rule for leaving out values the tester flagged — and the
+result means what the file says. Nothing here is fatal — the parse succeeded. Surface them: a silently discarded
 warning is how a partly-wrong load looks fine.
 
 `testDefs` and `testValues` are both keyed by **test number**, not test name — test numbers are the unique identity in STDF/ATDF; names are not guaranteed unique.
@@ -362,7 +378,16 @@ pub struct CsvHeadersResult {
 - **Byte readers are panic-free.** STDF/ATDF field readers are bounds-checked and return `Option`/`Result` rather than panicking on truncated input — a panic inside WASM aborts the whole module with no recovery, so this is a hard requirement, not a style preference.
 - **Big-endian and little-endian STDF** are both supported (detected from the FAR record's `CPU_TYPE`).
 - **Gzip is transparent** — every entry point decompresses `.gz` input automatically by sniffing the magic bytes; callers don't need to branch on compression.
-- **CSV/JSON/Parquet test numbers are a deterministic hash, not a real STDF test number.** STDF/ATDF have a real test number in the file; the other three don't, so one is synthesized — from the source column for wide format, from the test name for long format (`test_identity::stable_test_number`, FNV-1a with a fixed seed and a reserved floor, collision-probed so two tests in one file can never collide). Deliberately not sequential/encounter-order: a hash means the number for a given test doesn't change if the file is reordered or a column is added — the number is otherwise meaningless and callers should never rely on its value, only on it being stable and unique within one parse. `order` (see `TestDef` above) carries the file's own display order instead.
+- **MPR records are not read yet.** A multiple-result parametric record (STDF `MPR`, ATDF `MPR:`)
+  carries several results for one test; both parsers skip them — in the full parse and the
+  test-name scan alike — and count them in a `records-not-read` warning, so the missing tests are
+  never silent. PTR and FTR records are read in full.
+- **Every format applies STDF V4's value ranges.** A bin, coordinate or site number STDF cannot
+  store, or a test value that is not finite, is missing in CSV, JSON and Parquet exactly as in
+  STDF and ATDF, and is reported under the same codes (`bin-invalid`, `coordinate-invalid`,
+  `site-invalid`, `result-unusable`). The rule lives in one place (`SpecCheck` in `types.rs`),
+  which the flat formats reach through `flat_wafers::into_parsed`.
+- **CSV/JSON/Parquet test numbers fall back to a deterministic hash only when the file itself carries no real one.** Neither format has a *mandatory* STDF-style test number the way STDF/ATDF do, but a real one is used whenever the source data has it — see "Test identity — real number vs. synthesized one" above for the wide/tall rules. Hashing is the fallback, not the default: it fires per test only when no real number was mapped or the mapped column's value didn't parse (`test_identity::stable_test_number`, FNV-1a with a fixed seed and a reserved floor, collision-probed so two tests in one file can never collide, and never colliding with a genuine numeric-header/`testnumberCol` value either). Deliberately not sequential/encounter-order: a hash means the number for a given test doesn't change if the file is reordered or a column is added — a *hashed* number is otherwise meaningless and callers should never rely on its value, only on it being stable and unique within one parse. `order` (see `TestDef` above) carries the file's own display order instead.
 - **Parquet reads through a row-oriented API, not Arrow.** `parquet::record::Row`/`Field` rather than the `arrow` feature — a closer fit for this crate's row-based `DieResult` model, and a smaller WASM bundle (no Arrow array machinery pulled in). A typed Parquet cell is coerced to `f64` for numeric roles and to a plain string otherwise; a value that fails to coerce (e.g. a numeric role mapped to a genuinely string-typed column) is skipped and surfaced as one summarised entry in `warnings`, not a panic or a silent zero.
 - **Parquet's `zstd` codec is native-only.** `snappy`, `gzip`, `lz4`, and `brotli` build for `wasm32-unknown-unknown` with no extra toolchain; `zstd`'s C library needs a real C cross-compiler targeting wasm32, which a plain `wasm-pack build` doesn't assume is available. A `zstd`-compressed Parquet file parses natively but fails clearly on the WASM build.
 

package/llms.txt

@@ -37,7 +37,7 @@ resolve.
 - **Failures throw a `ParserError` — branch on `err.code`, display `err.message`.** It is a real `Error` (so `err.message` reads and `err instanceof Error` is true) carrying a stable code: `not-stdf`, `stdf-unsupported`, `file-read`, `gzip-invalid`, `encoding-invalid`, `csv-read`, `json-invalid`, `parquet-read`, `column-missing`, `mapping-invalid`, `internal`. Never match on the message — messages are prose and get reworded. `ParseErrorCode` is a union in the declarations, so a `switch` is exhaustively checked.
 - **The package ships real TypeScript declarations — use them.** `ParsedStdf`, `DieResult`, `TestDef`, `ParserWarning`, `ScanResult`, `FileMeta`, `ParquetHeadersResult`, `CsvMapping` and the two code unions are all in `testdata_parser.d.ts`, and every export is typed with its real return type. Do not redeclare these shapes in your own code, and do not guess field names: let the compiler tell you. (Versions before this one typed every return value as `any` — if that is what you see, the installed version is older than the declarations.)
 - **`x` and `y` are optional.** A die with no reported position has neither (never one of the two) and carries `dieIndex` instead: it still holds real measured data and counts toward non-spatial statistics, but cannot be placed on a wafer map. Filter those out before building a map; do not default them to 0, which invents a die at the origin.
-- **CSV/JSON/Parquet have no real test number, so one is made up.** In a tall layout the crate hashes the test's identity (FNV-1a, fixed seed, collision-probed); in a wide layout the caller assigns `CsvTestCol.testNumber` when building the mapping. A hashed number is stable and unique *within one parse*, and that is all — never display it to a user, persist it, or join two files on it. Hashed numbers are always >= 1,000,000, a band chosen so one can never be mistaken for a real STDF test number or a 1001-style sequential one. If the source does carry real numbers, pass them: `testnumberCol` (tall) or the column's own numeric header (wide) are used as-is rather than hashed. `TestDef.order` carries the file's own display order; STDF/ATDF omit it, because there the real test number already sorts meaningfully.
+- **CSV/JSON/Parquet only hash a test number when the source has no real one.** Neither format has a *mandatory* test number the way STDF/ATDF do, but if the source carries real numbers, use them: `testnumberCol` (tall layout) or the column's own numeric header (wide layout) are used as-is, never hashed. Hashing is strictly the fallback — in a tall layout the crate hashes the test's identity (FNV-1a, fixed seed, collision-probed) only when no real number was mapped or it failed to parse; in a wide layout the caller assigns `CsvTestCol.testNumber` when building the mapping, same rule. A *hashed* number is stable and unique *within one parse*, and that is all — never display it to a user, persist it, or join two files on it. Hashed numbers are always >= 1,000,000, a band chosen so one can never be mistaken for a real STDF test number, a 1001-style sequential one, or a genuine numeric-header/`testnumberCol` value. `TestDef.order` carries the file's own display order; STDF/ATDF omit it, because there the real test number already sorts meaningfully.
 - **Two-pass parsing loses test metadata unless you merge it back.** `testDefs` in a filtered result only covers tests seen in the second pass, so a test that appeared only on an early stop-on-fail die can vanish. Merge the first-pass `ScanResult.testDefs` into the filtered result.
 - **Empty collections are omitted, not emitted.** `warnings`, `hbinDefs`, `sbinDefs`, `passHbins`, `testValues` and `testPass` are all absent from the object when empty, so read them with `?.` and treat missing as empty.
 - **Warnings are structured, and half of them mean a number could be wrong.** Each is `{ code, message, severity }`. `severity: 'error'` means data was dropped or a value substituted, so a plot or yield built from the parse can mislead — `unpositioned-dies`, `soft-bin-mirrored`, `values-not-numeric`. `'warning'` means the parse made a documented interpretation you may want to change — `retests-assumed`, `wafer-split-by-column`, `column-varies-within-wafer`, `multiple-lot-records`. Branch on `code`, never the prose, and surface them: a discarded advisory is how a partly-wrong load looks fine.

package/package.json

@@ -2,7 +2,7 @@
   "name": "@wafertools/testdata-parser",
   "type": "module",
   "description": "Rust/WASM parsers for semiconductor test data formats (STDF, ATDF, CSV, JSON, Parquet)",
-  "version": "0.11.1",
+  "version": "0.13.0",
   "license": "MIT",
   "repository": {
     "type": "git",

package/testdata_parser.d.ts

@@ -27,9 +27,19 @@ export interface TestDef {
     name: string;
     /** "P" parametric (has a measured value) or "F" functional (verdict only). */
     testType: "P" | "F";
+    /** Test limits — the pass/fail limits (STDF LO_LIMIT/HI_LIMIT). */
     loLimit?: number;
     hiLimit?: number;
     units?: string;
+    /** Specification limits (STDF LO_SPEC/HI_SPEC) — what process capability is
+     *  judged against. Separate from the test limits. */
+    loSpec?: number;
+    hiSpec?: number;
+    /** `false` when a result equal to the low test limit fails (STDF PARM_FLG
+     *  bit 6 clear, ATDF Limit Compare `L`). Absent = it passes. */
+    loLimitInclusive?: boolean;
+    /** The same for the high test limit (STDF PARM_FLG bit 7, ATDF `H`). */
+    hiLimitInclusive?: boolean;
     /** The file's own display order. Absent for STDF/ATDF, where the real test
      *  number already sorts meaningfully. */
     order?: number;
@@ -47,7 +57,11 @@ export interface DieResult {
     hbin?: number;
     sbin?: number;
     siteNum?: number;
-    partId?: number;
+    /** STDF/ATDF PART_ID, as text. Data about the part, not an identifier. */
+    partId?: string;
+    /** The tester marked this record as replacing an earlier one: with the same
+     *  part ID (`"partId"`) or at the same X/Y (`"position"`). */
+    supersedes?: "partId" | "position";
     /** Measured values, keyed by test number as a string. Parametric tests only. */
     testValues?: Record<string, number>;
     /** Recorded pass/fail verdicts, `true` = pass, keyed like `testValues`.
@@ -58,6 +72,8 @@ export interface DieResult {
 
 export interface WaferData {
     waferId: string;
+    /** `waferId` is a placeholder (`W1`, `W2`…): the file gave this wafer no ID. */
+    waferIdPlaceholder?: boolean;
     results: DieResult[];
     partCount?: number;
     goodCount?: number;
@@ -69,7 +85,14 @@ export interface WaferData {
 /** Every advisory a parse can raise. Branch on this, never on the message. */
 export type ParserWarningCode =
 | "unpositioned-dies"
-| "soft-bin-mirrored"
+| "bin-invalid"
+| "coordinate-invalid"
+| "site-invalid"
+| "result-unusable"
+| "records-not-read"
+| "wafer-end-missing"
+| "file-truncated"
+| "record-malformed"
 | "values-not-numeric"
 | "retests-assumed"
 | "wafer-split-by-column"
@@ -171,8 +194,13 @@ export interface CsvMapping {
     testnumberCol?: string | null;
     /** Tall format: the column holding each row's measured value. */
     testvalueCol?: string | null;
+    /** Tall format: the columns holding each test's test limits (STDF LO_LIMIT/HI_LIMIT). */
     loLimitCol?: string | null;
     hiLimitCol?: string | null;
+    /** Tall format: the columns holding each test's spec limits (STDF LO_SPEC/HI_SPEC) —
+     *  a separate pair from the test limits, never mixed with them. */
+    loSpecCol?: string | null;
+    hiSpecCol?: string | null;
     unitsCol?: string | null;
     /** Bins counted as a pass in this file's own pass/fail summary. */
     passBins: number[];