edf2csv 0.5.92 → 0.5.93

package/CHANGELOG.md

@@ -3,6 +3,32 @@
 Notable changes to edf2csv. Versions follow [semantic versioning](https://semver.org); while the
 major version is 0, a minor bump may contain breaking changes.
 
+## 0.5.93
+
+### Fixed: the padding after a TAL was counted as an annotation that could not be read
+
+```
+$ edf2csv space-padded.edf --out out
+warning: 2 annotation entries were unreadable and could not be exported.
+
+$ cat out/annotations.csv
+onset_s,duration_s,description,record_index
+0.5,,Lights off,0
+```
+
+The file holds one annotation and it was exported in full. The two "entries" are the spaces
+filling the rest of each record's annotation slot.
+
+EDF+ pads that slot with `0x00`, which the decoder skips because NUL is also what ends one TAL
+and starts the next. Writers pad with spaces, and a run of spaces after the last TAL is a
+non-empty chunk that does not begin with a sign — so it took the malformed branch. Under
+`--strict` that is a failed run over the whitespace at the end of a slot, on a file that lost
+nothing.
+
+Whitespace-only chunks are padding now — space, tab, CR, LF and NUL. A chunk of anything else
+that does not parse is still counted and still reported, which is the case the warning exists
+for.
+
 ## 0.5.92
 
 ### Fixed: `--info` redirected into a full filesystem wrote nothing and exited 0

package/dist/edf/annotations.js

@@ -39,6 +39,23 @@ export function decodeRecordAnnotations(bytes, recordIndex, carriesTimekeeping =
             continue;
         if (i > start) {
             const chunk = bytes.subarray(start, i);
+            /*
+              Padding is not a lost annotation.
+      
+              The spec pads the slot with NUL, which the loop above already skips because it is what
+              separates one TAL from the next. Writers pad with spaces instead, and a run of spaces
+              after the last TAL is a non-empty chunk — so a file holding one perfectly readable
+              event, exported in full, was told "2 annotation entries were unreadable and could not
+              be exported", one per record. Nothing was lost. Under --strict that is a failed run
+              over the whitespace at the end of a slot.
+      
+              Only whitespace. A chunk of anything else that does not parse is a real loss and is
+              still counted, which is the case this warning exists for.
+            */
+            if (chunk.every(isPaddingByte)) {
+                start = i + 1;
+                continue;
+            }
             const parsed = parseTal(chunk, recordIndex);
             // The timekeeping TAL is the one in first POSITION, whether or not it decodes.
             // Clearing this flag only on a successful parse meant that an unreadable first TAL
@@ -94,6 +111,10 @@ export function decodeRecordAnnotations(bytes, recordIndex, carriesTimekeeping =
         negativeDurations,
     };
 }
+/** Space, tab, CR, LF or NUL — what a writer fills the rest of the slot with. */
+function isPaddingByte(byte) {
+    return byte === 0x20 || byte === 0x09 || byte === 0x0d || byte === 0x0a || byte === 0x00;
+}
 function parseTal(chunk, recordIndex) {
     // The onset must be explicitly signed; anything else is not a TAL.
     const first = chunk[0];

package/dist/edf/annotations.js.map

@@ -1 +1 @@
-{"version":3,"file":"annotations.js","sourceRoot":"","sources":["../../src/edf/annotations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,yDAAyD;AAChF,MAAM,YAAY,GAAG,IAAI,CAAC,CAAC,gCAAgC;AAC3D,MAAM,OAAO,GAAG,IAAI,CAAC;AAErB,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;AACpD,MAAM,iBAAiB,GAAG,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;AAuD5D;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CACrC,KAAiB,EACjB,WAAmB,EACnB,kBAAkB,GAAG,IAAI;IAEzB,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,OAAO;YAAE,SAAS;QAEzD,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAE5C,+EAA+E;YAC/E,mFAAmF;YACnF,+EAA+E;YAC/E,iFAAiF;YACjF,+EAA+E;YAC/E,uEAAuE;YACvE;;;;;;;;;cASE;YACF,MAAM,aAAa,GAAG,UAAU,IAAI,kBAAkB,CAAC;YACvD,UAAU,GAAG,KAAK,CAAC;YAEnB,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,aAAa;oBAAE,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC9C,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW;oBAAE,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC1E,mBAAmB,IAAI,MAAM,CAAC,mBAAmB,CAAC;gBAClD,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN;;;;;;;;;kBASE;gBACF,IAAI,aAAa;oBAAE,oBAAoB,EAAE,CAAC;;oBACrC,SAAS,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;QACD,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;IAED,OAAO;QACL,WAAW;QACX,WAAW;QACX,SAAS;QACT,oBAAoB;QACpB,mBAAmB;QACnB,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAWD,SAAS,QAAQ,CAAC,KAAiB,EAAE,WAAmB;IACtD,mEAAmE;IACnE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAElE,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE5B,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpD,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QACvC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC;;;;;;;;;;MAUE;IACF,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,kBAAkB,GAAG,KAAK,CAAC;IAC/B,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;QAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,QAAQ,GAAG,CAAC,CAAC;;YAChC,kBAAkB,GAAG,IAAI,CAAC;IACjC,CAAC;IAED;;;;;;;;;MASE;IACF,MAAM,gBAAgB,GAAG,QAAQ,KAAK,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC;IAE3D,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACjC,gFAAgF;QAChF,IAAI,GAAG,KAAK,EAAE;YAAE,SAAS;QACzB,WAAW,CAAC,IAAI,CAAC;YACf,KAAK;YACL,QAAQ;YACR,IAAI,EAAE,GAAG;YACT,WAAW;YACX,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;IAED,yFAAyF;IACzF,+CAA+C;IAC/C,OAAO;QACL,KAAK;QACL,WAAW;QACX,mBAAmB,EAAE,kBAAkB,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAChE,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;KAC7D,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC","sourcesContent":["/**\n * EDF+ annotation (TAL) decoding.\n *\n * The annotations channel stores UTF-8 text in place of samples. Its bytes are a\n * run of Time-stamped Annotation Lists, each terminated by a NUL, with the rest\n * of the channel NUL-padded:\n *\n *   +<onset>[<0x15><duration>]<0x14><text><0x14>...<0x00>\n *\n * The first TAL of every data record must carry that record's start time and no\n * text; that is how an EDF+D file states where each record actually sits in time.\n *\n *   +1.25<0x15>0.5<0x14>Seizure onset<0x14><0x00>\n */\n\nimport { decodeUtf8 } from './bytes.js';\n\nconst SEP_TEXT = 0x14; // separates onset/duration from text, and text from text\nconst SEP_DURATION = 0x15; // separates onset from duration\nconst TAL_END = 0x00;\n\nconst TEXT_SEP_CHAR = String.fromCharCode(SEP_TEXT);\nconst DURATION_SEP_CHAR = String.fromCharCode(SEP_DURATION);\n\nexport interface Annotation {\n  /** Seconds from the start of the recording. */\n  onset: number;\n  /**\n   * Seconds, or null when the TAL stated no duration that could be read.\n   *\n   * Null covers two cases the file distinguishes and this field does not: a TAL that omitted\n   * the duration, and a TAL that stated one which is not a number. They are told apart by\n   * `unreadableDurations`, which is what raises the warning; the value itself has nowhere\n   * honest to put \"the file said `abc`\".\n   */\n  duration: number | null;\n  text: string;\n  /** Index of the data record this annotation was stored in. */\n  recordIndex: number;\n  /**\n   * True when the file stated a duration that could not be read.\n   *\n   * `duration` is null either way, which is the ambiguity the counts beside it exist to\n   * flag — and those counts were of the whole file while `annotations.csv` is filtered to\n   * the requested window. A conversion of one second of a recording warned that \"1\n   * annotation states a duration that is not a number, so its duration_s cell is empty\"\n   * about an event two seconds outside it, and failed `--strict` for it. Carrying the fact\n   * on the event lets the count be taken where the window has already been applied.\n   */\n  durationUnreadable?: boolean;\n}\n\nexport interface DecodedRecordAnnotations {\n  /** Record start time in seconds, from the leading timekeeping TAL. */\n  recordStart: number | null;\n  annotations: Annotation[];\n  /** Non-empty chunks that were not valid TALs, so the caller can report them. */\n  malformed: number;\n  /** Unreadable TALs in first position, which carry a record's start time, not an event. */\n  malformedTimekeeping: number;\n  /**\n   * Events kept whose stated duration could not be read.\n   *\n   * Counted apart again, for the same reason the two above are: the entry was exported and\n   * nothing about it is missing except the one field, so calling it an entry that \"could not\n   * be exported\" describes a loss that did not happen and hides the one that did.\n   */\n  unreadableDurations: number;\n  /**\n   * Events kept whose stated duration is a readable number below zero.\n   *\n   * Separate from the count above because the value survives: it is written to the CSV as\n   * the file gave it, and what is wrong with it is arithmetic rather than parsing.\n   */\n  negativeDurations: number;\n}\n\n/**\n * Decode one data record's annotation bytes.\n *\n * Malformed TALs are skipped rather than thrown, because a single bad annotation\n * should not cost the user an entire conversion. The count of skipped chunks is\n * returned so the caller can tell the user rather than losing them in silence.\n */\nexport function decodeRecordAnnotations(\n  bytes: Uint8Array,\n  recordIndex: number,\n  carriesTimekeeping = true,\n): DecodedRecordAnnotations {\n  const annotations: Annotation[] = [];\n  let recordStart: number | null = null;\n  let isFirstTal = true;\n  let malformed = 0;\n  let malformedTimekeeping = 0;\n  let unreadableDurations = 0;\n  let negativeDurations = 0;\n\n  let start = 0;\n  for (let i = 0; i <= bytes.length; i++) {\n    if (i !== bytes.length && bytes[i] !== TAL_END) continue;\n\n    if (i > start) {\n      const chunk = bytes.subarray(start, i);\n      const parsed = parseTal(chunk, recordIndex);\n\n      // The timekeeping TAL is the one in first POSITION, whether or not it decodes.\n      // Clearing this flag only on a successful parse meant that an unreadable first TAL\n      // promoted the next ordinary annotation to timekeeping, and its onset silently\n      // became the record's start time — shifting every sample in that record. Leaving\n      // recordStart null instead is what the caller already handles, with a fallback\n      // timestamp and an ANNOTATION_DECODE_FAILED warning naming the record.\n      /*\n        Only one annotation channel carries a record's start time.\n\n        This flagged the first TAL of *every* annotation channel as timekeeping. In a second\n        channel the first TAL is an ordinary event — so when one failed to parse, the event\n        was dropped and counted as a lost timekeeping entry, which produced the warning\n        \"3 data records carry a timekeeping annotation that could not be read\" followed by\n        \"No event was lost\". Three events had been lost, and the timekeeping in that file was\n        perfectly readable. Both sentences false, about the same three records.\n      */\n      const isTimekeeping = isFirstTal && carriesTimekeeping;\n      isFirstTal = false;\n\n      if (parsed) {\n        if (isTimekeeping) recordStart = parsed.onset;\n        for (const annotation of parsed.annotations) annotations.push(annotation);\n        unreadableDurations += parsed.unreadableDurations;\n        negativeDurations += parsed.negativeDurations;\n      } else {\n        /*\n          Counted apart from the events, because losing one is a different loss.\n\n          A timekeeping TAL is never exported — it says where the record sits, not what\n          happened — so counting it among the entries that \"could not be exported\" both\n          overstated what was lost from annotations.csv and said nothing about the thing that\n          actually went missing, which is a record's position in time. A file with one\n          unreadable timekeeping TAL and three perfectly good events reported \"1 annotation\n          entry was unreadable and could not be exported\" while exporting all three.\n        */\n        if (isTimekeeping) malformedTimekeeping++;\n        else malformed++;\n      }\n    }\n    start = i + 1;\n  }\n\n  return {\n    recordStart,\n    annotations,\n    malformed,\n    malformedTimekeeping,\n    unreadableDurations,\n    negativeDurations,\n  };\n}\n\ninterface ParsedTal {\n  onset: number;\n  annotations: Annotation[];\n  /** How many of those annotations carry a duration the file stated and this could not read. */\n  unreadableDurations: number;\n  /** How many carry a duration that read as a number below zero. */\n  negativeDurations: number;\n}\n\nfunction parseTal(chunk: Uint8Array, recordIndex: number): ParsedTal | null {\n  // The onset must be explicitly signed; anything else is not a TAL.\n  const first = chunk[0];\n  if (first !== 0x2b /* + */ && first !== 0x2d /* - */) return null;\n\n  const text = decodeUtf8(chunk);\n  const parts = text.split(TEXT_SEP_CHAR);\n  const head = parts[0] ?? '';\n\n  let onsetText = head;\n  let durationText: string | null = null;\n  const durationSep = head.indexOf(DURATION_SEP_CHAR);\n  if (durationSep >= 0) {\n    onsetText = head.slice(0, durationSep);\n    durationText = head.slice(durationSep + 1);\n  }\n\n  const onset = Number(onsetText);\n  if (!Number.isFinite(onset)) return null;\n\n  /*\n    A duration the file stated and this could not read is not the same as no duration.\n\n    Both came out as `null` and so as an empty `duration_s` cell, which the documentation\n    defines as meaning the file gave no duration — so an event whose duration was written as\n    `abc` was exported as an event with no duration, indistinguishable from one beside it\n    that genuinely had none, and nothing anywhere said a field had been dropped. The onset is\n    already held to this standard: one that is not a number costs the whole TAL and is\n    reported. A duration is one field of an otherwise readable event, so the event is kept —\n    but it is counted, and the run says so.\n  */\n  let duration: number | null = null;\n  let durationUnreadable = false;\n  if (durationText !== null && durationText !== '') {\n    const d = Number(durationText);\n    if (Number.isFinite(d)) duration = d;\n    else durationUnreadable = true;\n  }\n\n  /*\n    A duration is a length of time, and a length below zero is not one.\n\n    The value is kept and written as the file gave it — inventing a zero, or dropping it to\n    an empty cell, would put a number in annotations.csv that no writer wrote, which is the\n    one thing this tool does not do. But it is reported, because everything downstream\n    quietly does the wrong thing with it: the recipe this documentation gives for the samples\n    an event covers is `onset_s + duration_s`, which for a duration of -3 ends three seconds\n    before the event starts and selects nothing at all, with no error anywhere.\n  */\n  const durationNegative = duration !== null && duration < 0;\n\n  const annotations: Annotation[] = [];\n  for (const raw of parts.slice(1)) {\n    // A trailing separator yields an empty segment; a timekeeping TAL is all empty.\n    if (raw === '') continue;\n    annotations.push({\n      onset,\n      duration,\n      text: raw,\n      recordIndex,\n      ...(durationUnreadable ? { durationUnreadable: true } : {}),\n    });\n  }\n\n  // Per event rather than per TAL: one TAL may carry several texts, and each becomes a row\n  // of annotations.csv with the same cell in it.\n  return {\n    onset,\n    annotations,\n    unreadableDurations: durationUnreadable ? annotations.length : 0,\n    negativeDurations: durationNegative ? annotations.length : 0,\n  };\n}\n\nexport { SEP_TEXT, SEP_DURATION, TAL_END };\n"]}
+{"version":3,"file":"annotations.js","sourceRoot":"","sources":["../../src/edf/annotations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,yDAAyD;AAChF,MAAM,YAAY,GAAG,IAAI,CAAC,CAAC,gCAAgC;AAC3D,MAAM,OAAO,GAAG,IAAI,CAAC;AAErB,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;AACpD,MAAM,iBAAiB,GAAG,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;AAuD5D;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CACrC,KAAiB,EACjB,WAAmB,EACnB,kBAAkB,GAAG,IAAI;IAEzB,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,OAAO;YAAE,SAAS;QAEzD,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACvC;;;;;;;;;;;;cAYE;YACF,IAAI,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC/B,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAE5C,+EAA+E;YAC/E,mFAAmF;YACnF,+EAA+E;YAC/E,iFAAiF;YACjF,+EAA+E;YAC/E,uEAAuE;YACvE;;;;;;;;;cASE;YACF,MAAM,aAAa,GAAG,UAAU,IAAI,kBAAkB,CAAC;YACvD,UAAU,GAAG,KAAK,CAAC;YAEnB,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,aAAa;oBAAE,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC9C,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW;oBAAE,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC1E,mBAAmB,IAAI,MAAM,CAAC,mBAAmB,CAAC;gBAClD,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN;;;;;;;;;kBASE;gBACF,IAAI,aAAa;oBAAE,oBAAoB,EAAE,CAAC;;oBACrC,SAAS,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;QACD,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;IAED,OAAO;QACL,WAAW;QACX,WAAW;QACX,SAAS;QACT,oBAAoB;QACpB,mBAAmB;QACnB,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAWD,iFAAiF;AACjF,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC;AAC3F,CAAC;AAED,SAAS,QAAQ,CAAC,KAAiB,EAAE,WAAmB;IACtD,mEAAmE;IACnE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAElE,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE5B,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpD,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QACvC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC;;;;;;;;;;MAUE;IACF,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,kBAAkB,GAAG,KAAK,CAAC;IAC/B,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;QAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,QAAQ,GAAG,CAAC,CAAC;;YAChC,kBAAkB,GAAG,IAAI,CAAC;IACjC,CAAC;IAED;;;;;;;;;MASE;IACF,MAAM,gBAAgB,GAAG,QAAQ,KAAK,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC;IAE3D,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACjC,gFAAgF;QAChF,IAAI,GAAG,KAAK,EAAE;YAAE,SAAS;QACzB,WAAW,CAAC,IAAI,CAAC;YACf,KAAK;YACL,QAAQ;YACR,IAAI,EAAE,GAAG;YACT,WAAW;YACX,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;IAED,yFAAyF;IACzF,+CAA+C;IAC/C,OAAO;QACL,KAAK;QACL,WAAW;QACX,mBAAmB,EAAE,kBAAkB,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAChE,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;KAC7D,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC","sourcesContent":["/**\n * EDF+ annotation (TAL) decoding.\n *\n * The annotations channel stores UTF-8 text in place of samples. Its bytes are a\n * run of Time-stamped Annotation Lists, each terminated by a NUL, with the rest\n * of the channel NUL-padded:\n *\n *   +<onset>[<0x15><duration>]<0x14><text><0x14>...<0x00>\n *\n * The first TAL of every data record must carry that record's start time and no\n * text; that is how an EDF+D file states where each record actually sits in time.\n *\n *   +1.25<0x15>0.5<0x14>Seizure onset<0x14><0x00>\n */\n\nimport { decodeUtf8 } from './bytes.js';\n\nconst SEP_TEXT = 0x14; // separates onset/duration from text, and text from text\nconst SEP_DURATION = 0x15; // separates onset from duration\nconst TAL_END = 0x00;\n\nconst TEXT_SEP_CHAR = String.fromCharCode(SEP_TEXT);\nconst DURATION_SEP_CHAR = String.fromCharCode(SEP_DURATION);\n\nexport interface Annotation {\n  /** Seconds from the start of the recording. */\n  onset: number;\n  /**\n   * Seconds, or null when the TAL stated no duration that could be read.\n   *\n   * Null covers two cases the file distinguishes and this field does not: a TAL that omitted\n   * the duration, and a TAL that stated one which is not a number. They are told apart by\n   * `unreadableDurations`, which is what raises the warning; the value itself has nowhere\n   * honest to put \"the file said `abc`\".\n   */\n  duration: number | null;\n  text: string;\n  /** Index of the data record this annotation was stored in. */\n  recordIndex: number;\n  /**\n   * True when the file stated a duration that could not be read.\n   *\n   * `duration` is null either way, which is the ambiguity the counts beside it exist to\n   * flag — and those counts were of the whole file while `annotations.csv` is filtered to\n   * the requested window. A conversion of one second of a recording warned that \"1\n   * annotation states a duration that is not a number, so its duration_s cell is empty\"\n   * about an event two seconds outside it, and failed `--strict` for it. Carrying the fact\n   * on the event lets the count be taken where the window has already been applied.\n   */\n  durationUnreadable?: boolean;\n}\n\nexport interface DecodedRecordAnnotations {\n  /** Record start time in seconds, from the leading timekeeping TAL. */\n  recordStart: number | null;\n  annotations: Annotation[];\n  /** Non-empty chunks that were not valid TALs, so the caller can report them. */\n  malformed: number;\n  /** Unreadable TALs in first position, which carry a record's start time, not an event. */\n  malformedTimekeeping: number;\n  /**\n   * Events kept whose stated duration could not be read.\n   *\n   * Counted apart again, for the same reason the two above are: the entry was exported and\n   * nothing about it is missing except the one field, so calling it an entry that \"could not\n   * be exported\" describes a loss that did not happen and hides the one that did.\n   */\n  unreadableDurations: number;\n  /**\n   * Events kept whose stated duration is a readable number below zero.\n   *\n   * Separate from the count above because the value survives: it is written to the CSV as\n   * the file gave it, and what is wrong with it is arithmetic rather than parsing.\n   */\n  negativeDurations: number;\n}\n\n/**\n * Decode one data record's annotation bytes.\n *\n * Malformed TALs are skipped rather than thrown, because a single bad annotation\n * should not cost the user an entire conversion. The count of skipped chunks is\n * returned so the caller can tell the user rather than losing them in silence.\n */\nexport function decodeRecordAnnotations(\n  bytes: Uint8Array,\n  recordIndex: number,\n  carriesTimekeeping = true,\n): DecodedRecordAnnotations {\n  const annotations: Annotation[] = [];\n  let recordStart: number | null = null;\n  let isFirstTal = true;\n  let malformed = 0;\n  let malformedTimekeeping = 0;\n  let unreadableDurations = 0;\n  let negativeDurations = 0;\n\n  let start = 0;\n  for (let i = 0; i <= bytes.length; i++) {\n    if (i !== bytes.length && bytes[i] !== TAL_END) continue;\n\n    if (i > start) {\n      const chunk = bytes.subarray(start, i);\n      /*\n        Padding is not a lost annotation.\n\n        The spec pads the slot with NUL, which the loop above already skips because it is what\n        separates one TAL from the next. Writers pad with spaces instead, and a run of spaces\n        after the last TAL is a non-empty chunk — so a file holding one perfectly readable\n        event, exported in full, was told \"2 annotation entries were unreadable and could not\n        be exported\", one per record. Nothing was lost. Under --strict that is a failed run\n        over the whitespace at the end of a slot.\n\n        Only whitespace. A chunk of anything else that does not parse is a real loss and is\n        still counted, which is the case this warning exists for.\n      */\n      if (chunk.every(isPaddingByte)) {\n        start = i + 1;\n        continue;\n      }\n      const parsed = parseTal(chunk, recordIndex);\n\n      // The timekeeping TAL is the one in first POSITION, whether or not it decodes.\n      // Clearing this flag only on a successful parse meant that an unreadable first TAL\n      // promoted the next ordinary annotation to timekeeping, and its onset silently\n      // became the record's start time — shifting every sample in that record. Leaving\n      // recordStart null instead is what the caller already handles, with a fallback\n      // timestamp and an ANNOTATION_DECODE_FAILED warning naming the record.\n      /*\n        Only one annotation channel carries a record's start time.\n\n        This flagged the first TAL of *every* annotation channel as timekeeping. In a second\n        channel the first TAL is an ordinary event — so when one failed to parse, the event\n        was dropped and counted as a lost timekeeping entry, which produced the warning\n        \"3 data records carry a timekeeping annotation that could not be read\" followed by\n        \"No event was lost\". Three events had been lost, and the timekeeping in that file was\n        perfectly readable. Both sentences false, about the same three records.\n      */\n      const isTimekeeping = isFirstTal && carriesTimekeeping;\n      isFirstTal = false;\n\n      if (parsed) {\n        if (isTimekeeping) recordStart = parsed.onset;\n        for (const annotation of parsed.annotations) annotations.push(annotation);\n        unreadableDurations += parsed.unreadableDurations;\n        negativeDurations += parsed.negativeDurations;\n      } else {\n        /*\n          Counted apart from the events, because losing one is a different loss.\n\n          A timekeeping TAL is never exported — it says where the record sits, not what\n          happened — so counting it among the entries that \"could not be exported\" both\n          overstated what was lost from annotations.csv and said nothing about the thing that\n          actually went missing, which is a record's position in time. A file with one\n          unreadable timekeeping TAL and three perfectly good events reported \"1 annotation\n          entry was unreadable and could not be exported\" while exporting all three.\n        */\n        if (isTimekeeping) malformedTimekeeping++;\n        else malformed++;\n      }\n    }\n    start = i + 1;\n  }\n\n  return {\n    recordStart,\n    annotations,\n    malformed,\n    malformedTimekeeping,\n    unreadableDurations,\n    negativeDurations,\n  };\n}\n\ninterface ParsedTal {\n  onset: number;\n  annotations: Annotation[];\n  /** How many of those annotations carry a duration the file stated and this could not read. */\n  unreadableDurations: number;\n  /** How many carry a duration that read as a number below zero. */\n  negativeDurations: number;\n}\n\n/** Space, tab, CR, LF or NUL — what a writer fills the rest of the slot with. */\nfunction isPaddingByte(byte: number): boolean {\n  return byte === 0x20 || byte === 0x09 || byte === 0x0d || byte === 0x0a || byte === 0x00;\n}\n\nfunction parseTal(chunk: Uint8Array, recordIndex: number): ParsedTal | null {\n  // The onset must be explicitly signed; anything else is not a TAL.\n  const first = chunk[0];\n  if (first !== 0x2b /* + */ && first !== 0x2d /* - */) return null;\n\n  const text = decodeUtf8(chunk);\n  const parts = text.split(TEXT_SEP_CHAR);\n  const head = parts[0] ?? '';\n\n  let onsetText = head;\n  let durationText: string | null = null;\n  const durationSep = head.indexOf(DURATION_SEP_CHAR);\n  if (durationSep >= 0) {\n    onsetText = head.slice(0, durationSep);\n    durationText = head.slice(durationSep + 1);\n  }\n\n  const onset = Number(onsetText);\n  if (!Number.isFinite(onset)) return null;\n\n  /*\n    A duration the file stated and this could not read is not the same as no duration.\n\n    Both came out as `null` and so as an empty `duration_s` cell, which the documentation\n    defines as meaning the file gave no duration — so an event whose duration was written as\n    `abc` was exported as an event with no duration, indistinguishable from one beside it\n    that genuinely had none, and nothing anywhere said a field had been dropped. The onset is\n    already held to this standard: one that is not a number costs the whole TAL and is\n    reported. A duration is one field of an otherwise readable event, so the event is kept —\n    but it is counted, and the run says so.\n  */\n  let duration: number | null = null;\n  let durationUnreadable = false;\n  if (durationText !== null && durationText !== '') {\n    const d = Number(durationText);\n    if (Number.isFinite(d)) duration = d;\n    else durationUnreadable = true;\n  }\n\n  /*\n    A duration is a length of time, and a length below zero is not one.\n\n    The value is kept and written as the file gave it — inventing a zero, or dropping it to\n    an empty cell, would put a number in annotations.csv that no writer wrote, which is the\n    one thing this tool does not do. But it is reported, because everything downstream\n    quietly does the wrong thing with it: the recipe this documentation gives for the samples\n    an event covers is `onset_s + duration_s`, which for a duration of -3 ends three seconds\n    before the event starts and selects nothing at all, with no error anywhere.\n  */\n  const durationNegative = duration !== null && duration < 0;\n\n  const annotations: Annotation[] = [];\n  for (const raw of parts.slice(1)) {\n    // A trailing separator yields an empty segment; a timekeeping TAL is all empty.\n    if (raw === '') continue;\n    annotations.push({\n      onset,\n      duration,\n      text: raw,\n      recordIndex,\n      ...(durationUnreadable ? { durationUnreadable: true } : {}),\n    });\n  }\n\n  // Per event rather than per TAL: one TAL may carry several texts, and each becomes a row\n  // of annotations.csv with the same cell in it.\n  return {\n    onset,\n    annotations,\n    unreadableDurations: durationUnreadable ? annotations.length : 0,\n    negativeDurations: durationNegative ? annotations.length : 0,\n  };\n}\n\nexport { SEP_TEXT, SEP_DURATION, TAL_END };\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "edf2csv",
-  "version": "0.5.92",
+  "version": "0.5.93",
   "description": "Convert EDF, EDF+ and BDF biosignal recordings (European Data Format) to CSV from the command line. Local, streaming, and never resamples or alters units.",
   "keywords": [
     "edf",