edf2csv 0.7.260 → 0.8.1

package/dist/convert/run.d.ts

@@ -106,6 +106,37 @@ export declare function durationDiagnostics(annotations: readonly Annotation[],
     from: number;
     to: number;
 }): Diagnostic[];
+/**
+ * What the descriptions in the events that will actually be written look like.
+ *
+ * EDF's four free-text header fields have had two warnings about where they land since they
+ * were written: `FORMULA_LABEL` for text a spreadsheet runs instead of reading, and
+ * `NONPRINTABLE_LABEL` for bytes that drive a terminal. Both say the same thing about the
+ * remedy — the text is written exactly as the file has it, because rewriting it would mean
+ * the CSV no longer says what the recording says — and both exist so that the tool is not
+ * silent about where it goes.
+ *
+ * `annotations.csv`'s `description` column is the same kind of text, out of the same file,
+ * into the same spreadsheet, and nothing was said about it at all. An event described
+ * `=HYPERLINK("http://…","Sleep stage W")` was written verbatim, exit 0, no warning, and
+ * opens as a live link nobody in the reading chain wrote; one carrying `\x1b[31m` turns the
+ * terminal red on `cat annotations.csv`. It is the more likely of the two to happen by
+ * accident, since a description is typed by a person at a scoring station while a channel
+ * label is written once by the recorder.
+ *
+ * It is also the only free text in the output that can carry a character above U+00FF —
+ * header text is decoded latin1, so every byte of it becomes a code point below U+0100, and
+ * a bidirectional override cannot reach a label. It can reach a description, which is UTF-8.
+ *
+ * Counted rather than raised per event, unlike the header's four fields: a night's scoring is
+ * thousands of events, and a warning each is not a report. The count is of the rows that
+ * reach `annotations.csv`, after the same window filter the writer applies, for the reason
+ * `durationDiagnostics` beside it gives.
+ */
+export declare function descriptionDiagnostics(annotations: readonly Annotation[], window: {
+    from: number;
+    to: number;
+}): Diagnostic[];
 /**
  * The window annotations are filtered by — the bounds as asked for, not as snapped to records.
  *

package/dist/convert/run.js

@@ -11,10 +11,11 @@ import { finished } from 'node:stream/promises';
 import { createGzip, gzipSync } from 'node:zlib';
 import path from 'node:path';
 import { EdfFile } from '../edf/reader.js';
-import { describeFormat, formatRates, formatWallClock } from '../edf/header.js';
+import { describeFormat, formatRates, formatWallClock, startsFormula } from '../edf/header.js';
 import { EdfError } from '../edf/errors.js';
 import { BufferedLineWriter, DEFAULT_FLUSH_THRESHOLD, UTF8_BOM, csvRow, escapeCsvField, } from '../format/csv.js';
 import { counted, listed } from '../format/list.js';
+import { escapeCharacter, unprintableIn } from '../format/unprintable.js';
 import { makeSampleFormatter, makeTimeFormatter, newOffsetBudget, newSampleCacheBudget, plain, } from '../format/number.js';
 import { TIME_COLUMN } from './channels.js';
 import { assertInputPath } from './options.js';
@@ -147,6 +148,7 @@ export async function convert(inputPath, options = {}) {
             // Reported against the rows that will be written, not against the file; see
             // durationDiagnostics.
             plan.diagnostics.push(...durationDiagnostics(annotationData.annotations, window));
+            plan.diagnostics.push(...descriptionDiagnostics(annotationData.annotations, window));
             const result = await writeAnnotationsCsv(outputDir, annotationData.annotations, window, options.gzip === true, options.bom === true);
             written.push(result);
             annotationsWritten = result.rows;
@@ -1048,6 +1050,70 @@ export function durationDiagnostics(annotations, window) {
     }
     return diagnostics;
 }
+/**
+ * What the descriptions in the events that will actually be written look like.
+ *
+ * EDF's four free-text header fields have had two warnings about where they land since they
+ * were written: `FORMULA_LABEL` for text a spreadsheet runs instead of reading, and
+ * `NONPRINTABLE_LABEL` for bytes that drive a terminal. Both say the same thing about the
+ * remedy — the text is written exactly as the file has it, because rewriting it would mean
+ * the CSV no longer says what the recording says — and both exist so that the tool is not
+ * silent about where it goes.
+ *
+ * `annotations.csv`'s `description` column is the same kind of text, out of the same file,
+ * into the same spreadsheet, and nothing was said about it at all. An event described
+ * `=HYPERLINK("http://…","Sleep stage W")` was written verbatim, exit 0, no warning, and
+ * opens as a live link nobody in the reading chain wrote; one carrying `\x1b[31m` turns the
+ * terminal red on `cat annotations.csv`. It is the more likely of the two to happen by
+ * accident, since a description is typed by a person at a scoring station while a channel
+ * label is written once by the recorder.
+ *
+ * It is also the only free text in the output that can carry a character above U+00FF —
+ * header text is decoded latin1, so every byte of it becomes a code point below U+0100, and
+ * a bidirectional override cannot reach a label. It can reach a description, which is UTF-8.
+ *
+ * Counted rather than raised per event, unlike the header's four fields: a night's scoring is
+ * thousands of events, and a warning each is not a report. The count is of the rows that
+ * reach `annotations.csv`, after the same window filter the writer applies, for the reason
+ * `durationDiagnostics` beside it gives.
+ */
+export function descriptionDiagnostics(annotations, window) {
+    const written = annotations.filter((a) => a.onset >= window.from && a.onset < window.to);
+    const diagnostics = [];
+    const formulaic = written.filter((a) => startsFormula(a.text));
+    if (formulaic.length > 0) {
+        const one = formulaic.length === 1;
+        const shown = [...new Set(formulaic.map((a) => a.text[0]))].join(', ');
+        diagnostics.push({
+            code: 'FORMULA_LABEL',
+            severity: 'warning',
+            message: `${counted(formulaic.length, 'annotation')} ${one ? 'has a description' : 'have descriptions'} ` +
+                `starting with ${shown}, which Excel, LibreOffice and Google Sheets read as the start ` +
+                `of a formula rather than as text.`,
+            hint: 'The text is written to annotations.csv exactly as the file has it, so the cell is ' +
+                'what the recording says. Open the CSV with pandas or R, or import it into the ' +
+                'spreadsheet as text, if you do not want it evaluated.',
+        });
+    }
+    const marked = written.filter((a) => unprintableIn(a.text).length > 0);
+    if (marked.length > 0) {
+        const one = marked.length === 1;
+        const shown = [...new Set(marked.flatMap((a) => unprintableIn(a.text)))]
+            .map(escapeCharacter)
+            .join(', ');
+        diagnostics.push({
+            code: 'NONPRINTABLE_LABEL',
+            severity: 'warning',
+            message: `${counted(marked.length, 'annotation')} ${one ? 'has a description' : 'have descriptions'} ` +
+                `carrying text a terminal does not print as itself (${shown}), written to ` +
+                `annotations.csv exactly as the file has ${one ? 'it' : 'them'}.`,
+            hint: 'A control byte can drive the terminal and a bidirectional override reverses what ' +
+                'follows it, so read the file with pandas or R rather than with cat. The cell is ' +
+                'what the recording says either way.',
+        });
+    }
+    return diagnostics;
+}
 /**
  * The window annotations are filtered by — the bounds as asked for, not as snapped to records.
  *