@dynamic-entity/server 1.14.0

package/write-template-C2dCEp48.d.ts

@@ -0,0 +1,332 @@
+import { ImportRowError, ConfigProblem, EntityFormConfig, MappingPlan, FormRule, ImportLookups, TemplateSpec } from '@dynamic-entity/core';
+import { Writable } from 'node:stream';
+
+/**
+ * limits.ts — every bound this package enforces, and its default.
+ *
+ * **Every limit has a finite default.** A limit that is only a limit when a consumer
+ * remembers to set one is documentation, not a guard, and this is the first
+ * attacker-reachable code in the repository. A consumer who wants more raises the number;
+ * nobody has to know the number exists to be protected by it.
+ *
+ * Each is enforced **during** streaming rather than after. A check that runs once the file is
+ * in memory has already lost the thing it was checking.
+ */
+interface ImportLimits {
+    /** Total bytes accepted from the request body. Counted as they arrive. */
+    maxBytes: number;
+    /** Multipart fields accepted, so a body of ten thousand tiny parts is not a parse loop. */
+    maxFields: number;
+    /** Bytes accepted in one non-file field — in practice the JSON `plan`. */
+    maxFieldBytes: number;
+    /** Bytes accepted in a field *name*, which is also attacker-chosen. */
+    maxFieldNameBytes: number;
+    /** Files accepted. One import is one file; more is either a mistake or an attack. */
+    maxFiles: number;
+    /** Data rows read. The header is not one of them. */
+    maxRows: number;
+    /** Columns read from the header row, and the width any row may reach. */
+    maxColumns: number;
+    /**
+     * Characters in one cell.
+     *
+     * This is also the bound on `validators.pattern`: a config-supplied regex with
+     * catastrophic backtracking is CPU-bound in the length of what it is run against, and on a
+     * server that input is attacker-chosen. See SECURITY.md.
+     */
+    maxCellLength: number;
+    /** Total inflated bytes across every entry, checked *as they inflate*. */
+    maxUncompressedBytes: number;
+    /** Inflated-to-compressed ratio at which an entry is refused. */
+    maxCompressionRatio: number;
+    /** Entries in the archive. A workbook has tens; a bomb has thousands. */
+    maxZipEntries: number;
+    /**
+     * Rows handed to `applyMapping`, and therefore to `onBatch`, at a time.
+     *
+     * Peak memory is a function of this number and not of the file's size. That is the entire
+     * reason this package exists.
+     */
+    batchSize: number;
+    /**
+     * Row errors **retained**. The count and the truncation flag are always exact.
+     *
+     * A file where every row fails accumulates one error object per failing row, and fifty
+     * thousand of those is the same unbounded growth as holding the file — the failure this
+     * package exists to avoid, arrived at from the other direction.
+     *
+     * Lowering it costs a user *reasons*, never *counts*: `errorCount` and `failed` are tallied
+     * as rows go past and do not depend on what was kept.
+     */
+    maxReportedErrors: number;
+    /** Rows a preview returns as a sample. */
+    sampleRows: number;
+    /** Milliseconds of silence on the request body before it is abandoned. Slowloris. */
+    idleTimeoutMs: number;
+    /** Milliseconds a whole request may take, however chatty it is. */
+    totalTimeoutMs: number;
+}
+/**
+ * Sized for "a large legitimate export", not for "the largest thing imaginable".
+ *
+ * 10 MB of xlsx is comfortably the fifty-thousand-row workbook that motivated the package;
+ * `maxRows` is above it so the byte limit is the one a user meets first, and the row limit is
+ * the backstop for a file that is mostly empty cells.
+ */
+declare const DEFAULT_LIMITS: ImportLimits;
+/**
+ * Fill in what a caller left out.
+ *
+ * A non-finite or non-positive override falls back to the default rather than being accepted:
+ * `maxBytes: 0` from a mis-parsed environment variable would otherwise disable every upload,
+ * and `maxBytes: Infinity` would disable the guard — the two ways a configuration mistake
+ * turns a limit off, both of which look deliberate in a diff.
+ */
+declare function resolveLimits(overrides?: Partial<ImportLimits>): ImportLimits;
+
+/**
+ * bytes.ts — the three things every reader here needs from a byte stream.
+ *
+ * Deliberately framework-neutral: a `ByteSource` is anything you can `for await` bytes out of,
+ * which a Node `Readable`, a `Buffer`, and a hand-rolled async generator all are. Nothing in
+ * this file knows what a request is.
+ */
+/** Anything bytes can be pulled from — a request, a file handle, a buffer, a test fixture. */
+type ByteSource = AsyncIterable<Uint8Array> | Iterable<Uint8Array> | Uint8Array;
+/** Normalise the three shapes above into one async iterable. */
+declare function toByteStream(source: ByteSource): AsyncGenerator<Uint8Array>;
+/**
+ * Refuse the upload at the byte that crosses the line, not once it has all arrived.
+ *
+ * Counting after the body is buffered is the guard that has already lost: the memory it was
+ * protecting is spent by the time it runs. This throws mid-stream, and the caller destroys
+ * the source — so a hostile 4 GB body costs `maxBytes` plus one chunk.
+ */
+declare function limitBytes(source: AsyncIterable<Uint8Array>, maxBytes: number): AsyncGenerator<Uint8Array>;
+/**
+ * Take the first `n` bytes without consuming them.
+ *
+ * Format is decided by what a file *is* rather than by what its name claims, which means
+ * looking at its first four bytes before choosing a reader — and then handing those same
+ * bytes to the reader that was chosen. Returning the head separately and re-yielding it is
+ * the whole trick; buffering the file to seek back to zero is the thing being avoided.
+ */
+declare function peek(source: AsyncIterable<Uint8Array>, n: number): Promise<{
+    head: Uint8Array;
+    stream: AsyncGenerator<Uint8Array>;
+}>;
+/** Best-effort close of whatever the bytes came from, once a guard has decided to stop. */
+declare function destroySource(source: unknown): void;
+
+/**
+ * read-sheet.ts — bytes in, a header row and an async stream of rows out.
+ *
+ * Two properties this file exists to hold:
+ *
+ * 1. **Format is decided by content, never by the filename.** `.csv` is client-controlled and
+ *    an attacker who wants a zip parsed will name their zip `.csv`. The first four bytes are
+ *    not. Both readers then meet the same row guards, so a workbook cannot buy itself a
+ *    larger sheet by being a workbook.
+ * 2. **Rows arrive one at a time.** Nothing here ever holds the sheet. `rows` is an
+ *    `AsyncIterable`, so a consumer that stops pulling stops the read.
+ *
+ * Cells are `unknown[]`, never `string[]`. An xlsx cell is a number, a boolean or a `Date`,
+ * and core's `coerceCell` reads a typed cell as its type — stringifying here to make the
+ * signature tidier is exactly the bug that would reintroduce.
+ */
+
+/** The formats this package reads. Both are decided by magic bytes. */
+type SheetFormat = 'csv' | 'xlsx';
+interface SheetSource {
+    /** What the bytes turned out to be, which may not be what the filename claimed. */
+    format: SheetFormat;
+    headers: string[];
+    /** Data rows, one at a time. Pulling stops the read; not pulling stops it too. */
+    rows: AsyncIterable<unknown[]>;
+}
+interface ReadSheetOptions {
+    stream: ByteSource;
+    /** Display only — it names the file in an error message and decides nothing. */
+    filename?: string;
+    limits?: Partial<ImportLimits>;
+}
+/**
+ * What these bytes actually are.
+ *
+ * Exported because the multipart layer wants the same answer for the same reason, and because
+ * a second copy of a magic-byte table is a second table to be wrong.
+ */
+declare function detectFormat(head: Uint8Array): SheetFormat;
+/**
+ * Enforce the row, column and cell bounds as rows go past, and pad short rows.
+ *
+ * Padding matches `parseCsv`, through the same `padRow` core exports, so a file imported on a
+ * server and the same file imported in a browser produce the same rows. A second padding rule
+ * here would be a place for the two halves to disagree, silently, about a row that ended early.
+ */
+declare function guardRows(rows: AsyncIterable<unknown[]>, width: number, limits: ImportLimits): AsyncGenerator<unknown[]>;
+/**
+ * Open a sheet: read its header row, and hand back the rest as a stream.
+ *
+ * The header row is pulled eagerly because everything downstream — the column count, the
+ * mapping suggestion, the padding width — is a function of it. Exactly one row is held.
+ */
+declare function readSheet(options: ReadSheetOptions): Promise<SheetSource>;
+
+/**
+ * run-import.ts — the package, really. Everything else is plumbing around this file.
+ *
+ * `runImport` pulls rows off the reader in batches, hands each batch to core's `applyMapping`,
+ * **awaits** `onBatch`, and drops the batch. Peak memory is a function of `batchSize`, not of
+ * the file's size — which is the one reason this package exists, and the claim every decision
+ * below is answerable against.
+ *
+ * Nothing here decides anything core already decides. The mapping rules, the coercion, the
+ * validation and the record shape all come from `applyMapping`; this file's entire job is to
+ * feed it and to not accumulate.
+ */
+
+/** Where a batch sits in the user's spreadsheet, for a consumer that logs or reports progress. */
+interface BatchInfo {
+    /** Zero-based batch number. */
+    index: number;
+    /** Spreadsheet row number of the first row this batch was built from. */
+    firstRow: number;
+    /** Records handed over before this batch, so `imported + records.length` is the running total. */
+    imported: number;
+}
+/**
+ * Write a batch of records.
+ *
+ * **It is awaited, and that is load-bearing.** A consumer writing to a database is slower than
+ * a parser reading a file; not awaiting would turn bounded memory into an unbounded queue of
+ * pending writes — the exact failure this package exists to avoid, reached from the other
+ * direction.
+ *
+ * **It must be idempotent.** A stream that fails at row 30,000 has already written 29,999
+ * records, and an HTTP client — or a proxy, or a user — will retry. See SECURITY.md.
+ */
+type OnBatch = (records: Record<string, unknown>[], info: BatchInfo) => void | Promise<void>;
+interface RunImportOptions {
+    stream: ByteSource;
+    /** Display only. The format is decided by the file's first bytes, never by its name. */
+    filename?: string;
+    plan: MappingPlan;
+    config: EntityFormConfig;
+    rules?: readonly FormRule[];
+    /** Values for every `listName` the config mentions. A server has no `LOOKUP_REGISTRY`. */
+    lookups?: ImportLookups;
+    lang?: string;
+    /** Omit to validate without writing: the identical pipeline, every error, nothing stored. */
+    onBatch?: OnBatch;
+    limits?: Partial<ImportLimits>;
+}
+interface ImportRunResult {
+    /**
+     * Records the run produced — and therefore, when `onBatch` was supplied, how many were
+     * handed over to be written. The records themselves are deliberately not here: returning
+     * fifty thousand of them would undo the streaming this whole file is for.
+     */
+    imported: number;
+    /** Rows that held no values at all. A blank line is not an error and is not a record. */
+    skipped: number;
+    /**
+     * Rows that produced at least one error, counted exactly — **not** derived from `errors`.
+     *
+     * `errors` is capped, so counting the distinct rows in it answers how many rows fitted in the
+     * cap rather than how many failed. Measured on a run of a thousand rows where two hundred
+     * failed: the sample held seven of them. The number a user acts on is "which rows do I fix",
+     * so it is the one that has to be exact.
+     *
+     * Row numbers never span a batch, so summing the per-batch distinct counts is exact rather
+     * than approximate.
+     */
+    failed: number;
+    /** A **sample**, capped at `limits.maxReportedErrors`. `errorCount` is the true total. */
+    errors: ImportRowError[];
+    /** Every problem found, whether or not it was retained. */
+    errorCount: number;
+    /** `errors` is shorter than `errorCount`. */
+    truncated: boolean;
+    /** What is wrong with the plan itself. An `error` here means no row was read at all. */
+    planProblems: ConfigProblem[];
+    /** Data rows read from the sheet, blank ones included. */
+    rowsRead: number;
+    /**
+     * What the file turned out to be — **absent when no byte of it was ever read**.
+     *
+     * A plan refused before the stream is touched has nothing to report here, and it used to
+     * report `'csv'` anyway. A guessed field is worse than a missing one: it is indistinguishable
+     * from a measured one.
+     */
+    format?: SheetFormat;
+}
+declare function runImport(options: RunImportOptions): Promise<ImportRunResult>;
+interface PreviewSheetOptions {
+    stream: ByteSource;
+    filename?: string;
+    config: EntityFormConfig;
+    lang?: string;
+    limits?: Partial<ImportLimits>;
+}
+/** What a first look at an uploaded file yields. Matches the renderer's `ImportPreview`. */
+interface SheetPreview {
+    headers: string[];
+    /** The first few rows as text, for the mapping screen to show under each column. */
+    sample: string[][];
+    /** Every inferred match is tagged `guess`. */
+    suggestion: MappingPlan;
+    /** Data rows in the file, which the whole stream is read to count. */
+    rowCount: number;
+    format: SheetFormat;
+}
+/**
+ * Headers, a sample, and a suggested mapping.
+ *
+ * The whole file is read, because `rowCount` is a number the user is shown before they commit
+ * and "about this many" is not a useful thing to tell someone about their own data. Reading it
+ * also means every limit applies to a preview exactly as it applies to an import — a preview
+ * endpoint that skipped them would be the cheapest way in.
+ */
+declare function previewSheet(options: PreviewSheetOptions): Promise<SheetPreview>;
+
+/**
+ * write-template.ts — the sheet to hand a user so their data arrives already in the right shape.
+ *
+ * **This is the one place in the library that writes a file another program executes.** A
+ * template is opened in Excel, and `=cmd|'/C calc'!A0` in a cell is a command, not text. So
+ * every cell goes through core's `escapeFormula` on the way in.
+ *
+ * The cell's *type* is the other half, and it cannot be set: exceljs derives it from the value
+ * and exposes `type` read-only. Handing it a string is therefore what makes a string cell —
+ * which is a claim about a library rather than a line of code here, so `write-template.spec.ts`
+ * reads the written XML back and asserts no `<f>` element appears in it. A guard that is an
+ * assertion in a test is a guard; a comment saying "exceljs writes this as text" is not.
+ *
+ * The header text itself is derived from config labels, which is authored data like any other
+ * and therefore not trusted here.
+ */
+
+/** The formats a generated template can be written in. */
+type TemplateFormat = 'csv' | 'xlsx';
+interface WriteTemplateOptions {
+    spec: TemplateSpec;
+    format: TemplateFormat;
+    /** Written into as the file is produced; an HTTP response is the intended one. */
+    out: Writable;
+    limits?: Partial<ImportLimits>;
+}
+/** The extension a written template should be offered under. */
+declare const TEMPLATE_EXTENSION: Record<TemplateFormat, string>;
+/** The media type a written template should be served as. */
+declare const TEMPLATE_MEDIA_TYPE: Record<TemplateFormat, string>;
+/**
+ * Write a template into `out`.
+ *
+ * Resolves when the file is complete and the stream has been ended. A caller that owns the
+ * stream for other reasons — an Express response, say — gets it back finished, which is what
+ * a download handler wants.
+ */
+declare function writeTemplate(options: WriteTemplateOptions): Promise<void>;
+
+export { type BatchInfo as B, DEFAULT_LIMITS as D, type ImportLimits as I, type OnBatch as O, type PreviewSheetOptions as P, type ReadSheetOptions as R, type SheetFormat as S, type TemplateFormat as T, type WriteTemplateOptions as W, type ByteSource as a, type ImportRunResult as b, type RunImportOptions as c, type SheetPreview as d, type SheetSource as e, TEMPLATE_EXTENSION as f, TEMPLATE_MEDIA_TYPE as g, destroySource as h, detectFormat as i, guardRows as j, previewSheet as k, limitBytes as l, resolveLimits as m, runImport as n, peek as p, readSheet as r, toByteStream as t, writeTemplate as w };