@murumets-ee/imports 0.0.0-preview-20261014110621

package/dist/worker-CRH_PjKO.d.mts

@@ -0,0 +1,940 @@
+import { c as TransformRegistry, i as RowTransform, t as DirectionalPair } from "./transform-WeiiLdYP.mjs";
+import { Logger } from "@murumets-ee/core";
+import { z } from "zod";
+import { EsClientLike } from "@murumets-ee/search-elasticsearch";
+import { JobDefinition, QueueTerminalNotifier } from "@murumets-ee/queue/client";
+import { AdminClient } from "@murumets-ee/entity/admin";
+
+//#region src/entities/import-run.d.ts
+/**
+ * Lifecycle stages.
+ *
+ * - `pending` — row exists, queue job has been enqueued, worker hasn't picked it up yet.
+ * - `running` — worker is streaming + bulk-writing.
+ * - `succeeded` — worker finished cleanly (note: per-row failures still possible — see `totals.failed`).
+ * - `failed` — worker threw and the queue marked the job dead. `errorSummary.fatal` carries the cause.
+ * - `cancelled` — manual operator action via PR 8a (out of scope for PR 7).
+ */
+declare const IMPORT_RUN_STATUSES: readonly ["pending", "running", "succeeded", "failed", "cancelled"];
+type ImportRunStatus = (typeof IMPORT_RUN_STATUSES)[number];
+declare const ImportRun: import("@murumets-ee/entity").Entity<{
+  id: import("@murumets-ee/entity").IdField;
+} & import("@murumets-ee/entity").AuditableFields & {
+  /** Operator-visible label, e.g. `"MERCEDES — ME_20251027075918.txt"`. Free-form. */label: import("@murumets-ee/entity").TextField & {
+    readonly required: true;
+    readonly maxLength: 255;
+  }; /** See {@link IMPORT_RUN_STATUSES}. */
+  status: import("@murumets-ee/entity").SelectField & {
+    options: readonly ["pending", "running", "succeeded", "failed", "cancelled"];
+  } & {
+    readonly options: readonly ["pending", "running", "succeeded", "failed", "cancelled"];
+    readonly default: "pending";
+    readonly indexed: true;
+  };
+  /**
+   * Path or storage key of the uploaded feed file. Generic string —
+   * could be `/var/lumi/uploads/<id>.txt` for the local-disk PoC or an
+   * S3 object key once storage adapter integration lands. The worker
+   * reads this with the configured `readFeed` resolver.
+   */
+  filePath: import("@murumets-ee/entity").TextField & {
+    readonly required: true;
+    readonly maxLength: 2048;
+  };
+  /**
+   * Name of the registered transform applied to each row. Resolved at
+   * worker-dispatch time against the transform registry contributed by
+   * the consumer. PoC: `'commerce:carmaker-feed'`.
+   */
+  transformName: import("@murumets-ee/entity").TextField & {
+    readonly required: true;
+    readonly maxLength: 128;
+    readonly indexed: true;
+  };
+  /**
+   * Opaque per-transform parameters. The carmaker transform expects
+   * `{ brandId, supplierId, codePrefix? }`. Validation happens inside
+   * the transform — the imports package never inspects this shape so
+   * a new transform with different params doesn't require a schema
+   * migration.
+   */
+  params: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>;
+  /**
+   * Row counters: `{ submitted, succeeded, failed, skipped, batches }`.
+   * Updated by the worker as each batch completes. Final values are
+   * what the operator reads; intermediate progress comes from
+   * `toolkit_jobs.progress` via the queue UI.
+   */
+  totals: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>;
+  /**
+   * Output of `ErrorTracker.getTopPatterns(totalRows)` — the top-50
+   * error signatures with up to 5 sample rows each. Empty `[]` until
+   * the worker writes it on completion.
+   */
+  errorSummary: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>;
+  /**
+   * NON-FATAL warning patterns (commerce-demo PR4) — same
+   * `ErrorTrackerSnapshot` shape as {@link errorSummary} but for advisories
+   * on rows that STILL succeeded (their doc was indexed), e.g.
+   * `no_purchase_price` (a part imported list-only, cost pending). Kept
+   * separate from `errorSummary` so a warning never reads as a failed row.
+   * Null/absent until a SUCCESSFUL run that produced warnings writes it — a
+   * fatal failure (`markFailed`) abandons the run mid-stream and only sets
+   * `errorSummary.fatal`, so the two summaries are not co-present.
+   *
+   * NOTE: because the warning aggregator reuses `ErrorTracker`, the persisted
+   * JSON reuses its field names — the `totalErrors` key inside this blob is
+   * really the WARNING count (and `patterns[].errorType` the warning type).
+   */
+  warningSummary: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>;
+  /**
+   * Price-distribution snapshot (F010 / issue #602) — `{ count, nullRate,
+   * min, median, max }`, computed over ES-accepted rows' derived purchase
+   * price (`PartsDocument.purchase_price_minor`, integer minor units) in
+   * the SAME streaming pass that builds `totals`. See
+   * `@murumets-ee/imports`' `price-stats.ts` for the exact shape and the
+   * bounded-sample cap.
+   *
+   * The pipeline's existing 6b header-drift check (commerce's
+   * register-handler.ts) only catches STRUCTURAL drift — a mapped column
+   * vanishing from the header row. It does NOT catch a supplier silently
+   * flipping their decimal/thousands separator, which changes no header
+   * at all and produces numerically valid but silently WRONG prices. The
+   * worker compares THIS run's `priceStats.median` against the most
+   * recent prior SUCCEEDED run for the same (transformName, brandId,
+   * supplierId) and, on a magnitude-scale shift (>=10x or <=0.1x — see
+   * `PRICE_ANOMALY_THRESHOLD_MULTIPLIER`), appends a NON-FATAL
+   * `price_anomaly` entry to {@link warningSummary}. Unlike the 6b 409,
+   * this never blocks the run — a real repricing is a legitimate reason
+   * prices would shift, so it's an advisory only.
+   *
+   * `null`/absent for runs written before this field existed, or for a
+   * schema/transform whose rows never carry a price sample.
+   */
+  priceStats: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>; /** Set when the worker picks up the job. */
+  startedAt: import("@murumets-ee/entity").DateField & Partial<import("@murumets-ee/entity").DateField>; /** Set when the worker finishes (success OR fatal failure). */
+  finishedAt: import("@murumets-ee/entity").DateField & Partial<import("@murumets-ee/entity").DateField>;
+  /**
+   * `toolkit_jobs.id` of the queue job processing this run — link
+   * back so PR 8a can show live progress without a second lookup.
+   */
+  queueJobId: import("@murumets-ee/entity").TextField & {
+    readonly maxLength: 64;
+    readonly indexed: true;
+  };
+}, "import_run", [import("@murumets-ee/entity").Behavior<import("@murumets-ee/entity").AuditableFields, string>, import("@murumets-ee/entity").SearchableBehavior]>;
+type ImportRunClient = AdminClient<typeof ImportRun.allFields>;
+//#endregion
+//#region src/streaming.d.ts
+/**
+ * Pre-pass: count data rows in a delimited file (excluding the header
+ * row when `hasHeader` is true; default `true` to match `streamFeed`).
+ *
+ * Implementation: stream the file as raw bytes and tally `\n`. NOT a
+ * full parse — escaped newlines inside quoted fields would over-count.
+ * That was unconditionally fine while `quote: false` was the only
+ * behaviour available; a mapping can now declare RFC 4180 (`quote`),
+ * and an embedded newline in a quoted field then makes this an UPPER
+ * BOUND rather than a count. It is used only as the progress
+ * denominator (`totalRows`), so the visible consequence is a bar that
+ * stops short of 100%, never a wrong import. The parsed row count from
+ * `streamFeed` is authoritative and is what the run records.
+ *
+ * Why bytes-only and not csv-parse: a 16 MB feed counts in ~50 ms with
+ * `\n` tally vs ~500 ms with csv-parse parsing. For 500 MB the gap
+ * matters — pre-pass needs to be a small fraction of the actual import
+ * or the operator perceives the run as "stuck on pre-pass" with no UI.
+ *
+ * Returns the number of DATA rows (header excluded when present).
+ * Empty trailing lines (no `\n` after the final row, or a single
+ * trailing `\n`) are handled identically to how csv-parse treats them
+ * with `skip_empty_lines: true`.
+ */
+declare function countDataRows(filePath: string, options?: {
+  hasHeader?: boolean;
+}): Promise<number>;
+interface StreamFeedOptions {
+  /** Path to the file on disk. The PoC uploads land on local disk; S3-keyed reads come later. */
+  filePath: string;
+  /**
+   * Single-character field delimiter. Default `\t` (the carmaker feed
+   * format). Pass `,` for CSV, `;` for some European dialects.
+   */
+  delimiter?: string;
+  /**
+   * `true` (default): the first row is the header and column names come
+   * from it. `false`: rows are emitted as positional `{ "0": ..., "1": ... }`
+   * and the transform reads by index — useful for headerless feeds that
+   * commit to a documented column order.
+   */
+  hasHeader?: boolean;
+  /**
+   * Optional explicit column-name list. When provided, takes precedence
+   * over `hasHeader` (header row, if present, is skipped but its values
+   * are ignored). Useful when the upstream header is unstable but the
+   * positional shape isn't.
+   */
+  columns?: ReadonlyArray<string>;
+  /**
+   * Forward to `csv-parse` `relax_column_count`. Default `false` —
+   * a row whose column count doesn't match the header surfaces as a
+   * parser error so the transform isn't silently fed truncated data.
+   */
+  relaxColumnCount?: boolean;
+  /**
+   * Forward to `csv-parse` `quote`. The character (or `false`) that
+   * encloses fields containing the delimiter. Default `false` — most
+   * supplier feeds in this package's target use case are tab-delimited
+   * exports that contain literal `"` characters in description fields
+   * (e.g. `LIVRE "INVERSION D'IMAGE"` from the Mercedes carmaker feed)
+   * with NO quote-as-field-wrapper convention. csv-parse's library
+   * default `'"'` then misreads the literal `"` as an opening quote
+   * and rejects the row with `Invalid Opening Quote`.
+   *
+   * Pass `'"'` (or any character) to opt back into RFC-4180-style
+   * CSV parsing for genuine quoted-CSV inputs. Pass `'\''` for
+   * single-quoted dialects.
+   */
+  quote?: string | false;
+}
+interface StreamFeedRow {
+  /** 1-based row number. Header (when present) is row 0; first data row is row 1. */
+  rowNumber: number;
+  /**
+   * Cell values keyed by column name (or string-position when
+   * `hasHeader: false` AND no `columns`).
+   *
+   * **Cell-value invariants:**
+   * - Empty cells (`A\t\tC`) → `''` (empty string).
+   * - Missing TRAILING cells in `relaxColumnCount: true` mode → the
+   *   key is **absent** from the object, not present-with-`''`. csv-parse
+   *   does not emit keys for short rows. Transforms reading those
+   *   columns get `undefined` from `row['col']` and must handle it
+   *   (`row['col'] ?? ''` is the canonical idiom).
+   * - With the default `relaxColumnCount: false`, short rows reject at
+   *   the parser, so this case never reaches the transform.
+   */
+  row: Record<string, string>;
+}
+/**
+ * Async-iterable over the parsed rows of a delimited file. Use with
+ * `for await (const { rowNumber, row } of streamFeed({ filePath, ... }))`.
+ *
+ * The iterator owns its file descriptor — the `for await` loop closes
+ * the underlying stream when it returns or breaks. Aborting mid-stream
+ * (`break`, `throw`, signal) is safe; csv-parse propagates the close.
+ */
+declare function streamFeed(options: StreamFeedOptions): AsyncIterable<StreamFeedRow>;
+//#endregion
+//#region src/read-feed.d.ts
+/**
+ * The reader-config subset safe to persist in `import_run.params.feed` and rehydrate
+ * in the worker (commerce-demo PR3). It is exactly the {@link ReadFeedOptions} fields
+ * that describe HOW to parse a feed — `filePath` is supplied by the worker after it
+ * materialises the upload, never carried in params. Validated at BOTH ends: the
+ * commerce run route builds it from a saved `import_mapping`, the worker re-parses it
+ * before handing it to {@link readFeed} (defense-in-depth against a tampered row).
+ */
+declare const feedReaderConfigSchema: z.ZodObject<{
+  format: z.ZodOptional<z.ZodEnum<["csv", "tsv", "xlsx"]>>;
+  delimiter: z.ZodOptional<z.ZodString>;
+  hasHeader: z.ZodOptional<z.ZodBoolean>;
+  headerRowIndex: z.ZodOptional<z.ZodNumber>;
+  sheet: z.ZodOptional<z.ZodString>;
+  /**
+   * csv-parse's `quote`. Absent (or `false`) = quotes are ordinary data, which is this
+   * package's long-standing default and what the tab-delimited supplier dumps require. A
+   * single character opts into RFC 4180 for feeds that genuinely quote their fields.
+   *
+   * Persisted deliberately: the two shapes are mutually exclusive on real data (R027 —
+   * each of the three ground-truth feeds hard-fails under the other's setting), so this
+   * is a per-file fact a mapping has to carry. Deriving it from `format` would be the
+   * same class of mistake as a global decimal separator.
+   */
+  quote: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodLiteral<false>]>>;
+}, "strict", z.ZodTypeAny, {
+  hasHeader?: boolean | undefined;
+  delimiter?: string | undefined;
+  quote?: string | false | undefined;
+  headerRowIndex?: number | undefined;
+  sheet?: string | undefined;
+  format?: "csv" | "tsv" | "xlsx" | undefined;
+}, {
+  hasHeader?: boolean | undefined;
+  delimiter?: string | undefined;
+  quote?: string | false | undefined;
+  headerRowIndex?: number | undefined;
+  sheet?: string | undefined;
+  format?: "csv" | "tsv" | "xlsx" | undefined;
+}>;
+type FeedReaderConfig = z.infer<typeof feedReaderConfigSchema>;
+/**
+ * Rehydrate the reader config a run persisted in `import_run.params.feed`.
+ *
+ * ONE implementation, because two would drift and drift here is silent: the
+ * worker uses it to decide how to READ a feed, and the #842 audit uses it to
+ * re-read the same file the same way. An audit that rehydrates differently from
+ * the run it is auditing invents corruption, or misses it.
+ *
+ * ABSENT means "the carmaker default", i.e. tab-delimited, which is what an
+ * empty config parses to — so a non-object `params`, or `params` with no
+ * `feed`, is a real answer rather than a gap. A PRESENT `feed` is validated
+ * whatever its shape: a scalar or an array there is a malformed persisted blob,
+ * and reading it as "no config" would silently import an xlsx run as tsv.
+ */
+declare function readPersistedFeedConfig(params: unknown): ReturnType<typeof feedReaderConfigSchema.safeParse>;
+interface ReadFeedOptions {
+  /** Path to the file on local disk. */
+  filePath: string;
+  /** Feed format. Omitted → tab-delimited (tsv), matching the legacy carmaker reader. */
+  format?: 'csv' | 'tsv' | 'xlsx';
+  /** Explicit delimiter override (csv/tsv). Defaults from `format` (`csv` → `,`, else `\t`). */
+  delimiter?: string;
+  /** First row is the header. Default `true`. */
+  hasHeader?: boolean;
+  /** 0-based header row index (xlsx). */
+  headerRowIndex?: number;
+  /** xlsx worksheet as a 1-based index string. Omit → first sheet. */
+  sheet?: string;
+  /** Explicit column-name list (csv/tsv) — see `streamFeed`. */
+  columns?: ReadonlyArray<string>;
+  /** Forward to csv-parse `relax_column_count` (csv/tsv). */
+  relaxColumnCount?: boolean;
+  /** Forward to csv-parse `quote` (csv/tsv). */
+  quote?: string | false;
+}
+/** Stream a feed file's data rows, dispatching on `format`. */
+declare function readFeed(options: ReadFeedOptions): AsyncIterable<StreamFeedRow>;
+//#endregion
+//#region src/error-tracker.d.ts
+/**
+ * Aggregates per-row errors into top-N pattern buckets so a feed of 1M
+ * malformed rows surfaces as a handful of actionable signatures rather
+ * than a million identical strings.
+ *
+ * Lifted from giga-test (`backend/src/workers/csv-importer.ts`) and
+ * generalized:
+ * - Configurable caps so the importer can tune memory bounds per run.
+ * - Pure data — no logging side effects, no I/O.
+ * - JSON-serialisable output via {@link ErrorTracker.snapshot} for the
+ *   `import_run.errorSummary` column.
+ *
+ * Pattern signature shape: `${errorType}:${field || 'GENERAL'}:${message}`.
+ * Same `errorType + field + message` collapses to one bucket; differing
+ * messages stay separate. This is intentional: a parser error on column
+ * `NetPrice/Discount` ("invalid number 'NA,5'") and the same on column
+ * `GrossPrice` are operationally distinct even if the parser is the same.
+ *
+ * Memory bounds: the patterns map is capped at `maxPatterns`. Once full,
+ * additional NEW signatures are dropped — known patterns keep accumulating
+ * counts. This is the "top-N most common" model: rare-but-novel errors
+ * past the cap are invisible, but the cap protects against a runaway
+ * adversarial feed exploding the map. Sample arrays are independently
+ * capped at `maxSamplesPerPattern`.
+ */
+/**
+ * Recursively-defined JSON-serialisable value. Mirrors the shape of
+ * `JsonValue` in `@murumets-ee/entity` without taking a dependency on
+ * that package — error-tracker is otherwise standalone, and the
+ * `ImportRun.errorSummary` JSONB column accepts anything in this shape.
+ */
+type ImportJsonValue = string | number | boolean | null | ImportJsonValue[] | {
+  [key: string]: ImportJsonValue;
+};
+/** One sample row attached to a pattern. `rowData` is the originally-parsed row. */
+interface ErrorSample {
+  rowNumber: number;
+  rowData: ImportJsonValue;
+}
+/** Public shape of an aggregated pattern as returned by {@link ErrorTracker.getTopPatterns}. */
+interface ErrorPattern {
+  errorType: string;
+  field: string | null;
+  message: string;
+  count: number;
+  firstOccurrence: number;
+  lastOccurrence: number;
+  samples: ReadonlyArray<ErrorSample>;
+  /** `count / totalErrors`, scaled 0..100. `0` when there are zero errors total. */
+  percentage: number;
+}
+interface ErrorTrackerConfig {
+  /** Hard cap on distinct signatures. Defaults to 50. New signatures past the cap are dropped. */
+  maxPatterns?: number;
+  /** Hard cap on samples retained per pattern. Defaults to 5. Excess samples are dropped. */
+  maxSamplesPerPattern?: number;
+}
+/** Default caps — match giga-test for compatibility. */
+declare const DEFAULT_MAX_PATTERNS = 50;
+declare const DEFAULT_MAX_SAMPLES_PER_PATTERN = 5;
+declare class ErrorTracker {
+  private readonly patterns;
+  private readonly maxPatterns;
+  private readonly maxSamplesPerPattern;
+  private droppedSignatures;
+  constructor(config?: ErrorTrackerConfig);
+  /**
+   * Record one error. Same `(errorType, field, message)` triple bumps the
+   * existing bucket; a new triple opens a new one (subject to {@link maxPatterns}).
+   *
+   * `field` is optional — pass `undefined` for errors not tied to a single
+   * column (e.g. parse errors at row level). Internally normalised to the
+   * literal string `'GENERAL'` so it shares a bucket with other genericised
+   * errors of the same type+message.
+   */
+  addError(rowNumber: number, errorType: string, message: string, field: string | undefined, rowData: ImportJsonValue): void;
+  /** Total count across every pattern. Counts errors, not patterns. */
+  getTotalErrorCount(): number;
+  /** Number of distinct signatures retained in the map (≤ `maxPatterns`). */
+  getDistinctPatternCount(): number;
+  /**
+   * Number of NEW signatures dropped because the map was already at
+   * capacity. Surfacing this in the import_run summary tells the operator
+   * "the top-N was saturated — there's a long tail you're not seeing".
+   */
+  getDroppedSignatureCount(): number;
+  /**
+   * Top patterns sorted by descending count, capped at `maxPatterns`.
+   * Stable secondary order is insertion order (Map iteration order is
+   * insertion order; Array.sort is stable in V8).
+   */
+  getTopPatterns(): ErrorPattern[];
+  /**
+   * Compact JSON-serialisable snapshot for `import_run.errorSummary`.
+   * Aside from the patterns array, includes the totals so a reader of
+   * just this column doesn't have to re-derive them.
+   */
+  snapshot(): ErrorTrackerSnapshot;
+}
+interface ErrorTrackerSnapshot {
+  totalErrors: number;
+  distinctPatterns: number;
+  droppedSignatures: number;
+  patterns: ErrorPattern[];
+}
+/**
+ * Merge two independently-produced `ErrorTrackerSnapshot`s into one — e.g.
+ * the runner's per-row `warningSummary` plus a worker-level signal computed
+ * AFTER the run completes (F010 / #602: the price-drift anomaly check runs
+ * in `worker.ts`, after `runImport` returns, using a fresh single-pattern
+ * tracker of its own). Patterns concatenate rather than re-dedup: the two
+ * snapshots are assumed to use disjoint `errorType` signature spaces (a
+ * worker-level tracker uses a type — `price_anomaly` — the runner's own
+ * warning aggregation never emits), so counts sum directly with no risk of
+ * double-counting one bucket as two.
+ */
+declare function mergeErrorTrackerSnapshots(a: ErrorTrackerSnapshot, b: ErrorTrackerSnapshot): ErrorTrackerSnapshot;
+//#endregion
+//#region src/price-stats.d.ts
+/**
+ * Price-distribution stats + prior-run anomaly detection (F010 / issue #602).
+ *
+ * The import pipeline's existing anomaly detection — register-handler.ts's
+ * "6b. Header-presence drift check" — only catches STRUCTURAL drift: a
+ * mapped column vanishing (or getting renamed) from the header row. It does
+ * NOT catch VALUE drift: a supplier silently flipping their decimal
+ * separator (e.g. `1.234,56` → `1234.56`) or thousands separator WITHOUT
+ * touching a single column header. Since the saved
+ * `import_mapping.numberFormat` (`{decimal, thousands}`) is unchanged,
+ * every price in that run parses as a NUMERICALLY VALID but WRONG value —
+ * silently corrupting the catalog's prices. Nothing upstream of this module
+ * flags it.
+ *
+ * This module computes a per-run price-distribution snapshot (`PriceStats`)
+ * over `PartsDocument.purchase_price_minor` — the DERIVED cost field,
+ * rather than the raw net/list price columns, because both the direct-net
+ * AND two-file derivation paths in `mapping/parts-schema.ts` (step 4) funnel
+ * through it, so a decimal-format flip on EITHER source column surfaces
+ * here regardless of which pricing path a supplier's feed uses.
+ *
+ * Collection (`PriceStatsCollector`) happens in the SAME streaming pass
+ * `runner.ts` already uses to build `totals` — one `.record()` call per
+ * ES-accepted row, no second pass over the file, no extra DB round-trip.
+ * The prior-run comparison (`detectPriceAnomaly`) happens in `worker.ts`
+ * AFTER the run completes, because it needs `AdminClient` to look the
+ * prior run up — this module stays DB-agnostic, pure data + pure functions.
+ */
+/** A run's price-distribution snapshot. Values are in MINOR units (cents), matching `PartsDocument.purchase_price_minor`'s own scale. */
+interface PriceStats {
+  /** ES-accepted rows considered — same accounting as `RunImportResult.rowsSucceeded`. */
+  count: number;
+  /** Fraction (0..1) of `count` whose price sample was absent (e.g. a list-only row with no resolvable purchase cost — the same rows that carry a `no_purchase_price` warning). `0` when `count` is `0`. */
+  nullRate: number;
+  /** Minimum of the non-null samples, in minor units. `null` when there are zero non-null samples. */
+  min: number | null;
+  /** Median of the non-null samples, in minor units. `null` when there are zero non-null samples. */
+  median: number | null;
+  /** Maximum of the non-null samples, in minor units. `null` when there are zero non-null samples. */
+  max: number | null;
+}
+/** The all-zero/empty snapshot — a run that produced zero ES-accepted rows. */
+declare const EMPTY_PRICE_STATS: PriceStats;
+/**
+ * Bounded sample cap for the min/median/max computation — the same
+ * "bounded, uniform sample" philosophy `ErrorTracker` uses top-N patterns
+ * for. 50,000 floats is ~400KB, trivial next to the ES bulk-write buffers
+ * already in flight for a run this size, and far more than needed for a
+ * stable median estimate. A feed with more ES-accepted rows than this cap
+ * still gets an EXACT `count` / `nullRate` (those are running counters, not
+ * sample-bounded) — only min/median/max become an approximation over a
+ * UNIFORM RANDOM sample of `PRICE_SAMPLE_CAP` non-null values, not "the
+ * first N encountered" (see `record`'s reservoir-sampling note for why
+ * order-independence matters specifically for this module's job).
+ */
+declare const PRICE_SAMPLE_CAP = 50000;
+/**
+ * Accumulates price samples during a run's streaming pass. `record` is O(1)
+ * (amortized array push below the cap, one RNG draw above it); `snapshot`
+ * sorts the retained samples once at the end — cheap relative to the ES
+ * bulk-write the run already performs every batch.
+ */
+declare class PriceStatsCollector {
+  private count;
+  private nullCount;
+  private nonNullSeen;
+  private readonly samples;
+  /**
+   * Record one ES-accepted row's price sample. Call exactly once per
+   * accepted row (mirrors the `rowsWarned` / `pairs` accounting in
+   * `runner.ts`'s flush loop) — `undefined` means "this row had no price to
+   * sample" and counts toward `nullRate` rather than being skipped.
+   *
+   * Reservoir sampling (Algorithm R) past the cap, not "first N wins": each
+   * new non-null value replaces a uniformly-random existing slot with
+   * probability `PRICE_SAMPLE_CAP / nonNullSeen`, giving every non-null
+   * sample seen so far an equal chance of surviving into the final
+   * reservoir. Order-independence matters here specifically — the anomaly
+   * this module exists to catch (a supplier's decimal-format flip) could
+   * start partway through a feed larger than the cap (e.g. a corrupted
+   * re-export appended to an otherwise-normal catalogue), and a first-N
+   * sample would never see a single row past the cutover point, silently
+   * missing exactly the drift F010 was built to detect.
+   */
+  record(value: number | undefined): void;
+  snapshot(): PriceStats;
+}
+/**
+ * Anomaly threshold: a run whose median purchase price is >= 10x higher OR
+ * <= 1/10 (0.1x) of the prior successful run's median (for the same
+ * transform + supplier + brand) is flagged.
+ *
+ * Chosen generously above the range a real repricing plausibly covers
+ * (even an aggressive markup/discount renegotiation is a low-single-digit
+ * multiplier — 2x-3x would already be an extreme outlier for a legitimate
+ * price change) but comfortably BELOW the ~100x-1000x magnitude a
+ * decimal/thousands-separator flip produces for typical EUR part prices:
+ * `"1.234,56"` (a European-format €1234.56) misparsed with dot-decimal
+ * conventions reads as `1.234,56` → `1234.56`, a ~1000x inflation if the
+ * true value was e.g. €1.23; a thousands-separator flip on `"1234,56"`
+ * (true €1234.56) misparsed as `123456` (dropping the decimal entirely) is
+ * a ~100x inflation. 10x leaves a wide safety margin against false
+ * positives on genuine price changes while still comfortably catching
+ * every realistic decimal/thousands-separator flip.
+ */
+declare const PRICE_ANOMALY_THRESHOLD_MULTIPLIER = 10;
+interface PriceAnomaly {
+  /** `curr.median / prev.median`. */
+  ratio: number;
+  /** The prior run's median (minor units). */
+  prevMedian: number;
+  /** This run's median (minor units). */
+  currMedian: number;
+}
+/**
+ * Compare two runs' price stats for a magnitude-scale anomaly. Returns
+ * `null` when there's nothing to compare (either side has no median — zero
+ * non-null price samples, e.g. a schema/transform that never emits a price
+ * sample, or a run with zero ES-accepted rows) or the ratio falls within
+ * the safe band.
+ */
+declare function detectPriceAnomaly(prev: PriceStats, curr: PriceStats): PriceAnomaly | null;
+/**
+ * Narrow an `unknown` value (a `field.json()` column read back through
+ * `AdminClient`, typed `JsonValue | undefined` on the DTO) to `PriceStats`,
+ * or `null` when it doesn't structurally match — e.g. a run written before
+ * this field existed, or a run whose transform never emitted a price
+ * sample so `min`/`median`/`max` serialised as `null`. Defensive, matching
+ * the `params` narrowing pattern already used in `worker.ts`.
+ */
+declare function parsePriceStats(value: unknown): PriceStats | null;
+//#endregion
+//#region src/runner.d.ts
+/** Soft default; chosen to match giga-test. ES bulk requests over ~5MB get split server-side anyway. */
+declare const DEFAULT_BATCH_SIZE = 1000;
+interface RunImportOptions<TDoc> {
+  /** UUID of the `import_run` row driving this run. Forwarded to every transform invocation. */
+  importRunId: string;
+  /** Operator-supplied label for the run. Forwarded to the transform context. */
+  runLabel: string;
+  /** Opaque per-run params copied from `import_run.params`. */
+  params: Record<string, unknown>;
+  /** Transform applied to every parsed row. */
+  transform: RowTransform<TDoc>;
+  /** Feed reader options — file path + format (csv/tsv/xlsx) + delimiter/header config. */
+  feed: ReadFeedOptions;
+  /** ES client (low-level shape from `@murumets-ee/search-elasticsearch`). */
+  esClient: EsClientLike;
+  /** Index alias to write to. Per D6, callers always pass an alias, never a physical index. */
+  esIndex: string;
+  /** Rows per `bulkUpsert` call. Default {@link DEFAULT_BATCH_SIZE}. */
+  batchSize?: number;
+  /**
+   * Callback invoked after every batch. The handler in `worker.ts`
+   * forwards this to `ctx.updateProgress` for the queue UI; tests
+   * inspect it directly. Synchronous + cheap so a slow callback can't
+   * back-pressure the importer.
+   */
+  onProgress?: (progress: ImportRunProgress) => void;
+  /** Optional: stop processing after this many rows. Tests use it; production passes `undefined`. */
+  rowLimit?: number;
+  /** Abort signal threaded into the underlying ES client request — cooperative cancel. */
+  signal?: AbortSignal;
+  /** Optional ErrorTracker config (caps). Default: top-50 patterns × 5 samples. */
+  errorTracker?: ErrorTracker;
+  /**
+   * Optional separate tracker for NON-FATAL row warnings (commerce-demo PR4 —
+   * `no_purchase_price`). Distinct from {@link errorTracker} so warnings never
+   * inflate `totals.failed` / the error summary: a warned row is a SUCCESS whose
+   * doc is indexed. Default: a fresh tracker with the same caps.
+   */
+  warningTracker?: ErrorTracker;
+  /**
+   * Optional price-stats collector (F010 / #602) — accumulates a
+   * min/median/max/nullRate distribution over ES-accepted rows' optional
+   * `priceSample`s in the SAME streaming pass that builds `totals`. Default:
+   * a fresh collector. Tests can inject their own to inspect it directly —
+   * mirrors the {@link errorTracker} / {@link warningTracker} DI shape.
+   */
+  priceStats?: PriceStatsCollector;
+  /**
+   * Optional structural logger for per-batch telemetry. Receives
+   * `{ batch, ms, succeeded, failed, rowsRead, rowsPerSecond }` after
+   * every flush, plus a one-shot `started` and `finished` event. The
+   * worker forwards `app.logger.child({ pkg: 'imports' })`; tests
+   * leave it undefined.
+   *
+   * Why: without per-batch visibility, a stalled run looks identical
+   * to "first batch in progress" forever — the only feedback is the
+   * queue's `progress` jsonb (which writes after each batch) and the
+   * eventual `succeeded`/`failed` job event. Adding this lets the
+   * operator tail the worker log and immediately see whether ES is
+   * rate-limiting, the transform is rejecting every row, or the
+   * stream is actually progressing.
+   */
+  logger?: {
+    info: (data: Record<string, unknown>, msg: string) => void;
+    warn: (data: Record<string, unknown>, msg: string) => void;
+  };
+}
+/**
+ * Progress payload written to `toolkit_jobs.progress` after every batch.
+ * Caps + flush rules live on the queue's `updateProgress` debounce —
+ * callers don't need to throttle.
+ */
+interface ImportRunProgress {
+  rowsRead: number;
+  rowsSucceeded: number;
+  rowsFailed: number;
+  rowsSkipped: number;
+  /** Rows indexed WITH a non-fatal warning (commerce-demo PR4 — `no_purchase_price`). */
+  rowsWarned: number;
+  batchesCompleted: number;
+  /** Wall-clock seconds since the runner started. */
+  elapsedSeconds: number;
+  /** Rows / second, computed at every batch. */
+  rowsPerSecond: number;
+  /** Distinct error patterns currently held by the tracker. Saturates at the cap. */
+  distinctErrorPatterns: number;
+  /**
+   * Total data-row count from a pre-pass scan, when the worker computed one.
+   * Lets the UI render `processed / total` percent. Absent when the worker
+   * skipped the count (file too large for the count budget, count failed,
+   * etc.) — UI must treat it as optional.
+   */
+  totalRows?: number;
+}
+/**
+ * Final result returned by {@link runImport}. The handler writes these
+ * onto the `import_run` row alongside the ErrorTracker snapshot.
+ */
+interface RunImportResult {
+  /** Total rows read from the file (excludes skipped empty lines). */
+  rowsRead: number;
+  /** Rows the transform turned into a successful doc AND the ES cluster acknowledged. */
+  rowsSucceeded: number;
+  /**
+   * Rows that the transform rejected (`{ kind: 'error' }`) OR that ES
+   * rejected on bulk-write (per-doc failure). Both are aggregated by
+   * `errorTracker` for the import_run summary.
+   */
+  rowsFailed: number;
+  /** Rows that the transform skipped (`{ kind: 'skip' }`) — header noise, blank lines, intentional drop. */
+  rowsSkipped: number;
+  /**
+   * Rows that succeeded (doc indexed) but carried ≥1 non-fatal warning —
+   * counted SEPARATELY from `rowsSucceeded`/`rowsFailed` (a warned row is also
+   * counted in `rowsSucceeded`). commerce-demo PR4: `no_purchase_price`.
+   */
+  rowsWarned: number;
+  /** Number of `bulkUpsert` calls made. */
+  batchesCompleted: number;
+  /** Final value of {@link ErrorTracker.snapshot}. */
+  errors: ReturnType<ErrorTracker['snapshot']>;
+  /** Aggregated NON-FATAL warnings (commerce-demo PR4). Empty when no row warned. */
+  warnings: ReturnType<ErrorTracker['snapshot']>;
+  /**
+   * Directional supersession pairs harvested as a side-output (commerce-demo PR6,
+   * SD007). Buffered in memory from ES-ACCEPTED rows only (same accounting as
+   * warnings — a row whose doc was rejected by ES contributes no pair). Empty unless
+   * a schema emits them (the `parts` schema, when a row's `superseding_code` is set).
+   * The worker hands these to an injected `pairSink` AFTER the run for the commerce
+   * replacement-merge engine; the runner itself never writes the consumer's tables.
+   */
+  pairs: readonly DirectionalPair[];
+  /**
+   * Doc-patches applied by the phase-2 pass (commerce-demo PR7): a `<code>*S`
+   * core-fee row is not a saleable doc — it emits a {@link DocPatch} that merges its
+   * deposit onto the PARENT part's `core_charge_minor`. The runner buffers these
+   * during the stream (sparse) and applies them via `bulkPartialUpdate` AFTER the ES
+   * pass. `patchesApplied` = targets the cluster merged; `patchesOrphaned` = patches
+   * whose parent genuinely wasn't in the index (a `document_missing`/404 → an
+   * `orphan_core_charge` warning). An INFRA failure (whole-batch transport error, or a
+   * non-404 per-doc failure) is NOT counted as orphaned — it lands in a distinct
+   * `patch_attach_failed` warning bucket so it can't masquerade as a data orphan. Zero
+   * unless a schema emits patches.
+   */
+  patchesApplied: number;
+  patchesOrphaned: number;
+  /**
+   * Price-distribution snapshot (F010 / #602) accumulated from ES-accepted
+   * rows' `priceSample`s during this same streaming pass. See `price-stats.ts`.
+   * `count` is `0` (all fields `null`/`0`) for a run whose schema never emits
+   * a price sample.
+   */
+  priceStats: PriceStats;
+}
+/**
+ * Apply the runner against a feed file. Stops on rowLimit OR end-of-file
+ * OR if `signal` aborts. Throws if the streaming reader / ES client
+ * throws — caller (the queue handler) catches that and writes
+ * `import_run.status = 'failed'` with the error message in
+ * `errorSummary.fatal`.
+ */
+declare function runImport<TDoc>(options: RunImportOptions<TDoc>): Promise<RunImportResult>;
+//#endregion
+//#region src/worker.d.ts
+/**
+ * Structural subset of `JobContext` the handler reads. Declared
+ * locally so we don't depend on a `JobHandler` import — the queue
+ * package doesn't re-export it from any subpath today (matches the
+ * `SendEmailJobContext` pattern in `@murumets-ee/notifications`).
+ *
+ * Function-param contravariance lets a handler returned with this
+ * narrower shape be assigned to the wider `JobHandler<TPayload>` that
+ * `registerJob` expects.
+ */
+interface ImportsRunJobContext {
+  id: string;
+  payload: ImportsRunJobPayload;
+  updateProgress(data: ImportRunProgress): void;
+}
+/**
+ * Payload schema. `importRunId` is the `import_run.id` UUID — the worker
+ * looks up everything else (filePath, transformName, params, …) from
+ * that row. Keeping the payload tiny means the queue's progress JSON
+ * column never bloats with feed metadata duplicated across `toolkit_jobs`.
+ */
+declare const importsRunJobPayloadSchema: z.ZodObject<{
+  importRunId: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+  importRunId: string;
+}, {
+  importRunId: string;
+}>;
+type ImportsRunJobPayload = z.infer<typeof importsRunJobPayloadSchema>;
+/**
+ * Job definition. Consumers register their handler against this with
+ * `registerJob(importsRunJob, createRunImportHandler({...}))`.
+ *
+ * `defaultRetries: 0` — re-running a multi-batch import against the
+ * same `import_run.id` while the previous handler may still be writing
+ * is a footgun (duplicate batches, double-counted progress). The
+ * operator handles retries explicitly via PR 8a's "retry" button by
+ * creating a new `import_run` row. When the resumable design lands
+ * (post-PoC), retries become safe to enable.
+ */
+declare const importsRunJob: JobDefinition<ImportsRunJobPayload>;
+/**
+ * Resolves the ES client at handler-invocation time. A function (rather
+ * than the bare `EsClientLike`) so the consumer can lazy-construct the
+ * client — typical Next.js setups create the ES connection in a
+ * route-handler initialiser, not at plugin-init time.
+ */
+type EsClientResolver = () => EsClientLike | Promise<EsClientLike>;
+/**
+ * Ensure the destination index/alias exists before the first bulk-write.
+ * Invoked once per job, BEFORE {@link runImport}. Idempotent: the canonical
+ * implementation (`ensureAliasedIndex` in `@murumets-ee/search-elasticsearch`)
+ * no-ops when the alias already resolves.
+ *
+ * **Why it lives here.** This is the home of the foot-gun it guards: a worker
+ * that bulk-writes to alias `<x>` before the alias exists makes ES
+ * auto-create a physical `<x>` index with default `text` mappings, breaking
+ * aggregations forever. The bootstrap used to run at app boot, which turned
+ * "ES is down" into a full-admin 500 (issue #610). Moving it onto the import
+ * path keeps the guarantee while scoping ES availability to imports/search —
+ * the rest of the admin boots and runs with ES down.
+ */
+type EnsureIndexResolver = () => Promise<unknown>;
+/**
+ * Resolves the value of `import_run.filePath` (whatever the upload
+ * route persisted there — typically a storage adapter key) to a
+ * readable LOCAL filesystem path that {@link runImport} can hand to
+ * `node:fs.createReadStream`.
+ *
+ * **Why this exists:** the streaming reader (`streamFeed`) reads from
+ * local disk via `createReadStream`. The upload route may persist a
+ * remote-storage object key (R2, S3, …) on `import_run.filePath`
+ * because that's what `@murumets-ee/storage` returns. Without a
+ * resolver, `createReadStream('uploads/2026/05/<uuid>/feed.txt')`
+ * crashes with `ENOENT`. The resolver is the documented integration
+ * point — typical wiring downloads the storage object to a tmpfile
+ * and returns its path.
+ *
+ * **`cleanup`** runs after the run finishes (success OR failure). The
+ * worker awaits it best-effort — a failed cleanup logs but does not
+ * crash the run.
+ *
+ * **No-resolver fallback:** when `resolveFilePath` is unset, the
+ * worker treats `import_run.filePath` as already a local FS path
+ * (back-compat with the original PoC design where uploads landed on
+ * local disk). This stays valid for fixture-driven tests + on-disk
+ * deployments.
+ */
+type FilePathResolver = (storageKey: string) => Promise<{
+  localPath: string;
+  cleanup?: () => Promise<void>;
+}>;
+/**
+ * Context handed to a {@link PairSink} alongside the harvested pairs. Carries the
+ * run identity so the sink can build `source` attribution
+ * (`feed:<supplierId>:<importBatchId>`, D20) and scope its writes.
+ */
+interface PairSinkContext {
+  importRunId: string;
+  /** The `import_run.params` object — the sink reads `supplierId` / `importBatchId`. */
+  params: Record<string, unknown>;
+  /**
+   * The run's structured logger (child), forwarded so the sink's INTERNAL telemetry —
+   * a params-parse miss that skips ingestion, an over-merge flag — is visible. Without it
+   * a misconfigured run looks successful from the worker's vantage (a params-miss returns
+   * normally, so the worker logs "applied"). Optional; the sink no-ops its logs if absent.
+   */
+  logger?: Logger;
+}
+/**
+ * Optional post-run consumer for the directional supersession pairs the runner
+ * harvested as a side-output (commerce-demo PR6, SD007). Invoked ONCE after a
+ * successful run with the full in-memory pair buffer (read-once / merge-after) so
+ * the consumer (the commerce replacement-merge engine) can build equivalence groups
+ * and bulk-write them to Postgres.
+ *
+ * **Why injected, not imported.** The merge writes the CONSUMER's Postgres tables,
+ * but the runner + this worker live in the `@murumets-ee/imports` leaf, which must
+ * not depend on commerce. The app composes the sink (commerce exports it, the app
+ * passes it to `imports({ pairSink })`) — the same injection shape as `esClient` /
+ * `resolveFilePath` / `ensureIndex`. The boundary holds.
+ *
+ * **Best-effort.** A sink throw is logged and SWALLOWED — the catalogue import
+ * (the ES write) has already succeeded and must not be undone by a secondary
+ * replacement-ingestion failure. The merge is idempotent, so a re-import recovers.
+ */
+type PairSink = (pairs: readonly DirectionalPair[], ctx: PairSinkContext) => Promise<void>;
+interface RunImportHandlerConfig {
+  /** AdminClient over the `import_run` entity. */
+  importRuns: ImportRunClient;
+  /** Transform registry to dispatch against. Defaults to the process-global singleton. */
+  transforms: TransformRegistry;
+  /** Resolver for the ES client. */
+  esClient: EsClientResolver;
+  /** ES alias / index to bulk-write into. Per D6 callers always pass an alias. */
+  esIndex: string;
+  /**
+   * Optional bootstrap for the destination index/alias, run once per job
+   * before any bulk-write. See {@link EnsureIndexResolver}. Omit for
+   * deployments that bootstrap the index out-of-band (e.g. a reindex job).
+   */
+  ensureIndex?: EnsureIndexResolver;
+  /**
+   * Optional resolver for `import_run.filePath`. See {@link FilePathResolver}.
+   * Required when uploads land in remote storage (R2/S3); optional for
+   * on-disk PoC setups.
+   */
+  resolveFilePath?: FilePathResolver;
+  /**
+   * Optional consumer for the directional supersession pairs harvested as a
+   * side-output (commerce-demo PR6, SD007). See {@link PairSink}. Omit for
+   * deployments that don't ingest replacement codes — the pairs are simply
+   * discarded after the run.
+   */
+  pairSink?: PairSink;
+  /** Optional structured logger. Defaults to silent. */
+  logger?: Logger;
+}
+/**
+ * Build the `JobHandler` for {@link importsRunJob}. The returned
+ * handler is what gets passed to `registerJob`.
+ *
+ * The body is wrapped in `runAsCli` so AdminClient calls inside have a
+ * synthetic `cli` admin context — `auditable()` records `updatedBy:
+ * 'cli'` rather than NULL, and the firewall checker passes. This is
+ * the documented worker entry-point pattern (see `runAsCli` JSDoc in
+ * `@murumets-ee/core`).
+ */
+declare function createRunImportHandler(config: RunImportHandlerConfig): (job: ImportsRunJobContext) => Promise<void>;
+interface ImportsRunDeadListenerConfig {
+  importRuns: ImportRunClient;
+  /** Optional structured logger. */
+  logger?: Logger;
+}
+/**
+ * Build a {@link QueueTerminalNotifier} that mirrors `imports:run`
+ * dead-job events into `import_run.status='failed'`.
+ *
+ * **Why this exists.** The `imports:run` handler's normal failure path
+ * already calls `markFailed` before re-throwing, so a synchronous
+ * handler error reaches the operator-visible `failed` status without
+ * help from anyone. The case THIS listener catches is the one
+ * `markFailed` never gets to: the worker process is killed mid-handler
+ * (OOM, container restart, kill -9), the queue's lock-recovery sweep
+ * eventually reclaims the row, and the queue ultimately marks the job
+ * `dead` (defaultRetries: 0 + repeated process death = dead-letter
+ * with `import_run.status` still `running`). Without this listener
+ * the row sits in the operator's Active panel forever.
+ *
+ * **Generic primitive.** This is a thin subscriber over the queue's
+ * multi-subscriber `QueueTerminalNotifier` registry — the same shape
+ * graphile-worker (`job:failed`) and BullMQ (`failed`) expose. Other
+ * domain plugins that mirror queue state (ticketing-reply, future
+ * exports/reports/...) can register their own listeners exactly the
+ * same way without coordinating via a per-domain sweep job.
+ *
+ * **Filtering.** The listener is invoked for EVERY dead event in the
+ * process. We early-return on any `jobType` other than `imports:run`
+ * so the per-event cost on unrelated jobs is a single property read.
+ *
+ * **Idempotency.** A dead event delivered after the row has already
+ * reached its own terminal state (succeeded / failed / cancelled) is
+ * ignored — the only transition we make is `running → failed`. This
+ * keeps a late replay or out-of-order delivery from clobbering a
+ * row whose worker reached `markFailed` faster than the queue's
+ * dead-letter dispatch.
+ *
+ * The handler is wrapped in `runAsCli` so AdminClient calls inside
+ * have a synthetic CLI admin context — same pattern as
+ * {@link createRunImportHandler}.
+ */
+declare function createImportsRunDeadListener(config: ImportsRunDeadListenerConfig): QueueTerminalNotifier;
+//#endregion
+export { ErrorSample as A, StreamFeedOptions as B, PriceStats as C, DEFAULT_MAX_PATTERNS as D, parsePriceStats as E, FeedReaderConfig as F, ImportRun as G, countDataRows as H, ReadFeedOptions as I, ImportRunClient as K, feedReaderConfigSchema as L, ErrorTrackerConfig as M, ErrorTrackerSnapshot as N, DEFAULT_MAX_SAMPLES_PER_PATTERN as O, mergeErrorTrackerSnapshots as P, readFeed as R, PriceAnomaly as S, detectPriceAnomaly as T, streamFeed as U, StreamFeedRow as V, IMPORT_RUN_STATUSES as W, RunImportResult as _, ImportsRunJobContext as a, PRICE_ANOMALY_THRESHOLD_MULTIPLIER as b, PairSinkContext as c, createRunImportHandler as d, importsRunJob as f, RunImportOptions as g, ImportRunProgress as h, ImportsRunDeadListenerConfig as i, ErrorTracker as j, ErrorPattern as k, RunImportHandlerConfig as l, DEFAULT_BATCH_SIZE as m, EsClientResolver as n, ImportsRunJobPayload as o, importsRunJobPayloadSchema as p, ImportRunStatus as q, FilePathResolver as r, PairSink as s, EnsureIndexResolver as t, createImportsRunDeadListener as u, runImport as v, PriceStatsCollector as w, PRICE_SAMPLE_CAP as x, EMPTY_PRICE_STATS as y, readPersistedFeedConfig as z };
+//# sourceMappingURL=worker-CRH_PjKO.d.mts.map