@murumets-ee/imports 0.0.0-preview-20261014110621

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/LICENSE +94 -0
  2. package/dist/analyze/index.d.mts +265 -0
  3. package/dist/analyze/index.d.mts.map +1 -0
  4. package/dist/analyze/index.mjs +2 -0
  5. package/dist/analyze/index.mjs.map +1 -0
  6. package/dist/carmaker-feed-B-QYr3PI.mjs +2 -0
  7. package/dist/carmaker-feed-B-QYr3PI.mjs.map +1 -0
  8. package/dist/carmaker-feed-CWrZLYUt.d.mts +59 -0
  9. package/dist/carmaker-feed-CWrZLYUt.d.mts.map +1 -0
  10. package/dist/cleanup-DDljBe0Z.mjs +2 -0
  11. package/dist/cleanup-DDljBe0Z.mjs.map +1 -0
  12. package/dist/engine-B0ExE4jL.mjs +2 -0
  13. package/dist/engine-B0ExE4jL.mjs.map +1 -0
  14. package/dist/engine-tkbUXKxB.d.mts +609 -0
  15. package/dist/engine-tkbUXKxB.d.mts.map +1 -0
  16. package/dist/index-Bm69N5DW.d.mts +97 -0
  17. package/dist/index-Bm69N5DW.d.mts.map +1 -0
  18. package/dist/index.d.mts +390 -0
  19. package/dist/index.d.mts.map +1 -0
  20. package/dist/index.mjs +2 -0
  21. package/dist/index.mjs.map +1 -0
  22. package/dist/mapping/index.d.mts +42 -0
  23. package/dist/mapping/index.d.mts.map +1 -0
  24. package/dist/mapping/index.mjs +1 -0
  25. package/dist/mapping-transform-CO4MNhMe.mjs +2 -0
  26. package/dist/mapping-transform-CO4MNhMe.mjs.map +1 -0
  27. package/dist/matching/index.d.mts +2 -0
  28. package/dist/matching/index.mjs +1 -0
  29. package/dist/matching-Sr0z4HvN.mjs +2 -0
  30. package/dist/matching-Sr0z4HvN.mjs.map +1 -0
  31. package/dist/parts-schema-M1AtjNJt.d.mts +61 -0
  32. package/dist/parts-schema-M1AtjNJt.d.mts.map +1 -0
  33. package/dist/parts-schema-QqLci_Pl.mjs +2 -0
  34. package/dist/parts-schema-QqLci_Pl.mjs.map +1 -0
  35. package/dist/plugin.d.mts +148 -0
  36. package/dist/plugin.d.mts.map +1 -0
  37. package/dist/plugin.mjs +2 -0
  38. package/dist/plugin.mjs.map +1 -0
  39. package/dist/publish-DCXIPwTE.mjs +2 -0
  40. package/dist/publish-DCXIPwTE.mjs.map +1 -0
  41. package/dist/read-feed-DsLgRDNR.mjs +2 -0
  42. package/dist/read-feed-DsLgRDNR.mjs.map +1 -0
  43. package/dist/realtime/index.d.mts +123 -0
  44. package/dist/realtime/index.d.mts.map +1 -0
  45. package/dist/realtime/index.mjs +1 -0
  46. package/dist/rolldown-runtime-DK3Fl9T5.mjs +1 -0
  47. package/dist/runner-BnFvID1z.mjs +2 -0
  48. package/dist/runner-BnFvID1z.mjs.map +1 -0
  49. package/dist/schema-registry-DB5aIZMN.mjs +2 -0
  50. package/dist/schema-registry-DB5aIZMN.mjs.map +1 -0
  51. package/dist/storage-resolver.d.mts +32 -0
  52. package/dist/storage-resolver.d.mts.map +1 -0
  53. package/dist/storage-resolver.mjs +2 -0
  54. package/dist/storage-resolver.mjs.map +1 -0
  55. package/dist/storage.d.mts +31 -0
  56. package/dist/storage.d.mts.map +1 -0
  57. package/dist/storage.mjs +2 -0
  58. package/dist/storage.mjs.map +1 -0
  59. package/dist/transform-Cu5YEoOl.mjs +2 -0
  60. package/dist/transform-Cu5YEoOl.mjs.map +1 -0
  61. package/dist/transform-WeiiLdYP.d.mts +246 -0
  62. package/dist/transform-WeiiLdYP.d.mts.map +1 -0
  63. package/dist/transforms.d.mts +2 -0
  64. package/dist/transforms.mjs +1 -0
  65. package/dist/worker-CRH_PjKO.d.mts +940 -0
  66. package/dist/worker-CRH_PjKO.d.mts.map +1 -0
  67. package/dist/worker.d.mts +2 -0
  68. package/dist/worker.mjs +2 -0
  69. package/dist/worker.mjs.map +1 -0
  70. package/dist/xlsx-reader-D9Ntxm29.mjs +2 -0
  71. package/dist/xlsx-reader-D9Ntxm29.mjs.map +1 -0
  72. package/package.json +81 -0
@@ -0,0 +1,97 @@
1
+ //#region src/matching/auto-assign.d.ts
2
+ /**
3
+ * Auto-assignment of a previewed feed's columns to target fields — the wizard's first guess
4
+ * when an operator uploads a sample file.
5
+ *
6
+ * Kept pure (no React, no server imports) so the matching rules are unit-testable against real
7
+ * supplier headers instead of only being visible by driving the UI.
8
+ *
9
+ * ## Why this is not a substring match
10
+ *
11
+ * The previous rule accepted a match when either string contained the other, which is wrong in
12
+ * both directions at once. It fired where it should not — `Barcode` contains `code`, so the
13
+ * barcode column was offered as the REQUIRED part-code field while the actual code column
14
+ * (`ArticleName`, `partno`) matched nothing — and it stayed silent where it mattered, because
15
+ * no real feed names a column after our internal key. Measured against the three ground-truth
16
+ * supplier samples, it assigned two columns correctly, one wrongly, and missed the rest,
17
+ * including every supersession column. A missed price column is loud; a missed supersession
18
+ * column is silent (prices import, the run succeeds, the replacement tables stay empty), which
19
+ * is how the supersession feature shipped switched-off and stayed that way.
20
+ *
21
+ * ## The rules
22
+ *
23
+ * 1. A field's candidate spellings are its `key`, its `label`, and its declared `aliases`, each
24
+ * reduced to lowercase alphanumerics.
25
+ * 2. A column matches a candidate only on a TOKEN BOUNDARY — the header is split on
26
+ * punctuation, whitespace and camelCase humps, and a candidate must equal a contiguous run
27
+ * of those tokens. So `Net Price EUR` and `NetPrice/Discount` still reach `net_price`, while
28
+ * `Barcode` can no longer reach `code`, because `code` is not one of its tokens.
29
+ * 3. Every (column, field) match competes globally and the strongest wins: a whole-header hit
30
+ * beats a partial one, and a longer candidate beats a shorter one. Assignment is greedy over
31
+ * that order, so `Description1` claims `description1` even when a plain `Description` column
32
+ * sits earlier in the file.
33
+ * 4. One target is used at most once. Two columns claiming the same field would be rejected by
34
+ * the mapping validator at save time, so the weaker claimant is left unassigned for the
35
+ * operator to resolve rather than silently producing an unsaveable mapping.
36
+ */
37
+ /** The subset of a target field this matcher reads. */
38
+ interface AutoAssignField {
39
+ key: string;
40
+ label: string;
41
+ aliases?: readonly string[] | undefined;
42
+ }
43
+ /**
44
+ * Split a header into comparison tokens: on punctuation/whitespace, and on camelCase humps so
45
+ * `ArticleName` → `article` + `name` and `CurrencyCd` → `currency` + `cd`. Truncated to
46
+ * {@link MAX_HEADER_TOKENS}.
47
+ *
48
+ * **Acronym boundaries are deliberately NOT split.** Adding `([A-Z]+)([A-Z][a-z])` looks like a
49
+ * free improvement — it makes `EANCode` read as `ean` + `code` — and it re-opens this module's
50
+ * founding bug: the standalone `code` token then outscores `barcode`'s `ean` alias (4 vs 3), so
51
+ * the BARCODE column gets offered as the required part-code field, silently, exactly as the old
52
+ * substring matcher did. It generalises to any `<Acronym>Code` header, which no alias list can
53
+ * enumerate. Compound spellings that matter are handled as whole-header aliases instead, which
54
+ * beat any partial run.
55
+ */
56
+ declare function tokenizeHeader(header: string): string[];
57
+ /**
58
+ * Suggest a target field for each column. Returns `column → target key`, omitting any column
59
+ * with no confident match — an absent entry means "the operator decides", never "no mapping
60
+ * needed".
61
+ */
62
+ declare function suggestTargets(columns: readonly string[], fields: readonly AutoAssignField[]): Map<string, string>;
63
+ //#endregion
64
+ //#region src/matching/header-snapshot.d.ts
65
+ /**
66
+ * The parsed-sample shape this builder consumes and emits — structurally identical to
67
+ * `FeedPreview` from `preview-feed.ts` and to the wizard's `FeedPreviewResponse`, and
68
+ * declared HERE rather than imported from either so this module keeps zero dependencies.
69
+ * That is what lets one implementation serve the browser wizard and the `lumi` CLI: an
70
+ * import of `preview-feed.js` would pull `csv-parse` and the xlsx reader toward a client
71
+ * bundle, and an import of the wizard's types would point the dependency the wrong way.
72
+ */
73
+ interface HeaderSample {
74
+ /** Source column names, in first-seen order. */
75
+ columns: string[];
76
+ /** Sample data rows, each `{ column -> cell }`. */
77
+ rows: Array<Record<string, string>>;
78
+ }
79
+ declare const SNAPSHOT_MAX_COLUMNS = 256;
80
+ declare const SNAPSHOT_MAX_COLUMN_CHARS = 256;
81
+ declare const SNAPSHOT_MAX_ROWS = 5;
82
+ declare const SNAPSHOT_MAX_CELL_CHARS = 500;
83
+ declare const SNAPSHOT_MAX_SERIALIZED_CHARS = 100000;
84
+ /**
85
+ * Build the file-less-edit `headerSnapshot` (#667) from the current preview.
86
+ * Rows are capped + cells truncated (they're display samples, not data); a
87
+ * preview that can't fit the caps at all (pathological column count / column
88
+ * names) returns `null` — the mapping saves fine, it just won't support
89
+ * file-less editing.
90
+ *
91
+ * Pure module (no React, no 'use client') so the caps mirror is unit-testable
92
+ * without page infrastructure — same shape as `find-saved-mapping.ts`.
93
+ */
94
+ declare function toHeaderSnapshot(preview: HeaderSample): HeaderSample | null;
95
+ //#endregion
96
+ export { SNAPSHOT_MAX_ROWS as a, AutoAssignField as c, SNAPSHOT_MAX_COLUMN_CHARS as i, suggestTargets as l, SNAPSHOT_MAX_CELL_CHARS as n, SNAPSHOT_MAX_SERIALIZED_CHARS as o, SNAPSHOT_MAX_COLUMNS as r, toHeaderSnapshot as s, HeaderSample as t, tokenizeHeader as u };
97
+ //# sourceMappingURL=index-Bm69N5DW.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-Bm69N5DW.d.mts","names":[],"sources":["../src/matching/auto-assign.ts","../src/matching/header-snapshot.ts"],"mappings":";;AAqCA;;;;;;;;AAGS;AAiCT;;;;AAA6C;AAkC7C;;;;;;;;;AAGM;;;;ACtGN;;;;;;;;UD6BiB,eAAA;EACf,GAAA;EACA,KAAA;EACA,OAAA;AAAA;;ACd+B;AACjC;;;;AAAsC;AACtC;;;;AAA8B;AAC9B;iBD4CgB,cAAA,CAAe,MAAc;;;AC5CT;AACpC;;iBD6EgB,cAAA,CACd,OAAA,qBACA,MAAA,WAAiB,eAAA,KAChB,GAAG;;;;AAzEN;;;;;;;UC7BiB,YAAA;EDgCR;EC9BP,OAAA;ED+D4B;EC7D5B,IAAA,EAAM,KAAK,CAAC,MAAA;AAAA;AAAA,cAcD,oBAAA;AAAA,cACA,yBAAA;AAAA,cACA,iBAAA;AAAA,cACA,uBAAA;AAAA,cACA,6BAAA;;;;;;ADgFP;;;;ACtGN;iBAkCgB,gBAAA,CAAiB,OAAA,EAAS,YAAA,GAAe,YAAY"}
@@ -0,0 +1,390 @@
1
+ import { a as RowWarning, c as TransformRegistry, i as RowTransform, l as getTransformRegistry, n as RowError, o as TransformContext, r as RowResult, s as TransformName, t as DirectionalPair, u as registerImportTransform } from "./transform-WeiiLdYP.mjs";
2
+ import { a as mappingColumnSchema, n as MappingConfig, o as mappingConfigSchema, t as MappingColumn } from "./engine-tkbUXKxB.mjs";
3
+ import { A as ErrorSample, B as StreamFeedOptions, C as PriceStats, D as DEFAULT_MAX_PATTERNS, E as parsePriceStats, F as FeedReaderConfig, G as ImportRun, H as countDataRows, I as ReadFeedOptions, K as ImportRunClient, L as feedReaderConfigSchema, M as ErrorTrackerConfig, N as ErrorTrackerSnapshot, O as DEFAULT_MAX_SAMPLES_PER_PATTERN, P as mergeErrorTrackerSnapshots, R as readFeed, S as PriceAnomaly, T as detectPriceAnomaly, U as streamFeed, V as StreamFeedRow, W as IMPORT_RUN_STATUSES, _ as RunImportResult, b as PRICE_ANOMALY_THRESHOLD_MULTIPLIER, c as PairSinkContext, g as RunImportOptions, h as ImportRunProgress, j as ErrorTracker, k as ErrorPattern, m as DEFAULT_BATCH_SIZE, n as EsClientResolver, q as ImportRunStatus, r as FilePathResolver, s as PairSink, t as EnsureIndexResolver, v as runImport, w as PriceStatsCollector, x as PRICE_SAMPLE_CAP, y as EMPTY_PRICE_STATS, z as readPersistedFeedConfig } from "./worker-CRH_PjKO.mjs";
4
+ import { n as CARMAKER_MAPPING } from "./carmaker-feed-CWrZLYUt.mjs";
5
+ import { ToolkitApp } from "@murumets-ee/core";
6
+ import { Behavior } from "@murumets-ee/entity";
7
+ import { Logger as Logger$1 } from "@murumets-ee/logging";
8
+ import { AdminClient } from "@murumets-ee/entity/admin";
9
+
10
+ //#region src/admin/audit-empty-header.d.ts
11
+ /**
12
+ * Per-run outcome of the file re-read.
13
+ *
14
+ * - `corrupt` — the patched reader refuses this file's header. The run imported
15
+ * empty rows and reported success.
16
+ * - `clean` — the header re-reads with at least one column today. See the
17
+ * false-negative caveat in the module docblock.
18
+ * - `source_unavailable` — the source file could not be read at all (reaped by
19
+ * the retention sweep, deleted, or storage refused). Undecidable from the
20
+ * file; `zeroRowsIndexed` is the only signal left.
21
+ * - `undetectable` — the run is outside what this detector can decide: a
22
+ * `hasHeader: false` run has no header for the guard to test, a non-xlsx run
23
+ * cannot have the defect, and a file that yields no data rows never reached
24
+ * the guard at all.
25
+ * - `error` — reading failed for some other reason; `detail` carries it.
26
+ */
27
+ declare const EMPTY_HEADER_VERDICTS: readonly ["corrupt", "clean", "source_unavailable", "undetectable", "error"];
28
+ type EmptyHeaderVerdict = (typeof EMPTY_HEADER_VERDICTS)[number];
29
+ interface InspectImportRunSourceArgs {
30
+ /** Local path OR storage key, verbatim from `import_run.filePath`. */
31
+ filePath: string;
32
+ /** The run's persisted reader config (`import_run.params.feed`). */
33
+ feed: FeedReaderConfig;
34
+ /**
35
+ * Resolves `filePath` to a readable local path, exactly as the worker does.
36
+ * Omit when `filePath` is already a local FS path (the on-disk deployments
37
+ * the worker's no-resolver fallback covers).
38
+ */
39
+ resolveFilePath?: FilePathResolver | undefined;
40
+ }
41
+ interface InspectImportRunSourceResult {
42
+ verdict: EmptyHeaderVerdict;
43
+ /** Human-readable reason. Always set for everything except `clean`. */
44
+ detail?: string;
45
+ }
46
+ /**
47
+ * Re-read ONE run's source file with the patched reader and classify it.
48
+ *
49
+ * Stops pulling rows once the header has been decided — the guard fires while
50
+ * the header is captured, before any data row is yielded, so the first row
51
+ * settles it.
52
+ *
53
+ * That bounds the rows parsed OF THE TARGET SHEET, and nothing else. The
54
+ * resolver has already downloaded the whole object, and `streamXlsx` parses the
55
+ * worksheets after the target on its way out — it has to, or exceljs strands a
56
+ * temp file and an fd per deferred worksheet. So the per-run cost is one file,
57
+ * plus its trailing sheets; what bounds the SWEEP is the item cap and the
58
+ * concurrency cap in `auditEmptyHeaderRuns`.
59
+ */
60
+ declare function inspectImportRunSource(args: InspectImportRunSourceArgs): Promise<InspectImportRunSourceResult>;
61
+ /**
62
+ * The counters this audit reads out of `import_run.totals`.
63
+ *
64
+ * A LOSSY view, deliberately: `rowsWarned` / `batchesCompleted` /
65
+ * `patchesApplied` / `patchesOrphaned` are dropped rather than passed through,
66
+ * because a report is easier to read for what it needs than for everything the
67
+ * run recorded. Read the `import_run` row itself if you want the rest.
68
+ */
69
+ interface RecordedTotals {
70
+ rowsRead?: number | undefined;
71
+ rowsSucceeded?: number | undefined;
72
+ rowsFailed?: number | undefined;
73
+ rowsSkipped?: number | undefined;
74
+ }
75
+ /** Narrow the opaque `totals` JSONB to the counters this audit reports. */
76
+ declare function readRecordedTotals(raw: unknown): RecordedTotals;
77
+ interface AuditedImportRun {
78
+ id: string;
79
+ label: string;
80
+ /** Verbatim `import_run.filePath` — the storage key, so an operator can go look. */
81
+ filePath: string;
82
+ transformName: string;
83
+ createdAt: Date | null;
84
+ finishedAt: Date | null;
85
+ /** Row counters the run recorded. A corrupted run that recorded a plausible count is the most misleading artefact of all. */
86
+ totals: RecordedTotals;
87
+ verdict: EmptyHeaderVerdict;
88
+ detail?: string;
89
+ /**
90
+ * The run finished having read rows but indexed none. Corroborating evidence
91
+ * that survives the source file being reaped, and the ONLY signal available
92
+ * for the transient variant of the defect — but it is also what a wrong
93
+ * column mapping looks like, so it is never folded into the verdict.
94
+ *
95
+ * THREE-VALUED on purpose. `'unknown'` means the run recorded no usable
96
+ * counters (older rows; any run audited with a non-`succeeded` status, since
97
+ * `markFailed` never writes `totals`). Collapsing that to `false` would render
98
+ * absence of evidence as evidence of absence — in the one signal that is
99
+ * supposed to outlive the file.
100
+ */
101
+ zeroRowsIndexed: boolean | 'unknown';
102
+ }
103
+ interface EmptyHeaderAuditReport {
104
+ /** Total xlsx runs matching the audited statuses. */
105
+ candidates: number;
106
+ /** How many of them this invocation actually read. */
107
+ examined: number;
108
+ /** `candidates - examined` — dropped by the cap, newest kept. Never silent: also logged. */
109
+ notExamined: number;
110
+ counts: Record<EmptyHeaderVerdict, number>;
111
+ /** Every examined run, verdict-annotated, newest first. */
112
+ runs: AuditedImportRun[];
113
+ }
114
+ interface AuditEmptyHeaderRunsArgs {
115
+ /** Toolkit app — provides the app-bound `AdminClient` for `import_run`. */
116
+ app: ToolkitApp;
117
+ /**
118
+ * Same resolver the imports plugin hands the worker (typically
119
+ * `storageResolveFilePath`). Omit only when `filePath` is a local FS path.
120
+ */
121
+ resolveFilePath?: FilePathResolver | undefined;
122
+ /** Statuses to audit. Default `['succeeded']` — a failed run announced itself. */
123
+ statuses?: readonly ImportRunStatus[] | undefined;
124
+ /** Max runs examined, newest first. Default 200, hard ceiling 5000. */
125
+ limit?: number | undefined;
126
+ /** Peak concurrent storage reads. Default 4, hard ceiling 16. */
127
+ concurrency?: number | undefined;
128
+ logger: Pick<Logger$1, 'info' | 'warn'>;
129
+ }
130
+ /**
131
+ * Audit past import runs for the silent empty-header corruption.
132
+ *
133
+ * Newest-first, because the answerable window is the recent one — the retention
134
+ * sweep reaps source files ~7 days after a run reaches terminal status by
135
+ * default, so the oldest runs are the ones whose evidence is already gone.
136
+ */
137
+ declare function auditEmptyHeaderRuns(args: AuditEmptyHeaderRunsArgs): Promise<EmptyHeaderAuditReport>;
138
+ //#endregion
139
+ //#region src/admin/cleanup.d.ts
140
+ interface CleanupOrphanedImportsArgs {
141
+ /** Toolkit app (provides db). */
142
+ app: ToolkitApp;
143
+ /**
144
+ * Bucket name the imports flow uses. Rows in `toolkit_files` are
145
+ * filtered to this bucket so the cleanup doesn't touch media,
146
+ * ticketing attachments, or any other consumer's uploads even if
147
+ * they share the same Postgres instance.
148
+ */
149
+ bucket: string;
150
+ /**
151
+ * Hours after which an unreferenced upload is considered orphaned.
152
+ * Operator-grace window: should be long enough that a normal
153
+ * upload-then-register round-trip never wipes a fresh upload, but
154
+ * short enough that storage waste from failed uploads doesn't
155
+ * accumulate. 2 hours is the default in the imports plugin
156
+ * config — calibrated against the typical "operator clicks
157
+ * upload, walks away, comes back to check" workflow.
158
+ */
159
+ ttlHours: number;
160
+ /**
161
+ * Maximum rows to process per run. Bounded so a sudden flood of
162
+ * orphans (e.g. after a brand-new supplier with unrecognised
163
+ * filenames floods the queue with rejected uploads) doesn't
164
+ * block the queue worker on a single multi-hour cleanup.
165
+ */
166
+ limit?: number;
167
+ /** Logger for diagnostics. */
168
+ logger: Pick<Logger$1, 'debug' | 'info' | 'warn'>;
169
+ }
170
+ /**
171
+ * Run one cleanup pass. Returns the count of rows actually deleted
172
+ * (which may be less than the count of orphans found if some
173
+ * `storage.delete` calls failed — see the `warn` log inside the loop).
174
+ */
175
+ declare function cleanupOrphanedImports(args: CleanupOrphanedImportsArgs): Promise<number>;
176
+ interface CleanupProcessedImportsArgs {
177
+ /** Toolkit app (provides db). */
178
+ app: ToolkitApp;
179
+ /** Bucket name to scope the cleanup. Same semantics as `cleanupOrphanedImports`. */
180
+ bucket: string;
181
+ /**
182
+ * Days after which a terminal-status `import_run`'s source file is
183
+ * deleted. Operator-grace window: long enough that an operator can
184
+ * inspect the source file if a transform bug surfaces shortly after
185
+ * a successful import; short enough that storage doesn't accumulate
186
+ * months of supplier feeds.
187
+ *
188
+ * The plugin default is 7. Set to 0 to delete the source file
189
+ * immediately at the next cleanup tick after the import row
190
+ * reaches terminal status — useful when storage cost matters more
191
+ * than forensic reprocess capability.
192
+ */
193
+ retentionDays: number;
194
+ /** Max rows per run. Defaults to 100 (same cap as the orphan sweep). */
195
+ limit?: number;
196
+ logger: Pick<Logger$1, 'debug' | 'info' | 'warn'>;
197
+ }
198
+ /**
199
+ * Cleanup source files for `import_run`s that finished (succeeded /
200
+ * failed / cancelled) more than `retentionDays` days ago.
201
+ *
202
+ * Counterpart to `cleanupOrphanedImports`: that one targets uploads
203
+ * with NO matching `import_run` (handshake never completed); this one
204
+ * targets uploads whose `import_run` row IS present and terminal.
205
+ * Together they cover every "useless storage row" case the imports
206
+ * flow can produce.
207
+ *
208
+ * Deletes ONLY the storage row (via `StorageClient.delete`); the
209
+ * `import_run` row is kept so the History panel + audit trail still
210
+ * surface the past run. The `import_run.filePath` value points at a
211
+ * now-deleted key after this runs — consumers that try to re-download
212
+ * the source file (e.g. a future "reprocess" button) will get a
213
+ * `getMetadata` → null and should show "source file unavailable" in
214
+ * the UI rather than crashing.
215
+ */
216
+ declare function cleanupProcessedImports(args: CleanupProcessedImportsArgs): Promise<number>;
217
+ //#endregion
218
+ //#region src/entities/import-mapping.d.ts
219
+ /**
220
+ * Registered target schemas a mapping may write into. This is the FIRST of two
221
+ * whitelists (the second is each schema's target-field set) — a mapping whose
222
+ * `targetSchema` is outside this list is rejected (deny-by-default, R02).
223
+ */
224
+ declare const IMPORT_MAPPING_TARGET_SCHEMAS: readonly ["parts", "discount_table"];
225
+ type ImportMappingTargetSchema = (typeof IMPORT_MAPPING_TARGET_SCHEMAS)[number];
226
+ /** File formats the reader dispatches on. `csv`/`tsv` → csv-parse; `xlsx` → exceljs. */
227
+ declare const IMPORT_MAPPING_FORMATS: readonly ["csv", "tsv", "xlsx"];
228
+ type ImportMappingFormat = (typeof IMPORT_MAPPING_FORMATS)[number];
229
+ declare const ImportMapping: import("@murumets-ee/entity").Entity<{
230
+ id: import("@murumets-ee/entity").IdField;
231
+ } & import("@murumets-ee/entity").AuditableFields & Record<never, never> & {
232
+ /** FK to commerce `supplier` (by name string — no cross-package import). Required. */supplierId: import("@murumets-ee/entity").ReferenceField & {
233
+ cardinality: "many" | "one";
234
+ } & {
235
+ readonly entity: "supplier";
236
+ readonly required: true;
237
+ readonly indexed: true;
238
+ }; /** FK to commerce `brand`. NULL = supplier default (applies to all that supplier's brands). */
239
+ brandId: import("@murumets-ee/entity").ReferenceField & {
240
+ cardinality: "many" | "one";
241
+ } & {
242
+ readonly entity: "brand";
243
+ readonly indexed: true;
244
+ }; /** Which registered target schema this mapping feeds — `parts` or `discount_table`. */
245
+ targetSchema: import("@murumets-ee/entity").SelectField & {
246
+ options: readonly ["parts", "discount_table"];
247
+ } & {
248
+ readonly options: readonly ["parts", "discount_table"];
249
+ readonly required: true;
250
+ readonly indexed: true;
251
+ }; /** Operator-facing profile name (a supplier can have e.g. "monthly full" vs "weekly delta"). */
252
+ name: import("@murumets-ee/entity").TextField & {
253
+ readonly required: true;
254
+ readonly maxLength: 255;
255
+ }; /** File format — drives reader dispatch + the default delimiter. */
256
+ format: import("@murumets-ee/entity").SelectField & {
257
+ options: readonly ["csv", "tsv", "xlsx"];
258
+ } & {
259
+ readonly options: readonly ["csv", "tsv", "xlsx"];
260
+ readonly required: true;
261
+ };
262
+ /**
263
+ * Explicit single-char field delimiter override. When omitted, the reader
264
+ * defaults from `format` (`csv` → `,`, `tsv` → `\t`). Ignored for `xlsx`.
265
+ */
266
+ delimiter: import("@murumets-ee/entity").TextField & {
267
+ readonly maxLength: 4;
268
+ }; /** First row is the header. `false` → positional `{ "0": …, "1": … }` source keys. */
269
+ hasHeader: import("@murumets-ee/entity").BooleanField & {
270
+ readonly default: true;
271
+ }; /** 0-based header row index (header hygiene — R03 feeds have leading/trailing-space headers). */
272
+ headerRowIndex: import("@murumets-ee/entity").NumberField & {
273
+ readonly integer: true;
274
+ readonly min: 0;
275
+ readonly default: 0;
276
+ }; /** xlsx sheet name (or 1-based index as a string). Ignored for csv/tsv. */
277
+ sheet: import("@murumets-ee/entity").TextField & {
278
+ readonly maxLength: 128;
279
+ };
280
+ /**
281
+ * Field-quoting policy for csv/tsv, forwarded to `csv-parse`'s `quote`.
282
+ *
283
+ * NULL — the default, and today's only behaviour — means quotes are ORDINARY DATA.
284
+ * A single character (in practice `"`) means RFC 4180: that character encloses fields
285
+ * that contain the delimiter, and is doubled to escape itself.
286
+ *
287
+ * **This is per-file evidence, not a house style, and the two settings are not
288
+ * interchangeable.** Measured against the real supplier feeds this pipeline ingests
289
+ * (R027): the Ford CSV has 1,476 rows whose description holds a quoted comma — parsed
290
+ * with quotes as data it dies at row 11,126 of 808,147 with
291
+ * `CSV_RECORD_INCONSISTENT_COLUMNS`. The two tab-delimited feeds carry 2,321 lines with
292
+ * an ODD number of `"` characters (`"O"-RING`, `RACCORDO A "T`, `ECROU A TOLE "C`) —
293
+ * parsed as RFC 4180 they die at row 1,476 with `INVALID_OPENING_QUOTE`. Each feed
294
+ * hard-fails under the other's setting, so this cannot be derived from `format` and
295
+ * cannot be a global default. `lumi imports:analyze` decides it per file by trial parse.
296
+ *
297
+ * Ignored for `xlsx`, which has no text quoting.
298
+ */
299
+ quote: import("@murumets-ee/entity").TextField & {
300
+ readonly maxLength: 1;
301
+ };
302
+ /**
303
+ * Number format for the `number` per-column transform:
304
+ * `{ decimal: '.'|',', thousands: ''|'.'|','|' ' }`. Opaque json; validated
305
+ * by the engine's Zod schema at consume time (matches `import_run.params`).
306
+ */
307
+ numberFormat: import("@murumets-ee/entity").JsonField & {
308
+ readonly default: {
309
+ readonly decimal: ".";
310
+ readonly thousands: "";
311
+ };
312
+ };
313
+ /**
314
+ * Ordered column map: `[{ source, target, transforms? }]`. `source` is a
315
+ * header name (or positional index string); `target` MUST be a field in the
316
+ * chosen schema's whitelist (validated by the engine's Zod schema, R02
317
+ * injection boundary); `transforms` is a bounded, enum-keyed list. Opaque
318
+ * json on the entity; typed + validated by the engine.
319
+ */
320
+ columnMap: import("@murumets-ee/entity").JsonField & {
321
+ readonly default: readonly [];
322
+ };
323
+ /**
324
+ * File-less edit snapshot (#667): `{ columns: string[], rows: [{column →
325
+ * cell}] }` — the sample feed's header + a few sample rows, captured from
326
+ * the wizard's preview at save time. Lets the editor re-open a saved
327
+ * mapping WITHOUT re-uploading the (potentially huge) feed just to tweak a
328
+ * transform. NULL on rows saved before this field existed (and when the
329
+ * preview exceeded the size caps) — the editor falls back to the
330
+ * upload-first flow. Staleness is benign: a source column that later
331
+ * vanishes from the real feed is caught at run time by the 6b drift check
332
+ * (409) / row errors, not by this snapshot. Size-capped by
333
+ * `import-mapping-validation`.
334
+ */
335
+ headerSnapshot: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>; /** Inactive mappings are skipped by resolution. */
336
+ active: import("@murumets-ee/entity").BooleanField & {
337
+ readonly default: true;
338
+ readonly indexed: true;
339
+ };
340
+ }, "import_mapping", [Behavior<import("@murumets-ee/entity").AuditableFields, string>, import("@murumets-ee/entity").SearchableBehavior, Behavior<Record<never, never>, "imports:import-mapping-unique">, Behavior<Record<never, never>, "imports:import-mapping-quote">, Behavior<Record<never, never>, "imports:import-mapping-validation">]>;
341
+ type ImportMappingClient = AdminClient<typeof ImportMapping.allFields>;
342
+ //#endregion
343
+ //#region src/entities/import-mapping-validation.d.ts
344
+ /**
345
+ * Throw a descriptive error unless the mapping is structurally valid AND honours its
346
+ * schema's field whitelist + required targets. The thrown message is operator-facing
347
+ * (surfaced verbatim by the admin CRUD route as a 4xx body).
348
+ */
349
+ declare function assertValidMapping(input: {
350
+ targetSchema: unknown;
351
+ numberFormat: unknown;
352
+ columnMap: unknown;
353
+ headerSnapshot?: unknown;
354
+ }): void;
355
+ /**
356
+ * Behavior wrapping {@link assertValidMapping}. Typed `Behavior<Record<never,
357
+ * never>, …>` (NOT plain `Behavior`, whose wide `fields?` would collapse the
358
+ * entity's inferred field types — the documented no-field-behavior gotcha).
359
+ * Contributes a hook, no fields.
360
+ */
361
+ declare const importMappingValidation: Behavior<Record<never, never>, 'imports:import-mapping-validation'>;
362
+ //#endregion
363
+ //#region src/preview-feed.d.ts
364
+ /** Hard ceiling on preview rows regardless of caller request — bounds work + payload. */
365
+ declare const MAX_PREVIEW_ROWS = 100;
366
+ /** Default sample size when the caller doesn't specify. */
367
+ declare const DEFAULT_PREVIEW_ROWS = 20;
368
+ interface PreviewFeedOptions extends ReadFeedOptions {
369
+ /** Rows to sample. Defaults to {@link DEFAULT_PREVIEW_ROWS}; clamped to [1, {@link MAX_PREVIEW_ROWS}]. */
370
+ maxRows?: number;
371
+ }
372
+ interface FeedPreview {
373
+ /**
374
+ * Source column names in first-seen order. With a header these are the (trimmed)
375
+ * header names; headerless feeds yield positional `"0"`, `"1"`, … keys. The union
376
+ * across sampled rows covers feeds whose first row omits trailing empty cells.
377
+ */
378
+ columns: string[];
379
+ /** Up to `maxRows` sampled data rows, each a `{ column → cell }` map (cells are strings). */
380
+ rows: Array<Record<string, string>>;
381
+ }
382
+ /**
383
+ * Read up to `maxRows` data rows from a feed file and return the detected columns +
384
+ * sample rows. Throws whatever the underlying reader throws (bad sheet index,
385
+ * duplicate header, unreadable file) — the caller maps that to a 4xx.
386
+ */
387
+ declare function previewFeed(options: PreviewFeedOptions): Promise<FeedPreview>;
388
+ //#endregion
389
+ export { type AuditEmptyHeaderRunsArgs, type AuditedImportRun, CARMAKER_MAPPING, type CleanupOrphanedImportsArgs, type CleanupProcessedImportsArgs, DEFAULT_BATCH_SIZE, DEFAULT_MAX_PATTERNS, DEFAULT_MAX_SAMPLES_PER_PATTERN, DEFAULT_PREVIEW_ROWS, type DirectionalPair, EMPTY_HEADER_VERDICTS, EMPTY_PRICE_STATS, type EmptyHeaderAuditReport, type EmptyHeaderVerdict, type EnsureIndexResolver, type ErrorPattern, type ErrorSample, ErrorTracker, type ErrorTrackerConfig, type ErrorTrackerSnapshot, type EsClientResolver, type FeedPreview, type FeedReaderConfig, type FilePathResolver, IMPORT_MAPPING_FORMATS, IMPORT_MAPPING_TARGET_SCHEMAS, IMPORT_RUN_STATUSES, ImportMapping, type ImportMappingClient, type ImportMappingFormat, type ImportMappingTargetSchema, ImportRun, type ImportRunClient, type ImportRunProgress, type ImportRunStatus, type InspectImportRunSourceArgs, type InspectImportRunSourceResult, MAX_PREVIEW_ROWS, type MappingColumn, type MappingConfig, PRICE_ANOMALY_THRESHOLD_MULTIPLIER, PRICE_SAMPLE_CAP, type PairSink, type PairSinkContext, type PreviewFeedOptions, type PriceAnomaly, type PriceStats, PriceStatsCollector, type ReadFeedOptions, type RecordedTotals, type RowError, type RowResult, type RowTransform, type RowWarning, type RunImportOptions, type RunImportResult, type StreamFeedOptions, type StreamFeedRow, type TransformContext, type TransformName, TransformRegistry, assertValidMapping, auditEmptyHeaderRuns, cleanupOrphanedImports, cleanupProcessedImports, countDataRows, detectPriceAnomaly, feedReaderConfigSchema, getTransformRegistry, importMappingValidation, inspectImportRunSource, mappingColumnSchema, mappingConfigSchema, mergeErrorTrackerSnapshots, parsePriceStats, previewFeed, readFeed, readPersistedFeedConfig, readRecordedTotals, registerImportTransform, runImport, streamFeed };
390
+ //# sourceMappingURL=index.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.mts","names":[],"sources":["../src/admin/audit-empty-header.ts","../src/admin/cleanup.ts","../src/entities/import-mapping.ts","../src/entities/import-mapping-validation.ts","../src/preview-feed.ts"],"mappings":";;;;;;;;;;;;;AAwQa;AASb;;;;AAAgE;AAWhE;;;;;;;cA9Ma,qBAAA;AAAA,KAOD,kBAAA,WAA6B,qBAAqB;AAAA,UAc7C,0BAAA;EA2Lf;EAzLA,QAAA;EA4LA;EA1LA,IAAA,EAAM,gBAAA;EA2LK;;;;;EArLX,eAAA,GAAkB,gBAAgB;AAAA;AAAA,UAGnB,4BAAA;EACf,OAAA,EAAS,kBAAkB;EAmMZ;EAjMf,MAAA;AAAA;;;;;;;;;;;;;;;iBAyDoB,sBAAA,CACpB,IAAA,EAAM,0BAAA,GACL,OAAA,CAAQ,4BAAA;;AAkJa;AAGxB;;;;;;UA/DiB,cAAA;EACf,QAAA;EACA,aAAA;EACA,UAAA;EACA,WAAA;AAAA;;iBASc,kBAAA,CAAmB,GAAA,YAAe,cAAc;AAAA,UAW/C,gBAAA;EACf,EAAA;EACA,KAAA;EAkDA;EAhDA,QAAA;EACA,aAAA;EACA,SAAA,EAAW,IAAA;EACX,UAAA,EAAY,IAAA;EA8CO;EA5CnB,MAAA,EAAQ,cAAA;EACR,OAAA,EAAS,kBAAA;EACT,MAAA;EA+EM;;;;;;;;;;AACyB;;EAnE/B,eAAA;AAAA;AAAA,UAGe,sBAAA;EClR0B;EDoRzC,UAAA;EClRK;EDoRL,QAAA;EC1PQ;ED4PR,WAAA;EACA,MAAA,EAAQ,MAAA,CAAO,kBAAA;ECvRf;EDyRA,IAAA,EAAM,gBAAA;AAAA;AAAA,UAGS,wBAAA;ECpQf;EDsQA,GAAA,EAAK,UAAA;ECpQG;;;AAAW;EDyQnB,eAAA,GAAkB,gBAAA;EClOwB;EDoO1C,QAAA,YAAoB,eAAA;ECpOiE;EDsOrF,KAAA;ECtO2C;EDwO3C,WAAA;EACA,MAAA,EAAQ,IAAA,CAAK,QAAA;AAAA;AC/If;;;;;;;AAAA,iBDmLsB,oBAAA,CACpB,IAAA,EAAM,wBAAA,GACL,OAAA,CAAQ,sBAAA;;;UClVM,0BAAA;ED4Ef;EC1EA,GAAA,EAAK,UAAA;ED4EL;;AAAM;AAyDR;;;EC9HE,MAAA;EDgIS;;;;;;;;;ECtHT,QAAA;ED4Me;;;;;;ECrMf,KAAA;EDyMA;ECvMA,MAAA,EAAQ,IAAA,CAAK,QAAA;AAAA;ADgNf;;;;AAAgE;AAAhE,iBCzKsB,sBAAA,CAAuB,IAAA,EAAM,0BAAA,GAA6B,OAAO;AAAA,UA0FtE,2BAAA;;EAEf,GAAA,EAAK,UAAA;ED+FO;EC7FZ,MAAA;EDgGS;;;;;;;;;;;;ECnFT,aAAA;EDmFA;ECjFA,KAAA;EACA,MAAA,EAAQ,IAAA,CAAK,QAAA;AAAA;;AD8FE;AAGjB;;;;;;;;;;;;;;;;iBC5EsB,uBAAA,CAAwB,IAAA,EAAM,2BAAA,GAA8B,OAAO;;;;;AD9HrD;AAGpC;;cEzEa,6BAAA;AAAA,KACD,yBAAA,WAAoC,6BAA6B;;cAGhE,sBAAA;AAAA,KACD,mBAAA,WAA8B,sBAAsB;AAAA,cAqFnD,aAAA,gCAAa,MAAA;;;EF2CkB;;;;;;KAC1C;;;;;;;;;;;;;KAoGc;;;;KAWC;;;;;;;EAUY;;;;;;KAJhB;;;KAGH;;;;;KAeO;;;;EAUA;;;;;;;;;;;;;;AAEO;AAGxB;;;;;;;EAcU;;;;;;;;;;;EAAR;;;;AAAmB;AAoCrB;;;;;EAEG;;;;;;;;AAA8B;;;;8GClVU;;;;;;KCwM/B,mBAAA,GAAsB,WAAW,QAAQ,aAAA,CAAc,SAAA;;;;;;AF1H3D;AAyDR;iBGjGgB,kBAAA,CAAmB,KAAA;EACjC,YAAA;EACA,YAAA;EACA,SAAA;EACA,cAAA;AAAA;;;;;;;cAoFW,uBAAA,EAAyB,QAAQ,CAC5C,MAAA;;;AH/EF;AAAA,cInEa,gBAAA;;cAEA,oBAAA;AAAA,UAEI,kBAAA,SAA2B,eAAe;EJ6E1C;EI3Ef,OAAO;AAAA;AAAA,UAGQ,WAAA;EJ0Ef;;;;;EIpEA,OAAA;EJ4EkC;EI1ElC,IAAA,EAAM,KAAK,CAAC,MAAA;AAAA;;;;;;iBAQQ,WAAA,CAAY,OAAA,EAAS,kBAAA,GAAqB,OAAA,CAAQ,WAAA"}
package/dist/index.mjs ADDED
@@ -0,0 +1,2 @@
1
+ import{a as e,c as t,i as n,l as r,n as i,o as a,r as o,s,t as c,u as l}from"./transform-Cu5YEoOl.mjs";import{a as u,n as d,o as f,r as p,t as m}from"./read-feed-DsLgRDNR.mjs";import{n as h,t as g}from"./cleanup-DDljBe0Z.mjs";import{n as _,r as v}from"./engine-B0ExE4jL.mjs";import{a as y,c as b,d as x,f as S,i as C,l as w,n as T,o as E,r as D,s as O,t as k,u as A}from"./runner-BnFvID1z.mjs";import{n as j}from"./carmaker-feed-B-QYr3PI.mjs";import{DEFAULT_BOUNDED_CONCURRENCY as M,runBounded as N}from"@murumets-ee/core";const P=[`corrupt`,`clean`,`source_unavailable`,`undetectable`,`error`],F=[`succeeded`],I=5e3;function L(e){return e instanceof Error?e.message:String(e)}const R=new Set([`ENOENT`,`EACCES`,`EPERM`,`EISDIR`,`ENOTDIR`]),z=/FILE_ENDED/;function B(e,t){if(typeof e!=`object`||!e)return!1;if(e instanceof t)return!0;if(`code`in e){let{code:t}=e;if(typeof t==`string`&&R.has(t))return!0}return e instanceof Error&&z.test(e.message)}async function V(e){let{filePath:t,feed:n,resolveFilePath:r}=e;if(n.format!==`xlsx`)return{verdict:`undetectable`,detail:`not an xlsx run (format: ${n.format??`tsv (default)`}) — the defect is exceljs-specific`};if(n.hasHeader===!1)return{verdict:`undetectable`,detail:`run used hasHeader:false — no header row for the guard to test`};let{streamXlsx:i,XlsxEmptyHeaderError:a,XlsxNotAFileError:o}=await import(`./xlsx-reader-D9Ntxm29.mjs`),s;try{s=r?await r(t):{localPath:t}}catch(e){return{verdict:`source_unavailable`,detail:L(e)}}try{let e=!1;for await(let t of i({filePath:s.localPath,...n.headerRowIndex!==void 0&&{headerRowIndex:n.headerRowIndex},...n.sheet!==void 0&&{sheet:n.sheet}})){e=!0;break}return e?{verdict:`clean`}:{verdict:`undetectable`,detail:`file yielded no data rows — the header guard never ran`}}catch(e){return e instanceof a?{verdict:`corrupt`,detail:e.message}:B(e,o)?{verdict:`source_unavailable`,detail:L(e)}:{verdict:`error`,detail:L(e)}}finally{await s.cleanup?.().catch(()=>{})}}function H(e,t){let n=e[t];return typeof n==`number`&&Number.isFinite(n)?n:void 0}function U(e){if(typeof e!=`object`||!e||Array.isArray(e))return{};let t=e;return{rowsRead:H(t,`rowsRead`),rowsSucceeded:H(t,`rowsSucceeded`),rowsFailed:H(t,`rowsFailed`),rowsSkipped:H(t,`rowsSkipped`)}}function W(){return{corrupt:0,clean:0,source_unavailable:0,undetectable:0,error:0}}function G(e){return e.rowsRead===void 0||e.rowsSucceeded===void 0?`unknown`:e.rowsRead>0&&e.rowsSucceeded===0}function K(e){if(e instanceof Date)return e;if(typeof e==`string`){let t=new Date(e);return Number.isNaN(t.getTime())?null:t}return null}async function q(e){let{app:t,resolveFilePath:n,logger:r}=e,i=e.statuses??F;if(i.length===0)throw Error(`auditEmptyHeaderRuns: statuses must name at least one import_run status`);let a=e.limit??200;if(!Number.isInteger(a)||a<1||a>I)throw Error(`auditEmptyHeaderRuns: limit must be an integer in [1, ${I}] (got ${a})`);let o=e.concurrency??M;if(!Number.isInteger(o)||o<1||o>16)throw Error(`auditEmptyHeaderRuns: concurrency must be an integer in [1, 16] (got ${o})`);let s=t.getClient(l),{and:c,desc:u,getTableColumns:d,inArray:f,sql:m}=await import(`drizzle-orm`),h=d(s.getTable()),g=h.status,_=h.params,v=h.createdAt;if(!g||!_||!v)throw Error(`auditEmptyHeaderRuns: import_run schema missing expected columns (status/params/createdAt)`);let y=c(f(g,[...i]),m`${_}->'feed'->>'format' = ${`xlsx`}`),b=await s.count({where:y}),x=await s.findMany({where:y,orderBy:u(v),limit:a}),S=Math.max(0,b-x.length);if(S>0&&r.warn({candidates:b,examined:x.length,notExamined:S,limit:a},`imports audit: capped at the most recent runs — re-run with a higher limit to cover the rest`),x.length===0)return r.info({statuses:i},`imports audit: no xlsx runs to audit`),{candidates:b,examined:0,notExamined:S,counts:W(),runs:[]};r.info({candidates:b,examining:x.length,concurrency:o},`imports audit: re-reading source files for the empty-header defect`);let C=await N(x,async e=>{let t=e.filePath,r=U(e.totals),i={id:e.id,label:e.label,filePath:t,transformName:e.transformName,createdAt:K(e.createdAt),finishedAt:K(e.finishedAt),totals:r,zeroRowsIndexed:G(r)};try{let r=p(e.params);if(!r.success){let e=r.error.issues[0]?.message??`invalid feed reader config`;return{...i,verdict:`error`,detail:`invalid params.feed: ${e}`}}let a=await V({filePath:t,feed:r.data,...n!==void 0&&{resolveFilePath:n}});return{...i,verdict:a.verdict,...a.detail!==void 0&&{detail:a.detail}}}catch(e){return{...i,verdict:`error`,detail:L(e)}}},o),w=W();for(let e of C)w[e.verdict]+=1,e.verdict===`corrupt`&&r.warn({importRunId:e.id,label:e.label,filePath:e.filePath,transformName:e.transformName,createdAt:e.createdAt,totals:e.totals},`imports audit: SILENTLY CORRUPTED run — its source file has no readable header`);let T=C.filter(e=>e.zeroRowsIndexed===!0).length,E=C.filter(e=>e.zeroRowsIndexed===`unknown`).length;return n===void 0&&w.source_unavailable===C.length&&r.warn({examined:C.length},`imports audit: every source was unreadable and no resolveFilePath was given — if filePath holds storage keys rather than local paths, pass storageResolveFilePath`),r.info({candidates:b,examined:C.length,notExamined:S,...w,zeroRowsIndexed:T,totalsUnknown:E},`imports audit: complete`),{candidates:b,examined:C.length,notExamined:S,counts:w,runs:C}}const J=100,Y=20;async function X(e){let{maxRows:t,...n}=e,r=t??20,i=Math.min(100,Math.max(1,Number.isFinite(r)?Math.floor(r):20)),a=[],o=new Map;for await(let{row:e}of d({...n,relaxColumnCount:!0})){for(let t of Object.keys(e))o.set(t,!0);if(a.push(e),a.length>=i)break}return{columns:[...o.keys()],rows:a}}export{j as CARMAKER_MAPPING,k as DEFAULT_BATCH_SIZE,w as DEFAULT_MAX_PATTERNS,A as DEFAULT_MAX_SAMPLES_PER_PATTERN,Y as DEFAULT_PREVIEW_ROWS,P as EMPTY_HEADER_VERDICTS,D as EMPTY_PRICE_STATS,x as ErrorTracker,n as IMPORT_MAPPING_FORMATS,e as IMPORT_MAPPING_TARGET_SCHEMAS,r as IMPORT_RUN_STATUSES,a as ImportMapping,l as ImportRun,J as MAX_PREVIEW_ROWS,C as PRICE_ANOMALY_THRESHOLD_MULTIPLIER,y as PRICE_SAMPLE_CAP,E as PriceStatsCollector,c as TransformRegistry,s as assertValidMapping,q as auditEmptyHeaderRuns,g as cleanupOrphanedImports,h as cleanupProcessedImports,u as countDataRows,O as detectPriceAnomaly,m as feedReaderConfigSchema,i as getTransformRegistry,t as importMappingValidation,V as inspectImportRunSource,_ as mappingColumnSchema,v as mappingConfigSchema,S as mergeErrorTrackerSnapshots,b as parsePriceStats,X as previewFeed,d as readFeed,p as readPersistedFeedConfig,U as readRecordedTotals,o as registerImportTransform,T as runImport,f as streamFeed};
2
+ //# sourceMappingURL=index.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.mjs","names":[],"sources":["../src/admin/audit-empty-header.ts","../src/preview-feed.ts"],"sourcesContent":["/**\n * Retroactive audit for the silent empty-header corruption fixed in #839\n * (issue #842).\n *\n * **What went wrong.** exceljs assigns `this.sharedStrings = []` at the top of\n * `_parseSharedStrings`, before parsing a single entry — and an empty array is\n * truthy, which is what its inline-worksheet guard tests. A shared-strings\n * entry that yields nothing therefore leaves every string cell resolving to\n * `undefined`: the header parses to zero columns, every data row maps to `{}`,\n * and the import reports SUCCESS with every value dropped. #839 made\n * `streamXlsx` refuse that file. It cannot see backwards, and a past occurrence\n * is indistinguishable from success in the `import_run` row alone — both\n * `status: 'succeeded'`, both no fatal error.\n *\n * **What this does.** Re-reads the retained source file of every past xlsx run\n * with the PATCHED reader and classifies each one. `import_run.filePath` is\n * `required: true`, so every run has a source to point at.\n *\n * **What a verdict is worth — read this before acting on a report.**\n *\n * - `corrupt` has no false positives worth worrying about: the guard fires on\n * precisely the condition that produced the empty parse, so a file that\n * throws today would have thrown then. There is no threshold to tune, unlike\n * inferring corruption from suspiciously-low row counts.\n * - `clean` is NOT proof the run was fine. The upstream defect has two causes:\n * a file that genuinely carries an empty/unparseable shared-strings part\n * (deterministic — caught here), and a truncated or starved read at import\n * time (transient — the same file re-reads perfectly today). The file re-read\n * is blind to the second. That is why every audited run also carries\n * {@link AuditedImportRun.zeroRowsIndexed}, which is derived from the\n * recorded totals and needs no file at all: a run that finished `succeeded`\n * having indexed zero rows is the observable signature of the corruption\n * whichever cause produced it. It is corroborating, not conclusive — an\n * operator who uploaded a feed with the wrong column mapping gets the same\n * shape — so it is reported as a separate flag and never folded into the\n * verdict.\n * - `source_unavailable` is the honest answer for a run whose evidence is gone,\n * and there will be some: `cleanupProcessedImports` deletes the source file\n * of any terminal run older than `processedRetentionDays` (plugin default 7),\n * on an hourly schedule. The retained-file window is therefore about a week\n * on a deployment running the queue worker with the default config — run this\n * audit sooner rather than later, and read `zeroRowsIndexed` for the runs\n * whose files have already been reaped.\n *\n * Read-only by construction: it reports, it never re-imports. Deciding what to\n * do about a corrupted run needs the operator.\n *\n * Bounded per CLAUDE.md's fan-out rule — it is `O(runs)` reads against storage,\n * so it caps both the item count AND the peak concurrency, and logs whatever it\n * drops rather than silently truncating. The per-run cost is one file: the\n * resolver downloads the object, and the reader parses to the header of the\n * target sheet plus whatever follows it. Neither cap substitutes for the other,\n * and neither substitutes for running the audit while the answer is still\n * there.\n */\n\nimport { DEFAULT_BOUNDED_CONCURRENCY, runBounded, type ToolkitApp } from '@murumets-ee/core'\nimport type { Logger } from '@murumets-ee/logging'\nimport { ImportRun, type ImportRunStatus } from '../entities/import-run.js'\nimport { type FeedReaderConfig, readPersistedFeedConfig } from '../read-feed.js'\nimport type { FilePathResolver } from '../worker.js'\n\n/**\n * Per-run outcome of the file re-read.\n *\n * - `corrupt` — the patched reader refuses this file's header. The run imported\n * empty rows and reported success.\n * - `clean` — the header re-reads with at least one column today. See the\n * false-negative caveat in the module docblock.\n * - `source_unavailable` — the source file could not be read at all (reaped by\n * the retention sweep, deleted, or storage refused). Undecidable from the\n * file; `zeroRowsIndexed` is the only signal left.\n * - `undetectable` — the run is outside what this detector can decide: a\n * `hasHeader: false` run has no header for the guard to test, a non-xlsx run\n * cannot have the defect, and a file that yields no data rows never reached\n * the guard at all.\n * - `error` — reading failed for some other reason; `detail` carries it.\n */\nexport const EMPTY_HEADER_VERDICTS = [\n 'corrupt',\n 'clean',\n 'source_unavailable',\n 'undetectable',\n 'error',\n] as const\nexport type EmptyHeaderVerdict = (typeof EMPTY_HEADER_VERDICTS)[number]\n\n/** Statuses audited when the caller doesn't name any. */\nconst DEFAULT_STATUSES: readonly ImportRunStatus[] = ['succeeded']\n\n/** Default cap on runs examined per invocation. */\nconst DEFAULT_LIMIT = 200\n\n/** Hard ceiling on the per-invocation cap. Protects storage from a fat-fingered `--limit`. */\nconst MAX_LIMIT = 5000\n\n/** Hard ceiling on peak concurrent reads. Bounds the burst the item cap alone does not. */\nconst MAX_CONCURRENCY = 16\n\nexport interface InspectImportRunSourceArgs {\n /** Local path OR storage key, verbatim from `import_run.filePath`. */\n filePath: string\n /** The run's persisted reader config (`import_run.params.feed`). */\n feed: FeedReaderConfig\n /**\n * Resolves `filePath` to a readable local path, exactly as the worker does.\n * Omit when `filePath` is already a local FS path (the on-disk deployments\n * the worker's no-resolver fallback covers).\n */\n resolveFilePath?: FilePathResolver | undefined\n}\n\nexport interface InspectImportRunSourceResult {\n verdict: EmptyHeaderVerdict\n /** Human-readable reason. Always set for everything except `clean`. */\n detail?: string\n}\n\n/** Best-effort message for an unknown thrown value. */\nfunction describeError(err: unknown): string {\n if (err instanceof Error) return err.message\n return String(err)\n}\n\n/**\n * Filesystem / object-store errors that mean \"the evidence is not readable\",\n * as distinct from \"the file is readable and its header is bad\".\n *\n * ENOENT is the retention sweep having got there first — the expected outcome\n * for anything older than `processedRetentionDays`. The rest are real\n * deployment conditions rather than theory: the imports directory has had uid\n * drift before (#661's `imports-init` chown), and this audit is run from an\n * operator shell, not under the worker's uid — so EACCES against a file the\n * worker wrote is exactly the shape to expect. All of them are \"cannot decide\",\n * never \"corrupt\".\n */\nconst UNREADABLE_SOURCE_CODES = new Set(['ENOENT', 'EACCES', 'EPERM', 'EISDIR', 'ENOTDIR'])\n\n/**\n * unzipper's error for a zip that ends mid-entry. A half-written retained\n * upload is evidence we cannot read, not a verdict about the run — checked:\n * truncating a workbook at 95% / 60% / 20% all reject cleanly with this rather\n * than crashing or hanging.\n */\nconst TRUNCATED_ARCHIVE_MESSAGE = /FILE_ENDED/\n\nfunction isUnreadableSource(err: unknown, notAFile: new (...args: never[]) => Error): boolean {\n if (typeof err !== 'object' || err === null) return false\n // A directory (or socket, or fifo) where a workbook should be. Matched by\n // TYPE — the reader exports the class precisely so this is not a regex.\n if (err instanceof notAFile) return true\n if ('code' in err) {\n const { code } = err as { code?: unknown }\n if (typeof code === 'string' && UNREADABLE_SOURCE_CODES.has(code)) return true\n }\n return err instanceof Error && TRUNCATED_ARCHIVE_MESSAGE.test(err.message)\n}\n\n/**\n * Re-read ONE run's source file with the patched reader and classify it.\n *\n * Stops pulling rows once the header has been decided — the guard fires while\n * the header is captured, before any data row is yielded, so the first row\n * settles it.\n *\n * That bounds the rows parsed OF THE TARGET SHEET, and nothing else. The\n * resolver has already downloaded the whole object, and `streamXlsx` parses the\n * worksheets after the target on its way out — it has to, or exceljs strands a\n * temp file and an fd per deferred worksheet. So the per-run cost is one file,\n * plus its trailing sheets; what bounds the SWEEP is the item cap and the\n * concurrency cap in `auditEmptyHeaderRuns`.\n */\nexport async function inspectImportRunSource(\n args: InspectImportRunSourceArgs,\n): Promise<InspectImportRunSourceResult> {\n const { filePath, feed, resolveFilePath } = args\n\n // The defect is exceljs-specific: csv/tsv go through csv-parse, which has no\n // shared-strings table to lose. A run with no persisted `format` defaulted to\n // tab-delimited in the worker, so absence is a real answer, not a gap.\n if (feed.format !== 'xlsx') {\n return {\n verdict: 'undetectable',\n detail: `not an xlsx run (format: ${feed.format ?? 'tsv (default)'}) — the defect is exceljs-specific`,\n }\n }\n // With `hasHeader: false` the reader keys rows positionally and never captures\n // a header, so the guard has nothing to fire on. Say so rather than returning\n // a `clean` that means \"not checked\".\n if (feed.hasHeader === false) {\n return {\n verdict: 'undetectable',\n detail: 'run used hasHeader:false — no header row for the guard to test',\n }\n }\n\n // Dynamic import, matching `read-feed.ts`: exceljs is heavy and Node-only,\n // and this module is reachable from the package's main entry. Bound BEFORE\n // the resolver runs, so a module-resolution failure cannot strand a tmpfile\n // the resolver just downloaded — and outside the try below, because `catch`\n // cannot see a `const` block-scoped to the `try`.\n const { streamXlsx, XlsxEmptyHeaderError, XlsxNotAFileError } = await import('../xlsx-reader.js')\n\n let resolved: { localPath: string; cleanup?: (() => Promise<void>) | undefined }\n try {\n resolved = resolveFilePath ? await resolveFilePath(filePath) : { localPath: filePath }\n } catch (err) {\n // A resolver throw is a storage miss (reaped / deleted / permissions) or a\n // transport failure. `detail` carries which — a 404 and a connection reset\n // read very differently to an operator.\n return { verdict: 'source_unavailable', detail: describeError(err) }\n }\n\n try {\n let sawRow = false\n for await (const _first of streamXlsx({\n filePath: resolved.localPath,\n ...(feed.headerRowIndex !== undefined && { headerRowIndex: feed.headerRowIndex }),\n ...(feed.sheet !== undefined && { sheet: feed.sheet }),\n })) {\n void _first\n sawRow = true\n break\n }\n if (!sawRow) {\n // The reader completed without yielding, which means the guard never ran:\n // an empty worksheet, or a `headerRowIndex` past the last row. Reporting\n // `clean` here would be the comforting-but-false signal this whole audit\n // exists to avoid — nothing was verified.\n return {\n verdict: 'undetectable',\n detail: 'file yielded no data rows — the header guard never ran',\n }\n }\n return { verdict: 'clean' }\n } catch (err) {\n if (err instanceof XlsxEmptyHeaderError) {\n return { verdict: 'corrupt', detail: err.message }\n }\n if (isUnreadableSource(err, XlsxNotAFileError)) {\n return { verdict: 'source_unavailable', detail: describeError(err) }\n }\n return { verdict: 'error', detail: describeError(err) }\n } finally {\n await resolved.cleanup?.().catch(() => {\n // Best-effort tmpfile removal, same posture as the worker: a failed\n // cleanup must not turn a read-only audit into a failure. The path is a\n // /tmp copy the OS reclaims anyway.\n })\n }\n}\n\n/**\n * The counters this audit reads out of `import_run.totals`.\n *\n * A LOSSY view, deliberately: `rowsWarned` / `batchesCompleted` /\n * `patchesApplied` / `patchesOrphaned` are dropped rather than passed through,\n * because a report is easier to read for what it needs than for everything the\n * run recorded. Read the `import_run` row itself if you want the rest.\n */\nexport interface RecordedTotals {\n rowsRead?: number | undefined\n rowsSucceeded?: number | undefined\n rowsFailed?: number | undefined\n rowsSkipped?: number | undefined\n}\n\nfunction readCount(source: Record<string, unknown>, key: string): number | undefined {\n const value = source[key]\n return typeof value === 'number' && Number.isFinite(value) ? value : undefined\n}\n\n/** Narrow the opaque `totals` JSONB to the counters this audit reports. */\nexport function readRecordedTotals(raw: unknown): RecordedTotals {\n if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return {}\n const source = raw as Record<string, unknown>\n return {\n rowsRead: readCount(source, 'rowsRead'),\n rowsSucceeded: readCount(source, 'rowsSucceeded'),\n rowsFailed: readCount(source, 'rowsFailed'),\n rowsSkipped: readCount(source, 'rowsSkipped'),\n }\n}\n\nexport interface AuditedImportRun {\n id: string\n label: string\n /** Verbatim `import_run.filePath` — the storage key, so an operator can go look. */\n filePath: string\n transformName: string\n createdAt: Date | null\n finishedAt: Date | null\n /** Row counters the run recorded. A corrupted run that recorded a plausible count is the most misleading artefact of all. */\n totals: RecordedTotals\n verdict: EmptyHeaderVerdict\n detail?: string\n /**\n * The run finished having read rows but indexed none. Corroborating evidence\n * that survives the source file being reaped, and the ONLY signal available\n * for the transient variant of the defect — but it is also what a wrong\n * column mapping looks like, so it is never folded into the verdict.\n *\n * THREE-VALUED on purpose. `'unknown'` means the run recorded no usable\n * counters (older rows; any run audited with a non-`succeeded` status, since\n * `markFailed` never writes `totals`). Collapsing that to `false` would render\n * absence of evidence as evidence of absence — in the one signal that is\n * supposed to outlive the file.\n */\n zeroRowsIndexed: boolean | 'unknown'\n}\n\nexport interface EmptyHeaderAuditReport {\n /** Total xlsx runs matching the audited statuses. */\n candidates: number\n /** How many of them this invocation actually read. */\n examined: number\n /** `candidates - examined` — dropped by the cap, newest kept. Never silent: also logged. */\n notExamined: number\n counts: Record<EmptyHeaderVerdict, number>\n /** Every examined run, verdict-annotated, newest first. */\n runs: AuditedImportRun[]\n}\n\nexport interface AuditEmptyHeaderRunsArgs {\n /** Toolkit app — provides the app-bound `AdminClient` for `import_run`. */\n app: ToolkitApp\n /**\n * Same resolver the imports plugin hands the worker (typically\n * `storageResolveFilePath`). Omit only when `filePath` is a local FS path.\n */\n resolveFilePath?: FilePathResolver | undefined\n /** Statuses to audit. Default `['succeeded']` — a failed run announced itself. */\n statuses?: readonly ImportRunStatus[] | undefined\n /** Max runs examined, newest first. Default 200, hard ceiling 5000. */\n limit?: number | undefined\n /** Peak concurrent storage reads. Default 4, hard ceiling 16. */\n concurrency?: number | undefined\n logger: Pick<Logger, 'info' | 'warn'>\n}\n\nfunction emptyCounts(): Record<EmptyHeaderVerdict, number> {\n return { corrupt: 0, clean: 0, source_unavailable: 0, undetectable: 0, error: 0 }\n}\n\n/**\n * Derive the corroborating flag from whatever counters the run recorded.\n * `rowsRead` is what the reader yielded; `rowsSucceeded` is what the cluster\n * acknowledged. Read-but-none-indexed is the shape a `{}`-row stream leaves\n * behind however the transform rejected it (error OR skip), which is why the\n * test is on succeeded rather than failed.\n */\nfunction deriveZeroRowsIndexed(totals: RecordedTotals): boolean | 'unknown' {\n if (totals.rowsRead === undefined || totals.rowsSucceeded === undefined) return 'unknown'\n return totals.rowsRead > 0 && totals.rowsSucceeded === 0\n}\n\n/** Coerce a DTO date field (Date | string | null | undefined) to `Date | null`. */\nfunction toDate(value: unknown): Date | null {\n if (value instanceof Date) return value\n if (typeof value === 'string') {\n const parsed = new Date(value)\n return Number.isNaN(parsed.getTime()) ? null : parsed\n }\n return null\n}\n\n/**\n * Audit past import runs for the silent empty-header corruption.\n *\n * Newest-first, because the answerable window is the recent one — the retention\n * sweep reaps source files ~7 days after a run reaches terminal status by\n * default, so the oldest runs are the ones whose evidence is already gone.\n */\nexport async function auditEmptyHeaderRuns(\n args: AuditEmptyHeaderRunsArgs,\n): Promise<EmptyHeaderAuditReport> {\n const { app, resolveFilePath, logger } = args\n const statuses = args.statuses ?? DEFAULT_STATUSES\n if (statuses.length === 0) {\n throw new Error('auditEmptyHeaderRuns: statuses must name at least one import_run status')\n }\n const limit = args.limit ?? DEFAULT_LIMIT\n if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) {\n throw new Error(\n `auditEmptyHeaderRuns: limit must be an integer in [1, ${MAX_LIMIT}] (got ${limit})`,\n )\n }\n // Concurrency needs its own ceiling, not just `runBounded`'s\n // positive-integer check: `{ limit: 5000, concurrency: 5000 }` type-checks and\n // is a 5000-wide burst at object storage, which is the amplification the item\n // cap alone does not prevent.\n const concurrency = args.concurrency ?? DEFAULT_BOUNDED_CONCURRENCY\n if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) {\n throw new Error(\n `auditEmptyHeaderRuns: concurrency must be an integer in [1, ${MAX_CONCURRENCY}] (got ${concurrency})`,\n )\n }\n\n const client = app.getClient(ImportRun)\n\n const { and, desc, getTableColumns, inArray, sql } = await import('drizzle-orm')\n type AnyPgColumn = import('drizzle-orm/pg-core').AnyPgColumn\n\n // `getTableColumns` rather than dot-access: the entity table is a generic\n // PgTable, so direct access widens every column to `T | undefined` (same\n // shape the cleanup helpers use).\n const cols = getTableColumns(client.getTable()) as Record<string, AnyPgColumn>\n const statusCol = cols.status\n const paramsCol = cols.params\n const createdAtCol = cols.createdAt\n if (!statusCol || !paramsCol || !createdAtCol) {\n throw new Error(\n 'auditEmptyHeaderRuns: import_run schema missing expected columns (status/params/createdAt)',\n )\n }\n\n // Narrow to xlsx runs IN SQL rather than paging every run and filtering in\n // JS — otherwise the `limit` cap would be spent on tsv rows and a deployment\n // with a long carmaker history would examine nothing. `params` is jsonb, and\n // a JSONB operator inside Drizzle's typed builder is the sanctioned use of\n // the `sql` template (CLAUDE.md \"Database Queries\" rule 5): the column ref is\n // a typed Drizzle object and the compared value is bound as a parameter. The\n // key path mirrors the worker's `params.feed` rehydration exactly.\n const where = and(\n inArray(statusCol, [...statuses]),\n sql`${paramsCol}->'feed'->>'format' = ${'xlsx'}`,\n )\n\n const candidates = await client.count({ where })\n const rows = await client.findMany({ where, orderBy: desc(createdAtCol), limit })\n\n const notExamined = Math.max(0, candidates - rows.length)\n if (notExamined > 0) {\n logger.warn(\n { candidates, examined: rows.length, notExamined, limit },\n 'imports audit: capped at the most recent runs — re-run with a higher limit to cover the rest',\n )\n }\n if (rows.length === 0) {\n logger.info({ statuses }, 'imports audit: no xlsx runs to audit')\n return { candidates, examined: 0, notExamined, counts: emptyCounts(), runs: [] }\n }\n\n logger.info(\n { candidates, examining: rows.length, concurrency },\n 'imports audit: re-reading source files for the empty-header defect',\n )\n\n const audited = await runBounded(\n rows,\n async (row): Promise<AuditedImportRun> => {\n // `filePath` / `label` / `transformName` are `required: true`, so the\n // inferred DTO already types them as `string`. `totals` is `field.json()`.\n const filePath = row.filePath\n const totals = readRecordedTotals(row.totals)\n const base = {\n id: row.id,\n label: row.label,\n filePath,\n transformName: row.transformName,\n createdAt: toDate(row.createdAt),\n finishedAt: toDate(row.finishedAt),\n totals,\n zeroRowsIndexed: deriveZeroRowsIndexed(totals),\n }\n\n // EVERY per-item failure degrades THIS run to `error` and no further.\n // `runBounded` rejects on the first worker rejection, so an unhandled\n // throw here would discard all N results — and the report is the entire\n // product of this function.\n try {\n // Rehydrated through the SAME helper the worker uses, so the audit\n // cannot re-read a run with options that run never used.\n const feedParsed = readPersistedFeedConfig(row.params)\n if (!feedParsed.success) {\n const reason = feedParsed.error.issues[0]?.message ?? 'invalid feed reader config'\n return { ...base, verdict: 'error', detail: `invalid params.feed: ${reason}` }\n }\n\n const result = await inspectImportRunSource({\n filePath,\n feed: feedParsed.data,\n ...(resolveFilePath !== undefined && { resolveFilePath }),\n })\n return {\n ...base,\n verdict: result.verdict,\n ...(result.detail !== undefined && { detail: result.detail }),\n }\n } catch (err) {\n return { ...base, verdict: 'error', detail: describeError(err) }\n }\n },\n concurrency,\n )\n\n const counts = emptyCounts()\n for (const run of audited) {\n counts[run.verdict] += 1\n if (run.verdict === 'corrupt') {\n logger.warn(\n {\n importRunId: run.id,\n label: run.label,\n filePath: run.filePath,\n transformName: run.transformName,\n createdAt: run.createdAt,\n totals: run.totals,\n },\n 'imports audit: SILENTLY CORRUPTED run — its source file has no readable header',\n )\n }\n }\n\n const zeroRowsIndexed = audited.filter((run) => run.zeroRowsIndexed === true).length\n const totalsUnknown = audited.filter((run) => run.zeroRowsIndexed === 'unknown').length\n\n // A run of nothing but `source_unavailable` usually means the audit was\n // pointed at storage keys with no resolver, not that a week of evidence\n // evaporated. Those read identically in the report, so say which is likelier.\n if (resolveFilePath === undefined && counts.source_unavailable === audited.length) {\n logger.warn(\n { examined: audited.length },\n 'imports audit: every source was unreadable and no resolveFilePath was given — ' +\n 'if filePath holds storage keys rather than local paths, pass storageResolveFilePath',\n )\n }\n\n logger.info(\n {\n candidates,\n examined: audited.length,\n notExamined,\n ...counts,\n zeroRowsIndexed,\n totalsUnknown,\n },\n 'imports audit: complete',\n )\n\n return { candidates, examined: audited.length, notExamined, counts, runs: audited }\n}\n","/**\n * `previewFeed` — read the first N data rows of a feed for the PR3 import wizard\n * (commerce-demo phase 02 §4). Reuses {@link readFeed} so the preview is parsed by\n * the EXACT same reader (csv-parse / exceljs), with the same delimiter / header /\n * sheet config, that the run will use — the operator maps against what they'll get,\n * not a re-implemented best-guess parser (R02: drift between preview + run is a\n * mapping foot-gun).\n *\n * Bounded: it consumes only `maxRows` rows then stops iterating, so on a delimited\n * feed it streams a tiny prefix rather than the whole file. (The caller materialises\n * the storage object to a local path first — see the commerce preview route — so for\n * a very large upload the materialise cost dominates; that's a demo-depth trade-off,\n * noted on the route.)\n */\n\nimport { type ReadFeedOptions, readFeed } from './read-feed.js'\n\n/** Hard ceiling on preview rows regardless of caller request — bounds work + payload. */\nexport const MAX_PREVIEW_ROWS = 100\n/** Default sample size when the caller doesn't specify. */\nexport const DEFAULT_PREVIEW_ROWS = 20\n\nexport interface PreviewFeedOptions extends ReadFeedOptions {\n /** Rows to sample. Defaults to {@link DEFAULT_PREVIEW_ROWS}; clamped to [1, {@link MAX_PREVIEW_ROWS}]. */\n maxRows?: number\n}\n\nexport interface FeedPreview {\n /**\n * Source column names in first-seen order. With a header these are the (trimmed)\n * header names; headerless feeds yield positional `\"0\"`, `\"1\"`, … keys. The union\n * across sampled rows covers feeds whose first row omits trailing empty cells.\n */\n columns: string[]\n /** Up to `maxRows` sampled data rows, each a `{ column → cell }` map (cells are strings). */\n rows: Array<Record<string, string>>\n}\n\n/**\n * Read up to `maxRows` data rows from a feed file and return the detected columns +\n * sample rows. Throws whatever the underlying reader throws (bad sheet index,\n * duplicate header, unreadable file) — the caller maps that to a 4xx.\n */\nexport async function previewFeed(options: PreviewFeedOptions): Promise<FeedPreview> {\n const { maxRows, ...feedOptions } = options\n // Coerce to a finite integer BEFORE clamping: `previewFeed` is public API, so a\n // caller passing `NaN`/`Infinity`/fractional must not slip through. `NaN` in\n // particular would make `rows.length >= limit` always false → read the whole file.\n const requested = maxRows ?? DEFAULT_PREVIEW_ROWS\n const safeRows = Number.isFinite(requested) ? Math.floor(requested) : DEFAULT_PREVIEW_ROWS\n const limit = Math.min(MAX_PREVIEW_ROWS, Math.max(1, safeRows))\n\n const rows: Array<Record<string, string>> = []\n // First-seen-order column set: a Map preserves insertion order and dedupes.\n const columns = new Map<string, true>()\n\n // Preview is EXPLORATORY — force lenient column counts (csv/tsv). With the wrong\n // delimiter a feed parses as a single odd column, OR a row's field count differs\n // from the header; strict parsing would abort the whole preview with a cryptic\n // \"Invalid Record Length\" instead of letting the operator SEE the bad parse and\n // fix the delimiter/format, then re-preview. The actual run keeps its own\n // (stricter) settings.\n for await (const { row } of readFeed({ ...feedOptions, relaxColumnCount: true })) {\n for (const key of Object.keys(row)) columns.set(key, true)\n rows.push(row)\n if (rows.length >= limit) break\n }\n\n return { columns: [...columns.keys()], rows }\n}\n"],"mappings":"2gBA8EA,MAAa,EAAwB,CACnC,UACA,QACA,qBACA,eACA,OACF,EAIM,EAA+C,CAAC,WAAW,EAM3D,EAAY,IAyBlB,SAAS,EAAc,EAAsB,CAE3C,OADI,aAAe,MAAc,EAAI,QAC9B,OAAO,CAAG,CACnB,CAcA,MAAM,EAA0B,IAAI,IAAI,CAAC,SAAU,SAAU,QAAS,SAAU,SAAS,CAAC,EAQpF,EAA4B,aAElC,SAAS,EAAmB,EAAc,EAAoD,CAC5F,GAAI,OAAO,GAAQ,WAAY,EAAc,MAAO,GAGpD,GAAI,aAAe,EAAU,MAAO,GACpC,GAAI,SAAU,EAAK,CACjB,GAAM,CAAE,QAAS,EACjB,GAAI,OAAO,GAAS,UAAY,EAAwB,IAAI,CAAI,EAAG,MAAO,EAC5E,CACA,OAAO,aAAe,OAAS,EAA0B,KAAK,EAAI,OAAO,CAC3E,CAgBA,eAAsB,EACpB,EACuC,CACvC,GAAM,CAAE,WAAU,OAAM,mBAAoB,EAK5C,GAAI,EAAK,SAAW,OAClB,MAAO,CACL,QAAS,eACT,OAAQ,4BAA4B,EAAK,QAAU,gBAAgB,mCACrE,EAKF,GAAI,EAAK,YAAc,GACrB,MAAO,CACL,QAAS,eACT,OAAQ,gEACV,EAQF,GAAM,CAAE,aAAY,uBAAsB,qBAAsB,MAAM,OAAO,8BAEzE,EACJ,GAAI,CACF,EAAW,EAAkB,MAAM,EAAgB,CAAQ,EAAI,CAAE,UAAW,CAAS,CACvF,OAAS,EAAK,CAIZ,MAAO,CAAE,QAAS,qBAAsB,OAAQ,EAAc,CAAG,CAAE,CACrE,CAEA,GAAI,CACF,IAAI,EAAS,GACb,UAAW,IAAM,KAAU,EAAW,CACpC,SAAU,EAAS,UACnB,GAAI,EAAK,iBAAmB,IAAA,IAAa,CAAE,eAAgB,EAAK,cAAe,EAC/E,GAAI,EAAK,QAAU,IAAA,IAAa,CAAE,MAAO,EAAK,KAAM,CACtD,CAAC,EAAG,CAEF,EAAS,GACT,KACF,CAWA,OAVK,EAUE,CAAE,QAAS,OAAQ,EALjB,CACL,QAAS,eACT,OAAQ,wDACV,CAGJ,OAAS,EAAK,CAOZ,OANI,aAAe,EACV,CAAE,QAAS,UAAW,OAAQ,EAAI,OAAQ,EAE/C,EAAmB,EAAK,CAAiB,EACpC,CAAE,QAAS,qBAAsB,OAAQ,EAAc,CAAG,CAAE,EAE9D,CAAE,QAAS,QAAS,OAAQ,EAAc,CAAG,CAAE,CACxD,QAAU,CACR,MAAM,EAAS,UAAU,CAAC,CAAC,UAAY,CAIvC,CAAC,CACH,CACF,CAiBA,SAAS,EAAU,EAAiC,EAAiC,CACnF,IAAM,EAAQ,EAAO,GACrB,OAAO,OAAO,GAAU,UAAY,OAAO,SAAS,CAAK,EAAI,EAAQ,IAAA,EACvE,CAGA,SAAgB,EAAmB,EAA8B,CAC/D,GAAI,OAAO,GAAQ,WAAY,GAAgB,MAAM,QAAQ,CAAG,EAAG,MAAO,CAAC,EAC3E,IAAM,EAAS,EACf,MAAO,CACL,SAAU,EAAU,EAAQ,UAAU,EACtC,cAAe,EAAU,EAAQ,eAAe,EAChD,WAAY,EAAU,EAAQ,YAAY,EAC1C,YAAa,EAAU,EAAQ,aAAa,CAC9C,CACF,CA0DA,SAAS,GAAkD,CACzD,MAAO,CAAE,QAAS,EAAG,MAAO,EAAG,mBAAoB,EAAG,aAAc,EAAG,MAAO,CAAE,CAClF,CASA,SAAS,EAAsB,EAA6C,CAE1E,OADI,EAAO,WAAa,IAAA,IAAa,EAAO,gBAAkB,IAAA,GAAkB,UACzE,EAAO,SAAW,GAAK,EAAO,gBAAkB,CACzD,CAGA,SAAS,EAAO,EAA6B,CAC3C,GAAI,aAAiB,KAAM,OAAO,EAClC,GAAI,OAAO,GAAU,SAAU,CAC7B,IAAM,EAAS,IAAI,KAAK,CAAK,EAC7B,OAAO,OAAO,MAAM,EAAO,QAAQ,CAAC,EAAI,KAAO,CACjD,CACA,OAAO,IACT,CASA,eAAsB,EACpB,EACiC,CACjC,GAAM,CAAE,MAAK,kBAAiB,UAAW,EACnC,EAAW,EAAK,UAAY,EAClC,GAAI,EAAS,SAAW,EACtB,MAAU,MAAM,yEAAyE,EAE3F,IAAM,EAAQ,EAAK,OAAS,IAC5B,GAAI,CAAC,OAAO,UAAU,CAAK,GAAK,EAAQ,GAAK,EAAQ,EACnD,MAAU,MACR,yDAAyD,EAAU,SAAS,EAAM,EACpF,EAMF,IAAM,EAAc,EAAK,aAAe,EACxC,GAAI,CAAC,OAAO,UAAU,CAAW,GAAK,EAAc,GAAK,EAAc,GACrE,MAAU,MACR,wEAAwF,EAAY,EACtG,EAGF,IAAM,EAAS,EAAI,UAAU,CAAS,EAEhC,CAAE,MAAK,OAAM,kBAAiB,UAAS,OAAQ,MAAM,OAAO,eAM5D,EAAO,EAAgB,EAAO,SAAS,CAAC,EACxC,EAAY,EAAK,OACjB,EAAY,EAAK,OACjB,EAAe,EAAK,UAC1B,GAAI,CAAC,GAAa,CAAC,GAAa,CAAC,EAC/B,MAAU,MACR,4FACF,EAUF,IAAM,EAAQ,EACZ,EAAQ,EAAW,CAAC,GAAG,CAAQ,CAAC,EAChC,CAAG,GAAG,EAAU,wBAAwB,QAC1C,EAEM,EAAa,MAAM,EAAO,MAAM,CAAE,OAAM,CAAC,EACzC,EAAO,MAAM,EAAO,SAAS,CAAE,QAAO,QAAS,EAAK,CAAY,EAAG,OAAM,CAAC,EAE1E,EAAc,KAAK,IAAI,EAAG,EAAa,EAAK,MAAM,EAOxD,GANI,EAAc,GAChB,EAAO,KACL,CAAE,aAAY,SAAU,EAAK,OAAQ,cAAa,OAAM,EACxD,8FACF,EAEE,EAAK,SAAW,EAElB,OADA,EAAO,KAAK,CAAE,UAAS,EAAG,sCAAsC,EACzD,CAAE,aAAY,SAAU,EAAG,cAAa,OAAQ,EAAY,EAAG,KAAM,CAAC,CAAE,EAGjF,EAAO,KACL,CAAE,aAAY,UAAW,EAAK,OAAQ,aAAY,EAClD,oEACF,EAEA,IAAM,EAAU,MAAM,EACpB,EACA,KAAO,IAAmC,CAGxC,IAAM,EAAW,EAAI,SACf,EAAS,EAAmB,EAAI,MAAM,EACtC,EAAO,CACX,GAAI,EAAI,GACR,MAAO,EAAI,MACX,WACA,cAAe,EAAI,cACnB,UAAW,EAAO,EAAI,SAAS,EAC/B,WAAY,EAAO,EAAI,UAAU,EACjC,SACA,gBAAiB,EAAsB,CAAM,CAC/C,EAMA,GAAI,CAGF,IAAM,EAAa,EAAwB,EAAI,MAAM,EACrD,GAAI,CAAC,EAAW,QAAS,CACvB,IAAM,EAAS,EAAW,MAAM,OAAO,EAAE,EAAE,SAAW,6BACtD,MAAO,CAAE,GAAG,EAAM,QAAS,QAAS,OAAQ,wBAAwB,GAAS,CAC/E,CAEA,IAAM,EAAS,MAAM,EAAuB,CAC1C,WACA,KAAM,EAAW,KACjB,GAAI,IAAoB,IAAA,IAAa,CAAE,iBAAgB,CACzD,CAAC,EACD,MAAO,CACL,GAAG,EACH,QAAS,EAAO,QAChB,GAAI,EAAO,SAAW,IAAA,IAAa,CAAE,OAAQ,EAAO,MAAO,CAC7D,CACF,OAAS,EAAK,CACZ,MAAO,CAAE,GAAG,EAAM,QAAS,QAAS,OAAQ,EAAc,CAAG,CAAE,CACjE,CACF,EACA,CACF,EAEM,EAAS,EAAY,EAC3B,IAAK,IAAM,KAAO,EAChB,EAAO,EAAI,UAAY,EACnB,EAAI,UAAY,WAClB,EAAO,KACL,CACE,YAAa,EAAI,GACjB,MAAO,EAAI,MACX,SAAU,EAAI,SACd,cAAe,EAAI,cACnB,UAAW,EAAI,UACf,OAAQ,EAAI,MACd,EACA,gFACF,EAIJ,IAAM,EAAkB,EAAQ,OAAQ,GAAQ,EAAI,kBAAoB,EAAI,CAAC,CAAC,OACxE,EAAgB,EAAQ,OAAQ,GAAQ,EAAI,kBAAoB,SAAS,CAAC,CAAC,OAyBjF,OApBI,IAAoB,IAAA,IAAa,EAAO,qBAAuB,EAAQ,QACzE,EAAO,KACL,CAAE,SAAU,EAAQ,MAAO,EAC3B,mKAEF,EAGF,EAAO,KACL,CACE,aACA,SAAU,EAAQ,OAClB,cACA,GAAG,EACH,kBACA,eACF,EACA,yBACF,EAEO,CAAE,aAAY,SAAU,EAAQ,OAAQ,cAAa,SAAQ,KAAM,CAAQ,CACpF,CC3gBA,MAAa,EAAmB,IAEnB,EAAuB,GAuBpC,eAAsB,EAAY,EAAmD,CACnF,GAAM,CAAE,UAAS,GAAG,GAAgB,EAI9B,EAAY,GAAA,GAEZ,EAAQ,KAAK,IAAA,IAAsB,KAAK,IAAI,EADjC,OAAO,SAAS,CAAS,EAAI,KAAK,MAAM,CAAS,EAAA,EACL,CAAC,EAExD,EAAsC,CAAC,EAEvC,EAAU,IAAI,IAQpB,UAAW,GAAM,CAAE,SAAS,EAAS,CAAE,GAAG,EAAa,iBAAkB,EAAK,CAAC,EAAG,CAChF,IAAK,IAAM,KAAO,OAAO,KAAK,CAAG,EAAG,EAAQ,IAAI,EAAK,EAAI,EAEzD,GADA,EAAK,KAAK,CAAG,EACT,EAAK,QAAU,EAAO,KAC5B,CAEA,MAAO,CAAE,QAAS,CAAC,GAAG,EAAQ,KAAK,CAAC,EAAG,MAAK,CAC9C"}
@@ -0,0 +1,42 @@
1
+ import { i as RowTransform } from "../transform-WeiiLdYP.mjs";
2
+ import { C as columnTransformSchema, D as toMinorUnits, E as parseNumber, S as applyColumnTransforms, T as parseEuropeanNumber, _ as COLUMN_TRANSFORM_OPS, a as mappingColumnSchema, b as ColumnTransformOp, c as resolveMapping, d as NumberOutcome, f as TargetField, g as textOrNull, h as rowError, i as buildMappingTransform, l as BuildRowContext, m as numberCell, n as MappingConfig, o as mappingConfigSchema, p as TargetSchema, r as ResolvableMapping, s as numberFormatSchema, t as MappingColumn, u as MappedRow, v as CellValue, w as NumberFormat, x as ColumnTransformOpDescriptor, y as ColumnTransform } from "../engine-tkbUXKxB.mjs";
3
+ import { i as partsSchema, n as normalizeCode, r as partsParamsSchema, t as PartsParams } from "../parts-schema-M1AtjNJt.mjs";
4
+
5
+ //#region src/mapping/discount-table-schema.d.ts
6
+ /** One parsed discount-table row: a group token + the purchase discount % off list. */
7
+ interface DiscountTableRow {
8
+ discountGroup: string;
9
+ discountPct: number;
10
+ }
11
+ declare const discountTableSchema: TargetSchema<DiscountTableRow>;
12
+ //#endregion
13
+ //#region src/mapping/mapping-transform.d.ts
14
+ /** Stable name under which the mapping-driven transform is registered. */
15
+ declare const MAPPING_TRANSFORM_NAME = "commerce:mapping";
16
+ /**
17
+ * The registered transform. Resolves + compiles the mapping snapshot from
18
+ * `ctx.params` (memoized), then delegates each row to it. A malformed snapshot or
19
+ * unknown schema throws — surfaced by the runner as a row-level `transform_threw`.
20
+ */
21
+ declare const mappingTransform: RowTransform<unknown>;
22
+ //#endregion
23
+ //#region src/mapping/schema-registry.d.ts
24
+ /** Resolve a registered schema by name, or `undefined` if not registered. */
25
+ declare function getTargetSchema(name: string): TargetSchema<unknown> | undefined;
26
+ /** All registered schema names (for the wizard's schema picker, PR3). */
27
+ declare function listTargetSchemas(): string[];
28
+ /** Operator-facing summary of a target schema: its name + the field whitelist (PR3 wizard). */
29
+ interface TargetSchemaSummary {
30
+ name: string;
31
+ fields: readonly TargetField[];
32
+ }
33
+ /**
34
+ * Describe every registered target schema (name + field whitelist) for the PR3
35
+ * mapping-wizard UI. Returns only metadata (no `buildRow`), so the wizard's RSC page
36
+ * can ship it across the client boundary as plain JSON without dragging the engine
37
+ * into the client bundle.
38
+ */
39
+ declare function describeTargetSchemas(): TargetSchemaSummary[];
40
+ //#endregion
41
+ export { type BuildRowContext, COLUMN_TRANSFORM_OPS, type CellValue, type ColumnTransform, type ColumnTransformOp, type ColumnTransformOpDescriptor, type DiscountTableRow, MAPPING_TRANSFORM_NAME, type MappedRow, type MappingColumn, type MappingConfig, type NumberFormat, type NumberOutcome, type PartsParams, type ResolvableMapping, type TargetField, type TargetSchema, type TargetSchemaSummary, applyColumnTransforms, buildMappingTransform, columnTransformSchema, describeTargetSchemas, discountTableSchema, getTargetSchema, listTargetSchemas, mappingColumnSchema, mappingConfigSchema, mappingTransform, normalizeCode, numberCell, numberFormatSchema, parseEuropeanNumber, parseNumber, partsParamsSchema, partsSchema, resolveMapping, rowError, textOrNull, toMinorUnits };
42
+ //# sourceMappingURL=index.d.mts.map