@murumets-ee/imports 0.34.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{cleanup-BzqyX9Jo.mjs → cleanup-DsQ42ki-.mjs} +2 -2
- package/dist/{cleanup-BzqyX9Jo.mjs.map → cleanup-DsQ42ki-.mjs.map} +1 -1
- package/dist/index.d.mts +3 -3
- package/dist/index.mjs +1 -1
- package/dist/mapping/index.d.mts +3 -3
- package/dist/mapping/index.mjs +1 -1
- package/dist/mapping/index.mjs.map +1 -1
- package/dist/{parts-schema-Bqau0MrJ.d.mts → parts-schema-Cmb0URGO.d.mts} +11 -3
- package/dist/{parts-schema-Bqau0MrJ.d.mts.map → parts-schema-Cmb0URGO.d.mts.map} +1 -1
- package/dist/{parts-schema-Vv0EJCjj.mjs → parts-schema-RH48v5VW.mjs} +2 -2
- package/dist/parts-schema-RH48v5VW.mjs.map +1 -0
- package/dist/plugin.d.mts +9 -1
- package/dist/plugin.d.mts.map +1 -1
- package/dist/plugin.mjs +1 -1
- package/dist/plugin.mjs.map +1 -1
- package/dist/runner-BOwPUm7W.mjs +2 -0
- package/dist/runner-BOwPUm7W.mjs.map +1 -0
- package/dist/{schema-registry-JU4gsC8E.mjs → schema-registry-TK4sBFmD.mjs} +2 -2
- package/dist/{schema-registry-JU4gsC8E.mjs.map → schema-registry-TK4sBFmD.mjs.map} +1 -1
- package/dist/storage-resolver.d.mts +1 -1
- package/dist/{transform-BRJJLSXE.d.mts → transform-DqteN-Lm.d.mts} +65 -2
- package/dist/transform-DqteN-Lm.d.mts.map +1 -0
- package/dist/{transform-BzacGXbo.mjs → transform-ytS1L5p6.mjs} +2 -2
- package/dist/{transform-BzacGXbo.mjs.map → transform-ytS1L5p6.mjs.map} +1 -1
- package/dist/transforms.d.mts +2 -2
- package/dist/transforms.d.mts.map +1 -1
- package/dist/transforms.mjs +1 -1
- package/dist/transforms.mjs.map +1 -1
- package/dist/{worker-Ck67AOnh.d.mts → worker-CO5zK62E.d.mts} +68 -3
- package/dist/{worker-Ck67AOnh.d.mts.map → worker-CO5zK62E.d.mts.map} +1 -1
- package/dist/worker.d.mts +2 -2
- package/dist/worker.mjs +1 -1
- package/dist/worker.mjs.map +1 -1
- package/dist/{xlsx-reader-oApgZRJd.mjs → xlsx-reader-CUbaPt3-.mjs} +2 -2
- package/dist/{xlsx-reader-oApgZRJd.mjs.map → xlsx-reader-CUbaPt3-.mjs.map} +1 -1
- package/package.json +8 -8
- package/dist/parts-schema-Vv0EJCjj.mjs.map +0 -1
- package/dist/runner-0r6UICjo.mjs +0 -2
- package/dist/runner-0r6UICjo.mjs.map +0 -1
- package/dist/transform-BRJJLSXE.d.mts.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { c as TransformRegistry, i as RowTransform, t as DirectionalPair } from "./transform-DqteN-Lm.mjs";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { EsClientLike } from "@murumets-ee/search-elasticsearch";
|
|
4
4
|
import { Logger } from "@murumets-ee/core";
|
|
@@ -473,6 +473,29 @@ interface RunImportResult {
|
|
|
473
473
|
errors: ReturnType<ErrorTracker['snapshot']>;
|
|
474
474
|
/** Aggregated NON-FATAL warnings (commerce-demo PR4). Empty when no row warned. */
|
|
475
475
|
warnings: ReturnType<ErrorTracker['snapshot']>;
|
|
476
|
+
/**
|
|
477
|
+
* Directional supersession pairs harvested as a side-output (commerce-demo PR6,
|
|
478
|
+
* SD007). Buffered in memory from ES-ACCEPTED rows only (same accounting as
|
|
479
|
+
* warnings — a row whose doc was rejected by ES contributes no pair). Empty unless
|
|
480
|
+
* a schema emits them (the `parts` schema, when a row's `superseding_code` is set).
|
|
481
|
+
* The worker hands these to an injected `pairSink` AFTER the run for the commerce
|
|
482
|
+
* replacement-merge engine; the runner itself never writes the consumer's tables.
|
|
483
|
+
*/
|
|
484
|
+
pairs: readonly DirectionalPair[];
|
|
485
|
+
/**
|
|
486
|
+
* Doc-patches applied by the phase-2 pass (commerce-demo PR7): a `<code>*S`
|
|
487
|
+
* core-fee row is not a saleable doc — it emits a {@link DocPatch} that merges its
|
|
488
|
+
* deposit onto the PARENT part's `core_charge_minor`. The runner buffers these
|
|
489
|
+
* during the stream (sparse) and applies them via `bulkPartialUpdate` AFTER the ES
|
|
490
|
+
* pass. `patchesApplied` = targets the cluster merged; `patchesOrphaned` = patches
|
|
491
|
+
* whose parent genuinely wasn't in the index (a `document_missing`/404 → an
|
|
492
|
+
* `orphan_core_charge` warning). An INFRA failure (whole-batch transport error, or a
|
|
493
|
+
* non-404 per-doc failure) is NOT counted as orphaned — it lands in a distinct
|
|
494
|
+
* `patch_attach_failed` warning bucket so it can't masquerade as a data orphan. Zero
|
|
495
|
+
* unless a schema emits patches.
|
|
496
|
+
*/
|
|
497
|
+
patchesApplied: number;
|
|
498
|
+
patchesOrphaned: number;
|
|
476
499
|
}
|
|
477
500
|
/**
|
|
478
501
|
* Apply the runner against a feed file. Stops on rowLimit OR end-of-file
|
|
@@ -576,6 +599,41 @@ type FilePathResolver = (storageKey: string) => Promise<{
|
|
|
576
599
|
localPath: string;
|
|
577
600
|
cleanup?: () => Promise<void>;
|
|
578
601
|
}>;
|
|
602
|
+
/**
|
|
603
|
+
* Context handed to a {@link PairSink} alongside the harvested pairs. Carries the
|
|
604
|
+
* run identity so the sink can build `source` attribution
|
|
605
|
+
* (`feed:<supplierId>:<importBatchId>`, D20) and scope its writes.
|
|
606
|
+
*/
|
|
607
|
+
interface PairSinkContext {
|
|
608
|
+
importRunId: string;
|
|
609
|
+
/** The `import_run.params` object — the sink reads `supplierId` / `importBatchId`. */
|
|
610
|
+
params: Record<string, unknown>;
|
|
611
|
+
/**
|
|
612
|
+
* The run's structured logger (child), forwarded so the sink's INTERNAL telemetry —
|
|
613
|
+
* a params-parse miss that skips ingestion, an over-merge flag — is visible. Without it
|
|
614
|
+
* a misconfigured run looks successful from the worker's vantage (a params-miss returns
|
|
615
|
+
* normally, so the worker logs "applied"). Optional; the sink no-ops its logs if absent.
|
|
616
|
+
*/
|
|
617
|
+
logger?: Logger;
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* Optional post-run consumer for the directional supersession pairs the runner
|
|
621
|
+
* harvested as a side-output (commerce-demo PR6, SD007). Invoked ONCE after a
|
|
622
|
+
* successful run with the full in-memory pair buffer (read-once / merge-after) so
|
|
623
|
+
* the consumer (the commerce replacement-merge engine) can build equivalence groups
|
|
624
|
+
* and bulk-write them to Postgres.
|
|
625
|
+
*
|
|
626
|
+
* **Why injected, not imported.** The merge writes the CONSUMER's Postgres tables,
|
|
627
|
+
* but the runner + this worker live in the `@murumets-ee/imports` leaf, which must
|
|
628
|
+
* not depend on commerce. The app composes the sink (commerce exports it, the app
|
|
629
|
+
* passes it to `imports({ pairSink })`) — the same injection shape as `esClient` /
|
|
630
|
+
* `resolveFilePath` / `ensureIndex`. The boundary holds.
|
|
631
|
+
*
|
|
632
|
+
* **Best-effort.** A sink throw is logged and SWALLOWED — the catalogue import
|
|
633
|
+
* (the ES write) has already succeeded and must not be undone by a secondary
|
|
634
|
+
* replacement-ingestion failure. The merge is idempotent, so a re-import recovers.
|
|
635
|
+
*/
|
|
636
|
+
type PairSink = (pairs: readonly DirectionalPair[], ctx: PairSinkContext) => Promise<void>;
|
|
579
637
|
interface RunImportHandlerConfig {
|
|
580
638
|
/** AdminClient over the `import_run` entity. */
|
|
581
639
|
importRuns: ImportRunClient;
|
|
@@ -597,6 +655,13 @@ interface RunImportHandlerConfig {
|
|
|
597
655
|
* on-disk PoC setups.
|
|
598
656
|
*/
|
|
599
657
|
resolveFilePath?: FilePathResolver;
|
|
658
|
+
/**
|
|
659
|
+
* Optional consumer for the directional supersession pairs harvested as a
|
|
660
|
+
* side-output (commerce-demo PR6, SD007). See {@link PairSink}. Omit for
|
|
661
|
+
* deployments that don't ingest replacement codes — the pairs are simply
|
|
662
|
+
* discarded after the run.
|
|
663
|
+
*/
|
|
664
|
+
pairSink?: PairSink;
|
|
600
665
|
/** Optional structured logger. Defaults to silent. */
|
|
601
666
|
logger?: Logger;
|
|
602
667
|
}
|
|
@@ -655,5 +720,5 @@ interface ImportsRunDeadListenerConfig {
|
|
|
655
720
|
*/
|
|
656
721
|
declare function createImportsRunDeadListener(config: ImportsRunDeadListenerConfig): QueueTerminalNotifier;
|
|
657
722
|
//#endregion
|
|
658
|
-
export {
|
|
659
|
-
//# sourceMappingURL=worker-
|
|
723
|
+
export { ErrorSample as A, StreamFeedOptions as C, DEFAULT_MAX_PATTERNS as D, streamFeed as E, ImportRunClient as F, ImportRunStatus as I, ErrorTrackerConfig as M, IMPORT_RUN_STATUSES as N, DEFAULT_MAX_SAMPLES_PER_PATTERN as O, ImportRun as P, readFeed as S, countDataRows as T, RunImportResult as _, ImportsRunJobContext as a, ReadFeedOptions as b, PairSinkContext as c, createRunImportHandler as d, importsRunJob as f, RunImportOptions as g, ImportRunProgress as h, ImportsRunDeadListenerConfig as i, ErrorTracker as j, ErrorPattern as k, RunImportHandlerConfig as l, DEFAULT_BATCH_SIZE as m, EsClientResolver as n, ImportsRunJobPayload as o, importsRunJobPayloadSchema as p, FilePathResolver as r, PairSink as s, EnsureIndexResolver as t, createImportsRunDeadListener as u, runImport as v, StreamFeedRow as w, feedReaderConfigSchema as x, FeedReaderConfig as y };
|
|
724
|
+
//# sourceMappingURL=worker-CO5zK62E.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-
|
|
1
|
+
{"version":3,"file":"worker-CO5zK62E.d.mts","names":[],"sources":["../src/entities/import-run.ts","../src/error-tracker.ts","../src/streaming.ts","../src/read-feed.ts","../src/runner.ts","../src/worker.ts"],"mappings":";;;;;;;;AA8ByD;AAEzD;;;;;;;;AAFyD,cAD5C,mBAAA;AAAA,KACD,eAAA,WAA0B,mBAAmB;AAAA,cAE5C,SAAA,gCAAS,MAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA8FV;;;;;;;;AAAwD;;;;AC9FpE;;8GAMoC;yGAA7B;;EAA6B;AAAA;AAGpC;;;;;;;KDqFY,eAAA,GAAkB,WAAW,QAAQ,SAAA,CAAU,SAAA;;;;;;;;;;;AAjG3D;;;;AAAsG;AACtG;;;;AAAyD;AAEzD;;;;;;;;;;;;;KCAY,eAAA,sCAKR,eAAA;EAAA,CACG,GAAA,WAAc,eAAe;AAAA;;UAGnB,WAAA;EACf,SAAA;EACA,OAAA,EAAS,eAAe;AAAA;;UAIT,YAAA;EACf,SAAA;EACA,KAAA;EACA,OAAA;EACA,KAAA;EACA,eAAA;EACA,cAAA;EACA,OAAA,EAAS,aAAa,CAAC,WAAA;;EAEvB,UAAA;AAAA;AAAA,UAGe,kBAAA;;EAEf,WAAA;;EAEA,oBAAoB;AAAA;;cAIT,oBAAA;AAAA,cACA,+BAAA;AAAA,cAYA,YAAA;EAAA,iBACM,QAAA;EAAA,iBACA,WAAA;EAAA,iBACA,oBAAA;EAAA,QACT,iBAAA;cAEI,MAAA,GAAQ,kBAAA;;;;;;;;;;EAcpB,QAAA,CACE,SAAA,UACA,SAAA,UACA,OAAA,UACA,KAAA,sBACA,OAAA,EAAS,eAAA;;EA+BX,kBAAA,CAAA;;EAOA,uBAAA,CAAA;;;;;;EASA,wBAAA,CAAA;ED1ByB;;;;;ECmCzB,cAAA,CAAA,GAAkB,YAAA;EDnCgD;AAAA;;;;ECuDlE,QAAA,CAAA,GAAY,oBAAA;AAAA;AAAA,UAUG,oBAAA;EACf,WAAA;EACA,gBAAA;EACA,iBAAA;EACA,QAAA,EAAU,YAAY;AAAA;;;;;;;;;;;;;;;;;;;;;;;;iBCpIF,aAAA,CACpB,QAAA,UACA,OAAA;EAAW,SAAA;AAAA,IACV,OAAO;AAAA,UAkBO,iBAAA;;EAEf,QAAA;;;;;EAKA,SAAA;;;;;;;EAOA,SAAA;;;;;;;EAOA,OAAA,GAAU,aAAa;;;;;;EAMvB,gBAAA;;;;;;AFeF;;;;;;;;AAAoE;EEAlE,KAAA;AAAA;AAAA,UAGe,aAAA;EDjGL;ECmGV,SAAA;;;;;;;AD7FkC;AAGpC;;;;;;;ECyGE,GAAA,EAAK,MAAM;AAAA;ADnGb;;;;;;;;AAAA,iBC8GuB,UAAA,CAAW,OAAA,EAAS,iBAAA,GAAoB,aAAA,CAAc,aAAA;;;;AF/HpB;AAEzD;;;;;;cGNa,sBAAA,EAAsB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;KASvB,gBAAA,GAAmB,CAAA,CAAE,KAAK,QAAQ,sBAAA;AAAA,UAE7B,eAAA;;EAEf,QAAA;;EAEA,MAAA;;EAEA,SAAA;;EAEA,SAAA;;EAEA,cAAA;;EAEA,KAAA;;EAEA,OAAA,GAAU,aAAa;;EAEvB,gBAAA;;EAEA,KAAA;AAAA;;iBAiBqB,QAAA,CAAS,OAAA,EAAS,eAAA,GAAkB,aAAA,CAAc,aAAA;;;;cC9B5D,kBAAA;AAAA,UAEI,gBAAA;;EAEf,WAAA;;EAEA,QAAA;;EAEA,MAAA,EAAQ,MAAA;;EAER,SAAA,EAAW,YAAA,CAAa,IAAA;;EAExB,IAAA,EAAM,eAAA;;EAEN,QAAA,EAAU,YAAA;;EAEV,OAAA;;EAEA,SAAA;;;;;;;EAOA,UAAA,IAAc,QAAA,EAAU,iBAAA;;EAExB,QAAA;;EAEA,MAAA,GAAS,WAAA;;EAET,YAAA,GAAe,YAAA;;;;;;;EAOf,cAAA,GAAiB,YAAA;;;;;;;;;;;;;;;;EAgBjB,MAAA;IACE,IAAA,GAAO,IAAA,EAAM,MAAA,mBAAyB,GAAA;IACtC,IAAA,GAAO,IAAA,EAAM,MAAA,mBAAyB,GAAA;EAAA;AAAA;;;;;;UASzB,iBAAA;EACf,QAAA;EACA,aAAA;EACA,UAAA;EACA,WAAA;EH/EU;EGiFV,UAAA;EACA,gBAAA;EH5EkC;EG8ElC,cAAA;EH9EK;EGgFL,aAAA;EHhFkC;EGkFlC,qBAAA;EH/Ee;;;;;;EGsFf,SAAA;AAAA;AHpFwB;AAI1B;;;AAJ0B,UG2FT,eAAA;EHtFf;EGwFA,QAAA;EHtFA;EGwFA,aAAA;EHtFA;;;;;EG4FA,UAAA;EHxFU;EG0FV,WAAA;EHvFe;;;;AAIK;EGyFpB,UAAA;EHrF+B;EGuF/B,gBAAA;EHvF+B;EGyF/B,MAAA,EAAQ,UAAA,CAAW,YAAA;EHxFR;EG0FX,QAAA,EAAU,UAAA,CAAW,YAAA;;;AH1FqB;AAY5C;;;;;EGuFE,KAAA,WAAgB,eAAA;EHcJ;;;;;;;;;;;;EGDZ,cAAA;EACA,eAAA;AAAA;;;;;;;;iBAUoB,SAAA,MAAA,CAAgB,OAAA,EAAS,gBAAA,CAAiB,IAAA,IAAQ,OAAA,CAAQ,eAAA;;;;;;;;;;;;;UChJ/D,oBAAA;EACf,EAAA;EACA,OAAA,EAAS,oBAAA;EACT,cAAA,CAAe,IAAA,EAAM,iBAAiB;AAAA;;;;;;;cAS3B,0BAAA,EAA0B,CAAA,CAAA,SAAA;;;;;;;KAG3B,oBAAA,GAAuB,CAAA,CAAE,KAAK,QAAQ,0BAAA;;;;;;;;;;;;cAarC,aAAA,EAAe,aAAa,CAAC,oBAAA;;;;;;;KAa9B,gBAAA,SAAyB,YAAA,GAAe,OAAA,CAAQ,YAAA;;;;;;;;;;;;;;;KAgBhD,mBAAA,SAA4B,OAAO;;ALsB/C;;;;;;;;AAAoE;;;;AC9FpE;;;;;;;;AAMoC;AAGpC;;;KI0FY,gBAAA,IACV,UAAA,aACG,OAAO;EAAG,SAAA;EAAmB,OAAA,SAAgB,OAAA;AAAA;;AJ1FxB;AAI1B;;;UI6FiB,eAAA;EACf,WAAA;EJ5FA;EI8FA,MAAA,EAAQ,MAAA;EJ5FR;;;;;;EImGA,MAAA,GAAS,MAAM;AAAA;AJ9FL;AAGZ;;;;AAIsB;AAItB;;;;AAAiC;AACjC;;;;AAA4C;AAY5C;AAxBY,KIkHA,QAAA,IAAY,KAAA,WAAgB,eAAA,IAAmB,GAAA,EAAK,eAAA,KAAoB,OAAA;AAAA,UAEnE,sBAAA;EJtFK;EIwFpB,UAAA,EAAY,eAAA;EJbM;EIelB,UAAA,EAAY,iBAAA;EJKoB;EIHhC,QAAA,EAAU,gBAAA;EJjGO;EImGjB,OAAA;EJjGiB;;;;;EIuGjB,WAAA,GAAc,mBAAA;EJrFZ;;;;;EI2FF,eAAA,GAAkB,gBAAA;EJxDlB;;;;;;EI+DA,QAAA,GAAW,QAAA;EJlBqB;EIoBhC,MAAA,GAAS,MAAA;AAAA;;;;;;;;;;AJNa;iBImBR,sBAAA,CACd,MAAA,EAAQ,sBAAA,IACN,GAAA,EAAK,oBAAA,KAAyB,OAAA;AAAA,UAkTjB,4BAAA;EACf,UAAA,EAAY,eAAA;EH5cQ;EG8cpB,MAAA,GAAS,MAAM;AAAA;;;;;;;AH3cP;AAkBV;;;;;;;;;;;;AA0CO;AAGP;;;;;;;;AAiBa;AAWb;;;;;;;;iBGwZgB,4BAAA,CACd,MAAA,EAAQ,4BAAA,GACP,qBAAqB"}
|
package/dist/worker.d.mts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { a as ImportsRunJobContext, c as
|
|
2
|
-
export { EnsureIndexResolver, EsClientResolver, FilePathResolver, ImportsRunDeadListenerConfig, ImportsRunJobContext, ImportsRunJobPayload, RunImportHandlerConfig, createImportsRunDeadListener, createRunImportHandler, importsRunJob, importsRunJobPayloadSchema };
|
|
1
|
+
import { a as ImportsRunJobContext, c as PairSinkContext, d as createRunImportHandler, f as importsRunJob, i as ImportsRunDeadListenerConfig, l as RunImportHandlerConfig, n as EsClientResolver, o as ImportsRunJobPayload, p as importsRunJobPayloadSchema, r as FilePathResolver, s as PairSink, t as EnsureIndexResolver, u as createImportsRunDeadListener } from "./worker-CO5zK62E.mjs";
|
|
2
|
+
export { EnsureIndexResolver, EsClientResolver, FilePathResolver, ImportsRunDeadListenerConfig, ImportsRunJobContext, ImportsRunJobPayload, PairSink, PairSinkContext, RunImportHandlerConfig, createImportsRunDeadListener, createRunImportHandler, importsRunJob, importsRunJobPayloadSchema };
|
package/dist/worker.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import{n as e,o as t,r as n}from"./runner-
|
|
1
|
+
import{n as e,o as t,r as n}from"./runner-BOwPUm7W.mjs";import{a as r,i,r as a}from"./publish-DCXIPwTE.mjs";import{eq as o}from"drizzle-orm";import{z as s}from"zod";import{runAsCli as c}from"@murumets-ee/core";import{defineJob as l}from"@murumets-ee/queue/client";const u=s.object({importRunId:s.string().uuid()}),d=l({name:`imports:run`,description:`Stream a feed file, transform rows, bulk-write to Elasticsearch.`,schema:u,defaultRetries:0});function f(a){let{importRuns:o,transforms:s,esClient:l,esIndex:u,resolveFilePath:d,ensureIndex:f,pairSink:m,logger:h}=a;return async a=>c(async()=>{let{importRunId:c}=a.payload,g=h?.child({jobId:a.id,importRunId:c,type:`imports:run`}),_=await o.findById(c);if(!_){g?.warn(`import_run row not found — skipping`);return}_.status===`running`&&g?.warn({previousQueueJobId:_.queueJobId,previousStartedAt:_.startedAt},`imports: re-claiming a row left in running by a prior worker — likely a crash recovered by queue lock-recovery`);let v=s.get(_.transformName);if(!v)throw await p(o,c,`Unknown transform "${_.transformName}"`),Error(`No transform registered for name "${_.transformName}"`);let y=_.params&&typeof _.params==`object`&&!Array.isArray(_.params)?_.params:{},b=y.feed&&typeof y.feed==`object`&&!Array.isArray(y.feed)?y.feed:{},x=n.safeParse(b);if(!x.success){let e=x.error.issues[0]?.message??`invalid feed reader config`;throw await p(o,c,`Invalid params.feed: ${e}`),Error(`Invalid params.feed for import_run ${c}: ${e}`)}let S=x.data;await o.update(c,{status:`running`,queueJobId:a.id,startedAt:new Date}),i(c);let C;try{C=await l()}catch(e){throw await p(o,c,`Could not resolve ES client: ${e instanceof Error?e.message:String(e)}`),e}if(f)try{await f()}catch(e){throw await p(o,c,`Could not ensure ES index/alias: ${e instanceof Error?e.message:String(e)}`),e}let w,T;if(d)try{let e=await d(_.filePath);w=e.localPath,T=e.cleanup}catch(e){throw await p(o,c,`Could not resolve filePath: ${e instanceof Error?e.message:String(e)}`),e}else w=_.filePath;let E;if(S.format!==`xlsx`)try{E=await t(w,S.hasHeader===void 0?{}:{hasHeader:S.hasHeader}),g?.info({importRunId:c,totalRows:E},`imports: pre-pass row count`)}catch(e){g?.warn({err:e},`imports: pre-pass row count failed — continuing without total`)}let D;try{let t={filePath:w,...S.format!==void 0&&{format:S.format},...S.delimiter!==void 0&&{delimiter:S.delimiter},...S.hasHeader!==void 0&&{hasHeader:S.hasHeader},...S.headerRowIndex!==void 0&&{headerRowIndex:S.headerRowIndex},...S.sheet!==void 0&&{sheet:S.sheet}};D=await e({importRunId:c,runLabel:_.label,params:y,transform:v,feed:t,esClient:C,esIndex:u,onProgress:e=>a.updateProgress(E===void 0?e:{...e,totalRows:E}),...g!==void 0&&{logger:g}})}catch(e){throw await p(o,c,e instanceof Error?e.message:String(e)),T&&await T().catch(e=>{g?.warn({err:e},`filePath cleanup after failed run threw — ignoring`)}),e}if(T&&await T().catch(e=>{g?.warn({err:e},`filePath cleanup after successful run threw — ignoring`)}),await o.update(c,{status:`succeeded`,finishedAt:new Date,totals:{rowsRead:D.rowsRead,rowsSucceeded:D.rowsSucceeded,rowsFailed:D.rowsFailed,rowsSkipped:D.rowsSkipped,rowsWarned:D.rowsWarned,batchesCompleted:D.batchesCompleted,patchesApplied:D.patchesApplied,patchesOrphaned:D.patchesOrphaned},errorSummary:JSON.parse(JSON.stringify(D.errors)),warningSummary:JSON.parse(JSON.stringify(D.warnings))}),r(c,{rowsRead:D.rowsRead,rowsSucceeded:D.rowsSucceeded,rowsFailed:D.rowsFailed}),m&&D.pairs.length>0)try{await m(D.pairs,{importRunId:c,params:y,...g!==void 0&&{logger:g}}),g?.info({importRunId:c,pairs:D.pairs.length},`imports: pairSink invoked`)}catch(e){g?.warn({importRunId:c,pairs:D.pairs.length,err:e},`imports: pairSink threw — replacement ingestion skipped, catalogue import unaffected`)}g?.info({rowsRead:D.rowsRead,rowsSucceeded:D.rowsSucceeded,rowsFailed:D.rowsFailed,rowsSkipped:D.rowsSkipped,batches:D.batchesCompleted},`import_run completed`)})}async function p(e,t,n){try{await e.update(t,{status:`failed`,finishedAt:new Date,errorSummary:{fatal:n}}),a(t,n)}catch{}}function m(e){let{importRuns:t,logger:n}=e;return{onJobDead:async e=>{e.jobType===`imports:run`&&await c(async()=>{let r=n?.child({event:`imports.run.dead`,jobId:e.jobId,type:`imports:run`}),i=t.getTable(),a=(await t.findMany({where:o(i.queueJobId,e.jobId),limit:1}))[0];if(!a){r?.warn(`queue.job.dead with no matching import_run — skipping`);return}if(a.status!==`running`&&a.status!==`pending`){r?.info({importRunId:a.id,status:a.status},`queue.job.dead arrived for a row already in a terminal state — leaving as-is`);return}a.status===`pending`&&r?.warn({importRunId:a.id,attempts:e.attempts},`queue.job.dead arrived for an import_run still in pending — worker crashed BEFORE claim/lookup`),await p(t,a.id,e.error),r?.info({importRunId:a.id,attempts:e.attempts,priorStatus:a.status},`queue.job.dead → import_run marked failed`)})}}}export{m as createImportsRunDeadListener,f as createRunImportHandler,d as importsRunJob,u as importsRunJobPayloadSchema};
|
|
2
2
|
//# sourceMappingURL=worker.mjs.map
|
package/dist/worker.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.mjs","names":[],"sources":["../src/worker.ts"],"sourcesContent":["/**\n * Queue handler for `imports:run`. The route handler that creates the\n * `import_run` row enqueues this job carrying `{ importRunId }`; the\n * worker module here resolves the run, the transform, and the ES\n * client, then runs `runImport` and writes results back.\n *\n * Per PLAN-ECOMMERCE.md PR 7 (PoC scope): one job type, no resumability.\n * A failed handler attempt re-runs from row 1 on retry — fine for the\n * tens-of-MB feed sizes the PoC exercises.\n *\n * Per CLAUDE.md \"leaf packages\" boundary rule: this handler reads the\n * import_run row via an injected `AdminClient`, NOT a direct `db.select`.\n * Construction happens at plugin init in `plugin.ts`.\n */\n\nimport { type Logger, runAsCli } from '@murumets-ee/core'\nimport {\n defineJob,\n type JobDefinition,\n type QueueJobDeadEvent,\n type QueueTerminalNotifier,\n} from '@murumets-ee/queue/client'\nimport type { EsClientLike } from '@murumets-ee/search-elasticsearch'\nimport { eq } from 'drizzle-orm'\nimport type { PgColumn } from 'drizzle-orm/pg-core'\nimport { z } from 'zod'\nimport type { ImportRunClient, ImportRunStatus } from './entities/import-run.js'\nimport { feedReaderConfigSchema, type ReadFeedOptions } from './read-feed.js'\nimport {\n publishImportRunFailed,\n publishImportRunStarted,\n publishImportRunSucceeded,\n} from './realtime/publish.js'\nimport { type ImportRunProgress, runImport } from './runner.js'\nimport { countDataRows } from './streaming.js'\nimport type { TransformRegistry } from './transform.js'\n\n/**\n * Structural subset of `JobContext` the handler reads. Declared\n * locally so we don't depend on a `JobHandler` import — the queue\n * package doesn't re-export it from any subpath today (matches the\n * `SendEmailJobContext` pattern in `@murumets-ee/notifications`).\n *\n * Function-param contravariance lets a handler returned with this\n * narrower shape be assigned to the wider `JobHandler<TPayload>` that\n * `registerJob` expects.\n */\nexport interface ImportsRunJobContext {\n id: string\n payload: ImportsRunJobPayload\n updateProgress(data: ImportRunProgress): void\n}\n\n/**\n * Payload schema. `importRunId` is the `import_run.id` UUID — the worker\n * looks up everything else (filePath, transformName, params, …) from\n * that row. Keeping the payload tiny means the queue's progress JSON\n * column never bloats with feed metadata duplicated across `toolkit_jobs`.\n */\nexport const importsRunJobPayloadSchema = z.object({\n importRunId: z.string().uuid(),\n})\nexport type ImportsRunJobPayload = z.infer<typeof importsRunJobPayloadSchema>\n\n/**\n * Job definition. Consumers register their handler against this with\n * `registerJob(importsRunJob, createRunImportHandler({...}))`.\n *\n * `defaultRetries: 0` — re-running a multi-batch import against the\n * same `import_run.id` while the previous handler may still be writing\n * is a footgun (duplicate batches, double-counted progress). The\n * operator handles retries explicitly via PR 8a's \"retry\" button by\n * creating a new `import_run` row. When the resumable design lands\n * (post-PoC), retries become safe to enable.\n */\nexport const importsRunJob: JobDefinition<ImportsRunJobPayload> = defineJob({\n name: 'imports:run',\n description: 'Stream a feed file, transform rows, bulk-write to Elasticsearch.',\n schema: importsRunJobPayloadSchema,\n defaultRetries: 0,\n})\n\n/**\n * Resolves the ES client at handler-invocation time. A function (rather\n * than the bare `EsClientLike`) so the consumer can lazy-construct the\n * client — typical Next.js setups create the ES connection in a\n * route-handler initialiser, not at plugin-init time.\n */\nexport type EsClientResolver = () => EsClientLike | Promise<EsClientLike>\n\n/**\n * Ensure the destination index/alias exists before the first bulk-write.\n * Invoked once per job, BEFORE {@link runImport}. Idempotent: the canonical\n * implementation (`ensureAliasedIndex` in `@murumets-ee/search-elasticsearch`)\n * no-ops when the alias already resolves.\n *\n * **Why it lives here.** This is the home of the foot-gun it guards: a worker\n * that bulk-writes to alias `<x>` before the alias exists makes ES\n * auto-create a physical `<x>` index with default `text` mappings, breaking\n * aggregations forever. The bootstrap used to run at app boot, which turned\n * \"ES is down\" into a full-admin 500 (issue #610). Moving it onto the import\n * path keeps the guarantee while scoping ES availability to imports/search —\n * the rest of the admin boots and runs with ES down.\n */\nexport type EnsureIndexResolver = () => Promise<unknown>\n\n/**\n * Resolves the value of `import_run.filePath` (whatever the upload\n * route persisted there — typically a storage adapter key) to a\n * readable LOCAL filesystem path that {@link runImport} can hand to\n * `node:fs.createReadStream`.\n *\n * **Why this exists:** the streaming reader (`streamFeed`) reads from\n * local disk via `createReadStream`. The upload route may persist a\n * remote-storage object key (R2, S3, …) on `import_run.filePath`\n * because that's what `@murumets-ee/storage` returns. Without a\n * resolver, `createReadStream('uploads/2026/05/<uuid>/feed.txt')`\n * crashes with `ENOENT`. The resolver is the documented integration\n * point — typical wiring downloads the storage object to a tmpfile\n * and returns its path.\n *\n * **`cleanup`** runs after the run finishes (success OR failure). The\n * worker awaits it best-effort — a failed cleanup logs but does not\n * crash the run.\n *\n * **No-resolver fallback:** when `resolveFilePath` is unset, the\n * worker treats `import_run.filePath` as already a local FS path\n * (back-compat with the original PoC design where uploads landed on\n * local disk). This stays valid for fixture-driven tests + on-disk\n * deployments.\n */\nexport type FilePathResolver = (\n storageKey: string,\n) => Promise<{ localPath: string; cleanup?: () => Promise<void> }>\n\nexport interface RunImportHandlerConfig {\n /** AdminClient over the `import_run` entity. */\n importRuns: ImportRunClient\n /** Transform registry to dispatch against. Defaults to the process-global singleton. */\n transforms: TransformRegistry\n /** Resolver for the ES client. */\n esClient: EsClientResolver\n /** ES alias / index to bulk-write into. Per D6 callers always pass an alias. */\n esIndex: string\n /**\n * Optional bootstrap for the destination index/alias, run once per job\n * before any bulk-write. See {@link EnsureIndexResolver}. Omit for\n * deployments that bootstrap the index out-of-band (e.g. a reindex job).\n */\n ensureIndex?: EnsureIndexResolver\n /**\n * Optional resolver for `import_run.filePath`. See {@link FilePathResolver}.\n * Required when uploads land in remote storage (R2/S3); optional for\n * on-disk PoC setups.\n */\n resolveFilePath?: FilePathResolver\n /** Optional structured logger. Defaults to silent. */\n logger?: Logger\n}\n\n/**\n * Build the `JobHandler` for {@link importsRunJob}. The returned\n * handler is what gets passed to `registerJob`.\n *\n * The body is wrapped in `runAsCli` so AdminClient calls inside have a\n * synthetic `cli` admin context — `auditable()` records `updatedBy:\n * 'cli'` rather than NULL, and the firewall checker passes. This is\n * the documented worker entry-point pattern (see `runAsCli` JSDoc in\n * `@murumets-ee/core`).\n */\nexport function createRunImportHandler(\n config: RunImportHandlerConfig,\n): (job: ImportsRunJobContext) => Promise<void> {\n const { importRuns, transforms, esClient, esIndex, resolveFilePath, ensureIndex, logger } = config\n\n return async (job: ImportsRunJobContext): Promise<void> =>\n runAsCli(async () => {\n const { importRunId } = job.payload\n const log = logger?.child({ jobId: job.id, importRunId, type: 'imports:run' })\n\n const run = await importRuns.findById(importRunId)\n if (!run) {\n // Row was deleted between enqueue and dispatch — nothing to do.\n // Don't throw: a thrown handler retries with no payoff because\n // the row stays gone. Log + return so the queue marks the job\n // completed.\n log?.warn('import_run row not found — skipping')\n return\n }\n\n // Re-claim detection. The queue's `recoverStuckJobs` sweep flips a\n // crashed worker's `toolkit_jobs` row from `processing` back to\n // `pending` — but the queue layer has no knowledge of `import_run`,\n // so this row stays at `status='running'` from the prior attempt.\n // When the queue re-dispatches the job, we land here with a row\n // that the operator-facing Active panel has been showing as live\n // for however long lock-recovery took (default 30 min). The\n // existing `update` below overwrites `queueJobId` + `startedAt` so\n // there is no STATE corruption — but emit a structured warning so\n // ops can correlate this re-run with the worker crash that\n // produced it. The sibling `sweepStaleImportRuns` job below catches\n // the case where re-dispatch never happens (queue marks the job\n // dead, or the queue worker is offline long-term).\n if (run.status === 'running') {\n log?.warn(\n {\n previousQueueJobId: run.queueJobId,\n previousStartedAt: run.startedAt,\n },\n 'imports: re-claiming a row left in running by a prior worker — likely a crash recovered by queue lock-recovery',\n )\n }\n\n // `transformName`, `filePath`, `label` are declared `field.text({\n // required: true })` so the inferred DTO already types them as\n // `string` — no cast needed. `params` is `field.json()` which\n // infers as `JsonValue | undefined`; narrow to a record so the\n // transform receives an object.\n const transform = transforms.get(run.transformName)\n if (!transform) {\n await markFailed(importRuns, importRunId, `Unknown transform \"${run.transformName}\"`)\n throw new Error(`No transform registered for name \"${run.transformName}\"`)\n }\n const params: Record<string, unknown> =\n run.params && typeof run.params === 'object' && !Array.isArray(run.params)\n ? (run.params as Record<string, unknown>)\n : {}\n\n // Reader config (format/delimiter/header/sheet) travels in\n // `params.feed` for mapping-driven runs (commerce-demo PR3). Carmaker\n // runs omit it → empty config → tab-delimited default, so the existing\n // path is byte-for-byte unchanged. Re-validate here as defense-in-depth\n // even though the enqueuing route already validated the saved mapping.\n const feedRaw =\n params.feed && typeof params.feed === 'object' && !Array.isArray(params.feed)\n ? params.feed\n : {}\n const feedParsed = feedReaderConfigSchema.safeParse(feedRaw)\n if (!feedParsed.success) {\n const reason = feedParsed.error.issues[0]?.message ?? 'invalid feed reader config'\n await markFailed(importRuns, importRunId, `Invalid params.feed: ${reason}`)\n throw new Error(`Invalid params.feed for import_run ${importRunId}: ${reason}`)\n }\n const feedConfig = feedParsed.data\n\n await importRuns.update(importRunId, {\n status: 'running' satisfies ImportRunStatus,\n queueJobId: job.id,\n startedAt: new Date(),\n })\n // Realtime: row transitioned pending → running. Subscribers\n // refetch to surface the change in the imports admin page.\n // Best-effort — a publish failure is logged inside `publishSafe`\n // and does NOT roll back the row update.\n publishImportRunStarted(importRunId)\n\n let resolvedClient: EsClientLike\n try {\n resolvedClient = await esClient()\n } catch (err) {\n const reason = err instanceof Error ? err.message : String(err)\n await markFailed(importRuns, importRunId, `Could not resolve ES client: ${reason}`)\n throw err\n }\n\n // Ensure the destination alias exists BEFORE any bulk-write. Moved here\n // from app boot (issue #610): the foot-gun is the worker auto-creating a\n // mis-mapped physical index, so the guard belongs on the import path —\n // not gating the whole app on ES. Idempotent + scoped: an ES outage (or\n // the `ensureAliasedIndex` physical-collision guard firing) fails THIS\n // job loudly while the rest of the admin keeps working.\n if (ensureIndex) {\n try {\n await ensureIndex()\n } catch (err) {\n const reason = err instanceof Error ? err.message : String(err)\n await markFailed(importRuns, importRunId, `Could not ensure ES index/alias: ${reason}`)\n throw err\n }\n }\n\n // Materialise the file locally if the consumer wired a resolver.\n // The `cleanup` callback (when present) runs in the `finally`\n // below regardless of run outcome so a tmpfile created here\n // doesn't outlive the handler.\n let localPath: string\n let fileCleanup: (() => Promise<void>) | undefined\n if (resolveFilePath) {\n try {\n const resolved = await resolveFilePath(run.filePath)\n localPath = resolved.localPath\n fileCleanup = resolved.cleanup\n } catch (err) {\n const reason = err instanceof Error ? err.message : String(err)\n await markFailed(importRuns, importRunId, `Could not resolve filePath: ${reason}`)\n throw err\n }\n } else {\n // Back-compat: no resolver wired → treat `filePath` as already\n // a local FS path. Fine for fixture-driven tests + the original\n // on-disk PoC design.\n localPath = run.filePath\n }\n\n // Pre-pass line count: a 16 MB tab-delimited feed counts in\n // ~50ms; a 500 MB feed in <2s. Cheap relative to the full\n // import. Gives the operator a denominator (`processed / total`)\n // so the imports-page panel can render an actual percent\n // instead of just \"rows so far\".\n // Pre-pass row count is a `\\n`-tally over a delimited file — meaningless\n // on an xlsx binary, so skip it there (the UI falls back to bare counts).\n // `hasHeader` from the feed config keeps the header-exclusion correct for\n // headerless mappings.\n let totalRows: number | undefined\n if (feedConfig.format !== 'xlsx') {\n try {\n totalRows = await countDataRows(\n localPath,\n feedConfig.hasHeader !== undefined ? { hasHeader: feedConfig.hasHeader } : {},\n )\n log?.info({ importRunId, totalRows }, 'imports: pre-pass row count')\n } catch (err) {\n // Don't fail the run on a count miss — the actual import is\n // unaffected; the operator just loses the percent denominator.\n log?.warn({ err }, 'imports: pre-pass row count failed — continuing without total')\n }\n }\n\n let result: Awaited<ReturnType<typeof runImport>>\n try {\n // Build feed options with the conditional-spread idiom (NOT\n // `...feedConfig`) so absent optionals stay ABSENT — spreading a\n // `T | undefined` value would set the key to `undefined`, which\n // `exactOptionalPropertyTypes` rejects against `ReadFeedOptions`.\n const feedOpts: ReadFeedOptions = {\n filePath: localPath,\n ...(feedConfig.format !== undefined && { format: feedConfig.format }),\n ...(feedConfig.delimiter !== undefined && { delimiter: feedConfig.delimiter }),\n ...(feedConfig.hasHeader !== undefined && { hasHeader: feedConfig.hasHeader }),\n ...(feedConfig.headerRowIndex !== undefined && {\n headerRowIndex: feedConfig.headerRowIndex,\n }),\n ...(feedConfig.sheet !== undefined && { sheet: feedConfig.sheet }),\n }\n result = await runImport({\n importRunId,\n runLabel: run.label,\n params,\n transform,\n feed: feedOpts,\n esClient: resolvedClient,\n esIndex,\n onProgress: (progress) =>\n job.updateProgress(totalRows !== undefined ? { ...progress, totalRows } : progress),\n ...(log !== undefined && { logger: log }),\n })\n } catch (err) {\n const reason = err instanceof Error ? err.message : String(err)\n await markFailed(importRuns, importRunId, reason)\n if (fileCleanup) {\n await fileCleanup().catch((cleanupErr: unknown) => {\n log?.warn({ err: cleanupErr }, 'filePath cleanup after failed run threw — ignoring')\n })\n }\n throw err\n }\n\n // Cleanup runs before the success-update so a cleanup throw\n // surfaces as a logged warning rather than a crashed handler at\n // the worst possible moment (just after the row was about to be\n // marked succeeded).\n if (fileCleanup) {\n await fileCleanup().catch((cleanupErr: unknown) => {\n log?.warn({ err: cleanupErr }, 'filePath cleanup after successful run threw — ignoring')\n })\n }\n\n await importRuns.update(importRunId, {\n status: 'succeeded' satisfies ImportRunStatus,\n finishedAt: new Date(),\n totals: {\n rowsRead: result.rowsRead,\n rowsSucceeded: result.rowsSucceeded,\n rowsFailed: result.rowsFailed,\n rowsSkipped: result.rowsSkipped,\n rowsWarned: result.rowsWarned,\n batchesCompleted: result.batchesCompleted,\n },\n // The typed `ErrorTrackerSnapshot` is fully JSON-serialisable but\n // doesn't structurally satisfy entity's recursive `JsonValue`\n // (objects without an index signature aren't `Record<string,\n // JsonValue>`). Round-trip through `JSON.parse(JSON.stringify(...))`\n // to land as the recursive JSON shape — the runtime cost is one\n // serialise per import_run completion, the type-cast cost is zero.\n errorSummary: JSON.parse(JSON.stringify(result.errors)),\n // Non-fatal warnings (commerce-demo PR4) — kept separate from\n // `errorSummary` so a `no_purchase_price` advisory never reads as a\n // failed row. Same serialise-roundtrip rationale as `errorSummary`.\n warningSummary: JSON.parse(JSON.stringify(result.warnings)),\n })\n // Realtime: row transitioned to terminal-success. Subscribers\n // refetch to move the row from Active → Recent.\n publishImportRunSucceeded(importRunId, {\n rowsRead: result.rowsRead,\n rowsSucceeded: result.rowsSucceeded,\n rowsFailed: result.rowsFailed,\n })\n\n log?.info(\n {\n rowsRead: result.rowsRead,\n rowsSucceeded: result.rowsSucceeded,\n rowsFailed: result.rowsFailed,\n rowsSkipped: result.rowsSkipped,\n batches: result.batchesCompleted,\n },\n 'import_run completed',\n )\n })\n}\n\n/** Write a `failed` row with a fatal error reason. Best-effort — logs but does not throw on DB failure. */\nasync function markFailed(\n importRuns: ImportRunClient,\n id: string,\n reason: string,\n): Promise<void> {\n try {\n await importRuns.update(id, {\n status: 'failed' satisfies ImportRunStatus,\n finishedAt: new Date(),\n errorSummary: { fatal: reason },\n })\n // Realtime: row transitioned to terminal-failure. Subscribers\n // refetch to surface the failure in the Recent panel. Publish\n // is best-effort and only fires when the persist succeeds —\n // skipping it on persist failure keeps subscribers consistent\n // with the DB (no event for a row that didn't transition).\n publishImportRunFailed(id, reason)\n } catch {\n // The handler already threw / will throw — the queue will surface\n // the original error. A second failure here just logs noise.\n }\n}\n\nexport interface ImportsRunDeadListenerConfig {\n importRuns: ImportRunClient\n /** Optional structured logger. */\n logger?: Logger\n}\n\n/**\n * Build a {@link QueueTerminalNotifier} that mirrors `imports:run`\n * dead-job events into `import_run.status='failed'`.\n *\n * **Why this exists.** The `imports:run` handler's normal failure path\n * already calls `markFailed` before re-throwing, so a synchronous\n * handler error reaches the operator-visible `failed` status without\n * help from anyone. The case THIS listener catches is the one\n * `markFailed` never gets to: the worker process is killed mid-handler\n * (OOM, container restart, kill -9), the queue's lock-recovery sweep\n * eventually reclaims the row, and the queue ultimately marks the job\n * `dead` (defaultRetries: 0 + repeated process death = dead-letter\n * with `import_run.status` still `running`). Without this listener\n * the row sits in the operator's Active panel forever.\n *\n * **Generic primitive.** This is a thin subscriber over the queue's\n * multi-subscriber `QueueTerminalNotifier` registry — the same shape\n * graphile-worker (`job:failed`) and BullMQ (`failed`) expose. Other\n * domain plugins that mirror queue state (ticketing-reply, future\n * exports/reports/...) can register their own listeners exactly the\n * same way without coordinating via a per-domain sweep job.\n *\n * **Filtering.** The listener is invoked for EVERY dead event in the\n * process. We early-return on any `jobType` other than `imports:run`\n * so the per-event cost on unrelated jobs is a single property read.\n *\n * **Idempotency.** A dead event delivered after the row has already\n * reached its own terminal state (succeeded / failed / cancelled) is\n * ignored — the only transition we make is `running → failed`. This\n * keeps a late replay or out-of-order delivery from clobbering a\n * row whose worker reached `markFailed` faster than the queue's\n * dead-letter dispatch.\n *\n * The handler is wrapped in `runAsCli` so AdminClient calls inside\n * have a synthetic CLI admin context — same pattern as\n * {@link createRunImportHandler}.\n */\nexport function createImportsRunDeadListener(\n config: ImportsRunDeadListenerConfig,\n): QueueTerminalNotifier {\n const { importRuns, logger } = config\n\n return {\n onJobDead: async (event: QueueJobDeadEvent): Promise<void> => {\n if (event.jobType !== 'imports:run') return\n\n await runAsCli(async () => {\n const log = logger?.child({\n event: 'imports.run.dead',\n jobId: event.jobId,\n type: 'imports:run',\n })\n\n // Look up the mirror row by `queueJobId`. Cap to 1 — the\n // upload route writes one `import_run` per enqueued job.\n //\n // `AdminClient.getTable()` returns `PgTableWithColumns<any>`\n // because entity tables have dynamic column shapes (see the\n // `getTable()` JSDoc in `packages/entity/src/admin/client.ts`\n // and the documented-`any`-cases list in CLAUDE.md). Cast\n // the table to a structural shape covering only the column\n // we touch, so both `eq()` and type-coverage see a typed\n // operand instead of an `any`-derived property access.\n const table = importRuns.getTable() as unknown as { queueJobId: PgColumn }\n const matches = await importRuns.findMany({\n where: eq(table.queueJobId, event.jobId),\n limit: 1,\n })\n const row = matches[0]\n if (!row) {\n // No mirror row for this dead job. Either it was deleted\n // by the operator, or the dead event is for a job\n // unrelated to imports despite the type filter\n // (shouldn't happen, but is cheap to log).\n log?.warn('queue.job.dead with no matching import_run — skipping')\n return\n }\n\n // Only transition rows that haven't already reached their\n // own terminal state. A late dead-event must not overwrite\n // a `succeeded` row that the worker reached via the normal\n // path before the queue's dead-letter dispatch fired.\n if (row.status !== 'running' && row.status !== 'pending') {\n log?.info(\n { importRunId: row.id, status: row.status },\n 'queue.job.dead arrived for a row already in a terminal state — leaving as-is',\n )\n return\n }\n\n // Dead-from-`pending` is the rarer + more interesting case:\n // the worker process died (or never claimed) BEFORE it could\n // flip the row to `running`. Log it at warn so ops can spot\n // a queue+worker-startup pathology that the standard `running\n // → failed` transition wouldn't surface. Dead-from-`running`\n // is the documented \"crash mid-import\" case — info is enough.\n if (row.status === 'pending') {\n log?.warn(\n { importRunId: row.id, attempts: event.attempts },\n 'queue.job.dead arrived for an import_run still in pending — worker crashed BEFORE claim/lookup',\n )\n }\n\n await markFailed(importRuns, row.id, event.error)\n log?.info(\n { importRunId: row.id, attempts: event.attempts, priorStatus: row.status },\n 'queue.job.dead → import_run marked failed',\n )\n })\n },\n }\n}\n"],"mappings":"wQA2DA,MAAa,EAA6B,EAAE,OAAO,CACjD,YAAa,EAAE,OAAO,CAAC,CAAC,KAAK,CAC/B,CAAC,EAcY,EAAqD,EAAU,CAC1E,KAAM,cACN,YAAa,mEACb,OAAQ,EACR,eAAgB,CAClB,CAAC,EA0FD,SAAgB,EACd,EAC8C,CAC9C,GAAM,CAAE,aAAY,aAAY,WAAU,UAAS,kBAAiB,cAAa,UAAW,EAE5F,OAAO,KAAO,IACZ,EAAS,SAAY,CACnB,GAAM,CAAE,eAAgB,EAAI,QACtB,EAAM,GAAQ,MAAM,CAAE,MAAO,EAAI,GAAI,cAAa,KAAM,aAAc,CAAC,EAEvE,EAAM,MAAM,EAAW,SAAS,CAAW,EACjD,GAAI,CAAC,EAAK,CAKR,GAAK,KAAK,qCAAqC,EAC/C,MACF,CAeI,EAAI,SAAW,WACjB,GAAK,KACH,CACE,mBAAoB,EAAI,WACxB,kBAAmB,EAAI,SACzB,EACA,gHACF,EAQF,IAAM,EAAY,EAAW,IAAI,EAAI,aAAa,EAClD,GAAI,CAAC,EAEH,MADA,MAAM,EAAW,EAAY,EAAa,sBAAsB,EAAI,cAAc,EAAE,EAC1E,MAAM,qCAAqC,EAAI,cAAc,EAAE,EAE3E,IAAM,EACJ,EAAI,QAAU,OAAO,EAAI,QAAW,UAAY,CAAC,MAAM,QAAQ,EAAI,MAAM,EACpE,EAAI,OACL,CAAC,EAOD,EACJ,EAAO,MAAQ,OAAO,EAAO,MAAS,UAAY,CAAC,MAAM,QAAQ,EAAO,IAAI,EACxE,EAAO,KACP,CAAC,EACD,EAAa,EAAuB,UAAU,CAAO,EAC3D,GAAI,CAAC,EAAW,QAAS,CACvB,IAAM,EAAS,EAAW,MAAM,OAAO,EAAE,EAAE,SAAW,6BAEtD,MADA,MAAM,EAAW,EAAY,EAAa,wBAAwB,GAAQ,EAChE,MAAM,sCAAsC,EAAY,IAAI,GAAQ,CAChF,CACA,IAAM,EAAa,EAAW,KAE9B,MAAM,EAAW,OAAO,EAAa,CACnC,OAAQ,UACR,WAAY,EAAI,GAChB,UAAW,IAAI,IACjB,CAAC,EAKD,EAAwB,CAAW,EAEnC,IAAI,EACJ,GAAI,CACF,EAAiB,MAAM,EAAS,CAClC,OAAS,EAAK,CAGZ,MADA,MAAM,EAAW,EAAY,EAAa,gCAD3B,aAAe,MAAQ,EAAI,QAAU,OAAO,CAAG,GACoB,EAC5E,CACR,CAQA,GAAI,EACF,GAAI,CACF,MAAM,EAAY,CACpB,OAAS,EAAK,CAGZ,MADA,MAAM,EAAW,EAAY,EAAa,oCAD3B,aAAe,MAAQ,EAAI,QAAU,OAAO,CAAG,GACwB,EAChF,CACR,CAOF,IAAI,EACA,EACJ,GAAI,EACF,GAAI,CACF,IAAM,EAAW,MAAM,EAAgB,EAAI,QAAQ,EACnD,EAAY,EAAS,UACrB,EAAc,EAAS,OACzB,OAAS,EAAK,CAGZ,MADA,MAAM,EAAW,EAAY,EAAa,+BAD3B,aAAe,MAAQ,EAAI,QAAU,OAAO,CAAG,GACmB,EAC3E,CACR,KAKA,GAAY,EAAI,SAYlB,IAAI,EACJ,GAAI,EAAW,SAAW,OACxB,GAAI,CACF,EAAY,MAAM,EAChB,EACA,EAAW,YAAc,IAAA,GAAkD,CAAC,EAAvC,CAAE,UAAW,EAAW,SAAU,CACzE,EACA,GAAK,KAAK,CAAE,cAAa,WAAU,EAAG,6BAA6B,CACrE,OAAS,EAAK,CAGZ,GAAK,KAAK,CAAE,KAAI,EAAG,+DAA+D,CACpF,CAGF,IAAI,EACJ,GAAI,CAKF,IAAM,EAA4B,CAChC,SAAU,EACV,GAAI,EAAW,SAAW,IAAA,IAAa,CAAE,OAAQ,EAAW,MAAO,EACnE,GAAI,EAAW,YAAc,IAAA,IAAa,CAAE,UAAW,EAAW,SAAU,EAC5E,GAAI,EAAW,YAAc,IAAA,IAAa,CAAE,UAAW,EAAW,SAAU,EAC5E,GAAI,EAAW,iBAAmB,IAAA,IAAa,CAC7C,eAAgB,EAAW,cAC7B,EACA,GAAI,EAAW,QAAU,IAAA,IAAa,CAAE,MAAO,EAAW,KAAM,CAClE,EACA,EAAS,MAAM,EAAU,CACvB,cACA,SAAU,EAAI,MACd,SACA,YACA,KAAM,EACN,SAAU,EACV,UACA,WAAa,GACX,EAAI,eAAe,IAAc,IAAA,GAAyC,EAA7B,CAAE,GAAG,EAAU,WAAU,CAAY,EACpF,GAAI,IAAQ,IAAA,IAAa,CAAE,OAAQ,CAAI,CACzC,CAAC,CACH,OAAS,EAAK,CAQZ,MANA,MAAM,EAAW,EAAY,EADd,aAAe,MAAQ,EAAI,QAAU,OAAO,CAAG,CACd,EAC5C,GACF,MAAM,EAAY,CAAC,CAAC,MAAO,GAAwB,CACjD,GAAK,KAAK,CAAE,IAAK,CAAW,EAAG,oDAAoD,CACrF,CAAC,EAEG,CACR,CAMI,GACF,MAAM,EAAY,CAAC,CAAC,MAAO,GAAwB,CACjD,GAAK,KAAK,CAAE,IAAK,CAAW,EAAG,wDAAwD,CACzF,CAAC,EAGH,MAAM,EAAW,OAAO,EAAa,CACnC,OAAQ,YACR,WAAY,IAAI,KAChB,OAAQ,CACN,SAAU,EAAO,SACjB,cAAe,EAAO,cACtB,WAAY,EAAO,WACnB,YAAa,EAAO,YACpB,WAAY,EAAO,WACnB,iBAAkB,EAAO,gBAC3B,EAOA,aAAc,KAAK,MAAM,KAAK,UAAU,EAAO,MAAM,CAAC,EAItD,eAAgB,KAAK,MAAM,KAAK,UAAU,EAAO,QAAQ,CAAC,CAC5D,CAAC,EAGD,EAA0B,EAAa,CACrC,SAAU,EAAO,SACjB,cAAe,EAAO,cACtB,WAAY,EAAO,UACrB,CAAC,EAED,GAAK,KACH,CACE,SAAU,EAAO,SACjB,cAAe,EAAO,cACtB,WAAY,EAAO,WACnB,YAAa,EAAO,YACpB,QAAS,EAAO,gBAClB,EACA,sBACF,CACF,CAAC,CACL,CAGA,eAAe,EACb,EACA,EACA,EACe,CACf,GAAI,CACF,MAAM,EAAW,OAAO,EAAI,CAC1B,OAAQ,SACR,WAAY,IAAI,KAChB,aAAc,CAAE,MAAO,CAAO,CAChC,CAAC,EAMD,EAAuB,EAAI,CAAM,CACnC,MAAQ,CAGR,CACF,CA6CA,SAAgB,EACd,EACuB,CACvB,GAAM,CAAE,aAAY,UAAW,EAE/B,MAAO,CACL,UAAW,KAAO,IAA4C,CACxD,EAAM,UAAY,eAEtB,MAAM,EAAS,SAAY,CACzB,IAAM,EAAM,GAAQ,MAAM,CACxB,MAAO,mBACP,MAAO,EAAM,MACb,KAAM,aACR,CAAC,EAYK,EAAQ,EAAW,SAAS,EAK5B,GAAM,MAJU,EAAW,SAAS,CACxC,MAAO,EAAG,EAAM,WAAY,EAAM,KAAK,EACvC,MAAO,CACT,CAAC,EAAA,CACmB,GACpB,GAAI,CAAC,EAAK,CAKR,GAAK,KAAK,uDAAuD,EACjE,MACF,CAMA,GAAI,EAAI,SAAW,WAAa,EAAI,SAAW,UAAW,CACxD,GAAK,KACH,CAAE,YAAa,EAAI,GAAI,OAAQ,EAAI,MAAO,EAC1C,8EACF,EACA,MACF,CAQI,EAAI,SAAW,WACjB,GAAK,KACH,CAAE,YAAa,EAAI,GAAI,SAAU,EAAM,QAAS,EAChD,gGACF,EAGF,MAAM,EAAW,EAAY,EAAI,GAAI,EAAM,KAAK,EAChD,GAAK,KACH,CAAE,YAAa,EAAI,GAAI,SAAU,EAAM,SAAU,YAAa,EAAI,MAAO,EACzE,2CACF,CACF,CAAC,CACH,CACF,CACF"}
|
|
1
|
+
{"version":3,"file":"worker.mjs","names":[],"sources":["../src/worker.ts"],"sourcesContent":["/**\n * Queue handler for `imports:run`. The route handler that creates the\n * `import_run` row enqueues this job carrying `{ importRunId }`; the\n * worker module here resolves the run, the transform, and the ES\n * client, then runs `runImport` and writes results back.\n *\n * Per PLAN-ECOMMERCE.md PR 7 (PoC scope): one job type, no resumability.\n * A failed handler attempt re-runs from row 1 on retry — fine for the\n * tens-of-MB feed sizes the PoC exercises.\n *\n * Per CLAUDE.md \"leaf packages\" boundary rule: this handler reads the\n * import_run row via an injected `AdminClient`, NOT a direct `db.select`.\n * Construction happens at plugin init in `plugin.ts`.\n */\n\nimport { type Logger, runAsCli } from '@murumets-ee/core'\nimport {\n defineJob,\n type JobDefinition,\n type QueueJobDeadEvent,\n type QueueTerminalNotifier,\n} from '@murumets-ee/queue/client'\nimport type { EsClientLike } from '@murumets-ee/search-elasticsearch'\nimport { eq } from 'drizzle-orm'\nimport type { PgColumn } from 'drizzle-orm/pg-core'\nimport { z } from 'zod'\nimport type { ImportRunClient, ImportRunStatus } from './entities/import-run.js'\nimport { feedReaderConfigSchema, type ReadFeedOptions } from './read-feed.js'\nimport {\n publishImportRunFailed,\n publishImportRunStarted,\n publishImportRunSucceeded,\n} from './realtime/publish.js'\nimport { type ImportRunProgress, runImport } from './runner.js'\nimport { countDataRows } from './streaming.js'\nimport type { DirectionalPair, TransformRegistry } from './transform.js'\n\n/**\n * Structural subset of `JobContext` the handler reads. Declared\n * locally so we don't depend on a `JobHandler` import — the queue\n * package doesn't re-export it from any subpath today (matches the\n * `SendEmailJobContext` pattern in `@murumets-ee/notifications`).\n *\n * Function-param contravariance lets a handler returned with this\n * narrower shape be assigned to the wider `JobHandler<TPayload>` that\n * `registerJob` expects.\n */\nexport interface ImportsRunJobContext {\n id: string\n payload: ImportsRunJobPayload\n updateProgress(data: ImportRunProgress): void\n}\n\n/**\n * Payload schema. `importRunId` is the `import_run.id` UUID — the worker\n * looks up everything else (filePath, transformName, params, …) from\n * that row. Keeping the payload tiny means the queue's progress JSON\n * column never bloats with feed metadata duplicated across `toolkit_jobs`.\n */\nexport const importsRunJobPayloadSchema = z.object({\n importRunId: z.string().uuid(),\n})\nexport type ImportsRunJobPayload = z.infer<typeof importsRunJobPayloadSchema>\n\n/**\n * Job definition. Consumers register their handler against this with\n * `registerJob(importsRunJob, createRunImportHandler({...}))`.\n *\n * `defaultRetries: 0` — re-running a multi-batch import against the\n * same `import_run.id` while the previous handler may still be writing\n * is a footgun (duplicate batches, double-counted progress). The\n * operator handles retries explicitly via PR 8a's \"retry\" button by\n * creating a new `import_run` row. When the resumable design lands\n * (post-PoC), retries become safe to enable.\n */\nexport const importsRunJob: JobDefinition<ImportsRunJobPayload> = defineJob({\n name: 'imports:run',\n description: 'Stream a feed file, transform rows, bulk-write to Elasticsearch.',\n schema: importsRunJobPayloadSchema,\n defaultRetries: 0,\n})\n\n/**\n * Resolves the ES client at handler-invocation time. A function (rather\n * than the bare `EsClientLike`) so the consumer can lazy-construct the\n * client — typical Next.js setups create the ES connection in a\n * route-handler initialiser, not at plugin-init time.\n */\nexport type EsClientResolver = () => EsClientLike | Promise<EsClientLike>\n\n/**\n * Ensure the destination index/alias exists before the first bulk-write.\n * Invoked once per job, BEFORE {@link runImport}. Idempotent: the canonical\n * implementation (`ensureAliasedIndex` in `@murumets-ee/search-elasticsearch`)\n * no-ops when the alias already resolves.\n *\n * **Why it lives here.** This is the home of the foot-gun it guards: a worker\n * that bulk-writes to alias `<x>` before the alias exists makes ES\n * auto-create a physical `<x>` index with default `text` mappings, breaking\n * aggregations forever. The bootstrap used to run at app boot, which turned\n * \"ES is down\" into a full-admin 500 (issue #610). Moving it onto the import\n * path keeps the guarantee while scoping ES availability to imports/search —\n * the rest of the admin boots and runs with ES down.\n */\nexport type EnsureIndexResolver = () => Promise<unknown>\n\n/**\n * Resolves the value of `import_run.filePath` (whatever the upload\n * route persisted there — typically a storage adapter key) to a\n * readable LOCAL filesystem path that {@link runImport} can hand to\n * `node:fs.createReadStream`.\n *\n * **Why this exists:** the streaming reader (`streamFeed`) reads from\n * local disk via `createReadStream`. The upload route may persist a\n * remote-storage object key (R2, S3, …) on `import_run.filePath`\n * because that's what `@murumets-ee/storage` returns. Without a\n * resolver, `createReadStream('uploads/2026/05/<uuid>/feed.txt')`\n * crashes with `ENOENT`. The resolver is the documented integration\n * point — typical wiring downloads the storage object to a tmpfile\n * and returns its path.\n *\n * **`cleanup`** runs after the run finishes (success OR failure). The\n * worker awaits it best-effort — a failed cleanup logs but does not\n * crash the run.\n *\n * **No-resolver fallback:** when `resolveFilePath` is unset, the\n * worker treats `import_run.filePath` as already a local FS path\n * (back-compat with the original PoC design where uploads landed on\n * local disk). This stays valid for fixture-driven tests + on-disk\n * deployments.\n */\nexport type FilePathResolver = (\n storageKey: string,\n) => Promise<{ localPath: string; cleanup?: () => Promise<void> }>\n\n/**\n * Context handed to a {@link PairSink} alongside the harvested pairs. Carries the\n * run identity so the sink can build `source` attribution\n * (`feed:<supplierId>:<importBatchId>`, D20) and scope its writes.\n */\nexport interface PairSinkContext {\n importRunId: string\n /** The `import_run.params` object — the sink reads `supplierId` / `importBatchId`. */\n params: Record<string, unknown>\n /**\n * The run's structured logger (child), forwarded so the sink's INTERNAL telemetry —\n * a params-parse miss that skips ingestion, an over-merge flag — is visible. Without it\n * a misconfigured run looks successful from the worker's vantage (a params-miss returns\n * normally, so the worker logs \"applied\"). Optional; the sink no-ops its logs if absent.\n */\n logger?: Logger\n}\n\n/**\n * Optional post-run consumer for the directional supersession pairs the runner\n * harvested as a side-output (commerce-demo PR6, SD007). Invoked ONCE after a\n * successful run with the full in-memory pair buffer (read-once / merge-after) so\n * the consumer (the commerce replacement-merge engine) can build equivalence groups\n * and bulk-write them to Postgres.\n *\n * **Why injected, not imported.** The merge writes the CONSUMER's Postgres tables,\n * but the runner + this worker live in the `@murumets-ee/imports` leaf, which must\n * not depend on commerce. The app composes the sink (commerce exports it, the app\n * passes it to `imports({ pairSink })`) — the same injection shape as `esClient` /\n * `resolveFilePath` / `ensureIndex`. The boundary holds.\n *\n * **Best-effort.** A sink throw is logged and SWALLOWED — the catalogue import\n * (the ES write) has already succeeded and must not be undone by a secondary\n * replacement-ingestion failure. The merge is idempotent, so a re-import recovers.\n */\nexport type PairSink = (pairs: readonly DirectionalPair[], ctx: PairSinkContext) => Promise<void>\n\nexport interface RunImportHandlerConfig {\n /** AdminClient over the `import_run` entity. */\n importRuns: ImportRunClient\n /** Transform registry to dispatch against. Defaults to the process-global singleton. */\n transforms: TransformRegistry\n /** Resolver for the ES client. */\n esClient: EsClientResolver\n /** ES alias / index to bulk-write into. Per D6 callers always pass an alias. */\n esIndex: string\n /**\n * Optional bootstrap for the destination index/alias, run once per job\n * before any bulk-write. See {@link EnsureIndexResolver}. Omit for\n * deployments that bootstrap the index out-of-band (e.g. a reindex job).\n */\n ensureIndex?: EnsureIndexResolver\n /**\n * Optional resolver for `import_run.filePath`. See {@link FilePathResolver}.\n * Required when uploads land in remote storage (R2/S3); optional for\n * on-disk PoC setups.\n */\n resolveFilePath?: FilePathResolver\n /**\n * Optional consumer for the directional supersession pairs harvested as a\n * side-output (commerce-demo PR6, SD007). See {@link PairSink}. Omit for\n * deployments that don't ingest replacement codes — the pairs are simply\n * discarded after the run.\n */\n pairSink?: PairSink\n /** Optional structured logger. Defaults to silent. */\n logger?: Logger\n}\n\n/**\n * Build the `JobHandler` for {@link importsRunJob}. The returned\n * handler is what gets passed to `registerJob`.\n *\n * The body is wrapped in `runAsCli` so AdminClient calls inside have a\n * synthetic `cli` admin context — `auditable()` records `updatedBy:\n * 'cli'` rather than NULL, and the firewall checker passes. This is\n * the documented worker entry-point pattern (see `runAsCli` JSDoc in\n * `@murumets-ee/core`).\n */\nexport function createRunImportHandler(\n config: RunImportHandlerConfig,\n): (job: ImportsRunJobContext) => Promise<void> {\n const {\n importRuns,\n transforms,\n esClient,\n esIndex,\n resolveFilePath,\n ensureIndex,\n pairSink,\n logger,\n } = config\n\n return async (job: ImportsRunJobContext): Promise<void> =>\n runAsCli(async () => {\n const { importRunId } = job.payload\n const log = logger?.child({ jobId: job.id, importRunId, type: 'imports:run' })\n\n const run = await importRuns.findById(importRunId)\n if (!run) {\n // Row was deleted between enqueue and dispatch — nothing to do.\n // Don't throw: a thrown handler retries with no payoff because\n // the row stays gone. Log + return so the queue marks the job\n // completed.\n log?.warn('import_run row not found — skipping')\n return\n }\n\n // Re-claim detection. The queue's `recoverStuckJobs` sweep flips a\n // crashed worker's `toolkit_jobs` row from `processing` back to\n // `pending` — but the queue layer has no knowledge of `import_run`,\n // so this row stays at `status='running'` from the prior attempt.\n // When the queue re-dispatches the job, we land here with a row\n // that the operator-facing Active panel has been showing as live\n // for however long lock-recovery took (default 30 min). The\n // existing `update` below overwrites `queueJobId` + `startedAt` so\n // there is no STATE corruption — but emit a structured warning so\n // ops can correlate this re-run with the worker crash that\n // produced it. The sibling `sweepStaleImportRuns` job below catches\n // the case where re-dispatch never happens (queue marks the job\n // dead, or the queue worker is offline long-term).\n if (run.status === 'running') {\n log?.warn(\n {\n previousQueueJobId: run.queueJobId,\n previousStartedAt: run.startedAt,\n },\n 'imports: re-claiming a row left in running by a prior worker — likely a crash recovered by queue lock-recovery',\n )\n }\n\n // `transformName`, `filePath`, `label` are declared `field.text({\n // required: true })` so the inferred DTO already types them as\n // `string` — no cast needed. `params` is `field.json()` which\n // infers as `JsonValue | undefined`; narrow to a record so the\n // transform receives an object.\n const transform = transforms.get(run.transformName)\n if (!transform) {\n await markFailed(importRuns, importRunId, `Unknown transform \"${run.transformName}\"`)\n throw new Error(`No transform registered for name \"${run.transformName}\"`)\n }\n const params: Record<string, unknown> =\n run.params && typeof run.params === 'object' && !Array.isArray(run.params)\n ? (run.params as Record<string, unknown>)\n : {}\n\n // Reader config (format/delimiter/header/sheet) travels in\n // `params.feed` for mapping-driven runs (commerce-demo PR3). Carmaker\n // runs omit it → empty config → tab-delimited default, so the existing\n // path is byte-for-byte unchanged. Re-validate here as defense-in-depth\n // even though the enqueuing route already validated the saved mapping.\n const feedRaw =\n params.feed && typeof params.feed === 'object' && !Array.isArray(params.feed)\n ? params.feed\n : {}\n const feedParsed = feedReaderConfigSchema.safeParse(feedRaw)\n if (!feedParsed.success) {\n const reason = feedParsed.error.issues[0]?.message ?? 'invalid feed reader config'\n await markFailed(importRuns, importRunId, `Invalid params.feed: ${reason}`)\n throw new Error(`Invalid params.feed for import_run ${importRunId}: ${reason}`)\n }\n const feedConfig = feedParsed.data\n\n await importRuns.update(importRunId, {\n status: 'running' satisfies ImportRunStatus,\n queueJobId: job.id,\n startedAt: new Date(),\n })\n // Realtime: row transitioned pending → running. Subscribers\n // refetch to surface the change in the imports admin page.\n // Best-effort — a publish failure is logged inside `publishSafe`\n // and does NOT roll back the row update.\n publishImportRunStarted(importRunId)\n\n let resolvedClient: EsClientLike\n try {\n resolvedClient = await esClient()\n } catch (err) {\n const reason = err instanceof Error ? err.message : String(err)\n await markFailed(importRuns, importRunId, `Could not resolve ES client: ${reason}`)\n throw err\n }\n\n // Ensure the destination alias exists BEFORE any bulk-write. Moved here\n // from app boot (issue #610): the foot-gun is the worker auto-creating a\n // mis-mapped physical index, so the guard belongs on the import path —\n // not gating the whole app on ES. Idempotent + scoped: an ES outage (or\n // the `ensureAliasedIndex` physical-collision guard firing) fails THIS\n // job loudly while the rest of the admin keeps working.\n if (ensureIndex) {\n try {\n await ensureIndex()\n } catch (err) {\n const reason = err instanceof Error ? err.message : String(err)\n await markFailed(importRuns, importRunId, `Could not ensure ES index/alias: ${reason}`)\n throw err\n }\n }\n\n // Materialise the file locally if the consumer wired a resolver.\n // The `cleanup` callback (when present) runs in the `finally`\n // below regardless of run outcome so a tmpfile created here\n // doesn't outlive the handler.\n let localPath: string\n let fileCleanup: (() => Promise<void>) | undefined\n if (resolveFilePath) {\n try {\n const resolved = await resolveFilePath(run.filePath)\n localPath = resolved.localPath\n fileCleanup = resolved.cleanup\n } catch (err) {\n const reason = err instanceof Error ? err.message : String(err)\n await markFailed(importRuns, importRunId, `Could not resolve filePath: ${reason}`)\n throw err\n }\n } else {\n // Back-compat: no resolver wired → treat `filePath` as already\n // a local FS path. Fine for fixture-driven tests + the original\n // on-disk PoC design.\n localPath = run.filePath\n }\n\n // Pre-pass line count: a 16 MB tab-delimited feed counts in\n // ~50ms; a 500 MB feed in <2s. Cheap relative to the full\n // import. Gives the operator a denominator (`processed / total`)\n // so the imports-page panel can render an actual percent\n // instead of just \"rows so far\".\n // Pre-pass row count is a `\\n`-tally over a delimited file — meaningless\n // on an xlsx binary, so skip it there (the UI falls back to bare counts).\n // `hasHeader` from the feed config keeps the header-exclusion correct for\n // headerless mappings.\n let totalRows: number | undefined\n if (feedConfig.format !== 'xlsx') {\n try {\n totalRows = await countDataRows(\n localPath,\n feedConfig.hasHeader !== undefined ? { hasHeader: feedConfig.hasHeader } : {},\n )\n log?.info({ importRunId, totalRows }, 'imports: pre-pass row count')\n } catch (err) {\n // Don't fail the run on a count miss — the actual import is\n // unaffected; the operator just loses the percent denominator.\n log?.warn({ err }, 'imports: pre-pass row count failed — continuing without total')\n }\n }\n\n let result: Awaited<ReturnType<typeof runImport>>\n try {\n // Build feed options with the conditional-spread idiom (NOT\n // `...feedConfig`) so absent optionals stay ABSENT — spreading a\n // `T | undefined` value would set the key to `undefined`, which\n // `exactOptionalPropertyTypes` rejects against `ReadFeedOptions`.\n const feedOpts: ReadFeedOptions = {\n filePath: localPath,\n ...(feedConfig.format !== undefined && { format: feedConfig.format }),\n ...(feedConfig.delimiter !== undefined && { delimiter: feedConfig.delimiter }),\n ...(feedConfig.hasHeader !== undefined && { hasHeader: feedConfig.hasHeader }),\n ...(feedConfig.headerRowIndex !== undefined && {\n headerRowIndex: feedConfig.headerRowIndex,\n }),\n ...(feedConfig.sheet !== undefined && { sheet: feedConfig.sheet }),\n }\n result = await runImport({\n importRunId,\n runLabel: run.label,\n params,\n transform,\n feed: feedOpts,\n esClient: resolvedClient,\n esIndex,\n onProgress: (progress) =>\n job.updateProgress(totalRows !== undefined ? { ...progress, totalRows } : progress),\n ...(log !== undefined && { logger: log }),\n })\n } catch (err) {\n const reason = err instanceof Error ? err.message : String(err)\n await markFailed(importRuns, importRunId, reason)\n if (fileCleanup) {\n await fileCleanup().catch((cleanupErr: unknown) => {\n log?.warn({ err: cleanupErr }, 'filePath cleanup after failed run threw — ignoring')\n })\n }\n throw err\n }\n\n // Cleanup runs before the success-update so a cleanup throw\n // surfaces as a logged warning rather than a crashed handler at\n // the worst possible moment (just after the row was about to be\n // marked succeeded).\n if (fileCleanup) {\n await fileCleanup().catch((cleanupErr: unknown) => {\n log?.warn({ err: cleanupErr }, 'filePath cleanup after successful run threw — ignoring')\n })\n }\n\n await importRuns.update(importRunId, {\n status: 'succeeded' satisfies ImportRunStatus,\n finishedAt: new Date(),\n totals: {\n rowsRead: result.rowsRead,\n rowsSucceeded: result.rowsSucceeded,\n rowsFailed: result.rowsFailed,\n rowsSkipped: result.rowsSkipped,\n rowsWarned: result.rowsWarned,\n batchesCompleted: result.batchesCompleted,\n // Phase-2 core-charge attach (commerce-demo PR7) — durable on the run row so\n // an operator sees how many `*S` deposits attached vs orphaned, not just the\n // orphan warnings in `warningSummary`.\n patchesApplied: result.patchesApplied,\n patchesOrphaned: result.patchesOrphaned,\n },\n // The typed `ErrorTrackerSnapshot` is fully JSON-serialisable but\n // doesn't structurally satisfy entity's recursive `JsonValue`\n // (objects without an index signature aren't `Record<string,\n // JsonValue>`). Round-trip through `JSON.parse(JSON.stringify(...))`\n // to land as the recursive JSON shape — the runtime cost is one\n // serialise per import_run completion, the type-cast cost is zero.\n errorSummary: JSON.parse(JSON.stringify(result.errors)),\n // Non-fatal warnings (commerce-demo PR4) — kept separate from\n // `errorSummary` so a `no_purchase_price` advisory never reads as a\n // failed row. Same serialise-roundtrip rationale as `errorSummary`.\n warningSummary: JSON.parse(JSON.stringify(result.warnings)),\n })\n // Realtime: row transitioned to terminal-success. Subscribers\n // refetch to move the row from Active → Recent.\n publishImportRunSucceeded(importRunId, {\n rowsRead: result.rowsRead,\n rowsSucceeded: result.rowsSucceeded,\n rowsFailed: result.rowsFailed,\n })\n\n // Side-output: hand the harvested supersession pairs to the injected sink\n // (commerce-demo PR6, SD007) AFTER the catalogue run is marked succeeded — the\n // ES import is primary; replacement ingestion is secondary. Best-effort: a sink\n // throw is logged + swallowed so a Postgres hiccup can't undo a successful\n // catalogue import (the merge is idempotent → a re-import recovers).\n if (pairSink && result.pairs.length > 0) {\n try {\n await pairSink(result.pairs, {\n importRunId,\n params,\n ...(log !== undefined && { logger: log }),\n })\n // \"invoked\" not \"applied\": the sink may internally skip (e.g. missing run params);\n // it logs its own applied/skipped outcome via the forwarded logger.\n log?.info({ importRunId, pairs: result.pairs.length }, 'imports: pairSink invoked')\n } catch (err) {\n log?.warn(\n { importRunId, pairs: result.pairs.length, err },\n 'imports: pairSink threw — replacement ingestion skipped, catalogue import unaffected',\n )\n }\n }\n\n log?.info(\n {\n rowsRead: result.rowsRead,\n rowsSucceeded: result.rowsSucceeded,\n rowsFailed: result.rowsFailed,\n rowsSkipped: result.rowsSkipped,\n batches: result.batchesCompleted,\n },\n 'import_run completed',\n )\n })\n}\n\n/** Write a `failed` row with a fatal error reason. Best-effort — logs but does not throw on DB failure. */\nasync function markFailed(importRuns: ImportRunClient, id: string, reason: string): Promise<void> {\n try {\n await importRuns.update(id, {\n status: 'failed' satisfies ImportRunStatus,\n finishedAt: new Date(),\n errorSummary: { fatal: reason },\n })\n // Realtime: row transitioned to terminal-failure. Subscribers\n // refetch to surface the failure in the Recent panel. Publish\n // is best-effort and only fires when the persist succeeds —\n // skipping it on persist failure keeps subscribers consistent\n // with the DB (no event for a row that didn't transition).\n publishImportRunFailed(id, reason)\n } catch {\n // The handler already threw / will throw — the queue will surface\n // the original error. A second failure here just logs noise.\n }\n}\n\nexport interface ImportsRunDeadListenerConfig {\n importRuns: ImportRunClient\n /** Optional structured logger. */\n logger?: Logger\n}\n\n/**\n * Build a {@link QueueTerminalNotifier} that mirrors `imports:run`\n * dead-job events into `import_run.status='failed'`.\n *\n * **Why this exists.** The `imports:run` handler's normal failure path\n * already calls `markFailed` before re-throwing, so a synchronous\n * handler error reaches the operator-visible `failed` status without\n * help from anyone. The case THIS listener catches is the one\n * `markFailed` never gets to: the worker process is killed mid-handler\n * (OOM, container restart, kill -9), the queue's lock-recovery sweep\n * eventually reclaims the row, and the queue ultimately marks the job\n * `dead` (defaultRetries: 0 + repeated process death = dead-letter\n * with `import_run.status` still `running`). Without this listener\n * the row sits in the operator's Active panel forever.\n *\n * **Generic primitive.** This is a thin subscriber over the queue's\n * multi-subscriber `QueueTerminalNotifier` registry — the same shape\n * graphile-worker (`job:failed`) and BullMQ (`failed`) expose. Other\n * domain plugins that mirror queue state (ticketing-reply, future\n * exports/reports/...) can register their own listeners exactly the\n * same way without coordinating via a per-domain sweep job.\n *\n * **Filtering.** The listener is invoked for EVERY dead event in the\n * process. We early-return on any `jobType` other than `imports:run`\n * so the per-event cost on unrelated jobs is a single property read.\n *\n * **Idempotency.** A dead event delivered after the row has already\n * reached its own terminal state (succeeded / failed / cancelled) is\n * ignored — the only transition we make is `running → failed`. This\n * keeps a late replay or out-of-order delivery from clobbering a\n * row whose worker reached `markFailed` faster than the queue's\n * dead-letter dispatch.\n *\n * The handler is wrapped in `runAsCli` so AdminClient calls inside\n * have a synthetic CLI admin context — same pattern as\n * {@link createRunImportHandler}.\n */\nexport function createImportsRunDeadListener(\n config: ImportsRunDeadListenerConfig,\n): QueueTerminalNotifier {\n const { importRuns, logger } = config\n\n return {\n onJobDead: async (event: QueueJobDeadEvent): Promise<void> => {\n if (event.jobType !== 'imports:run') return\n\n await runAsCli(async () => {\n const log = logger?.child({\n event: 'imports.run.dead',\n jobId: event.jobId,\n type: 'imports:run',\n })\n\n // Look up the mirror row by `queueJobId`. Cap to 1 — the\n // upload route writes one `import_run` per enqueued job.\n //\n // `AdminClient.getTable()` returns `PgTableWithColumns<any>`\n // because entity tables have dynamic column shapes (see the\n // `getTable()` JSDoc in `packages/entity/src/admin/client.ts`\n // and the documented-`any`-cases list in CLAUDE.md). Cast\n // the table to a structural shape covering only the column\n // we touch, so both `eq()` and type-coverage see a typed\n // operand instead of an `any`-derived property access.\n const table = importRuns.getTable() as unknown as { queueJobId: PgColumn }\n const matches = await importRuns.findMany({\n where: eq(table.queueJobId, event.jobId),\n limit: 1,\n })\n const row = matches[0]\n if (!row) {\n // No mirror row for this dead job. Either it was deleted\n // by the operator, or the dead event is for a job\n // unrelated to imports despite the type filter\n // (shouldn't happen, but is cheap to log).\n log?.warn('queue.job.dead with no matching import_run — skipping')\n return\n }\n\n // Only transition rows that haven't already reached their\n // own terminal state. A late dead-event must not overwrite\n // a `succeeded` row that the worker reached via the normal\n // path before the queue's dead-letter dispatch fired.\n if (row.status !== 'running' && row.status !== 'pending') {\n log?.info(\n { importRunId: row.id, status: row.status },\n 'queue.job.dead arrived for a row already in a terminal state — leaving as-is',\n )\n return\n }\n\n // Dead-from-`pending` is the rarer + more interesting case:\n // the worker process died (or never claimed) BEFORE it could\n // flip the row to `running`. Log it at warn so ops can spot\n // a queue+worker-startup pathology that the standard `running\n // → failed` transition wouldn't surface. Dead-from-`running`\n // is the documented \"crash mid-import\" case — info is enough.\n if (row.status === 'pending') {\n log?.warn(\n { importRunId: row.id, attempts: event.attempts },\n 'queue.job.dead arrived for an import_run still in pending — worker crashed BEFORE claim/lookup',\n )\n }\n\n await markFailed(importRuns, row.id, event.error)\n log?.info(\n { importRunId: row.id, attempts: event.attempts, priorStatus: row.status },\n 'queue.job.dead → import_run marked failed',\n )\n })\n },\n }\n}\n"],"mappings":"wQA2DA,MAAa,EAA6B,EAAE,OAAO,CACjD,YAAa,EAAE,OAAO,CAAC,CAAC,KAAK,CAC/B,CAAC,EAcY,EAAqD,EAAU,CAC1E,KAAM,cACN,YAAa,mEACb,OAAQ,EACR,eAAgB,CAClB,CAAC,EAsID,SAAgB,EACd,EAC8C,CAC9C,GAAM,CACJ,aACA,aACA,WACA,UACA,kBACA,cACA,WACA,UACE,EAEJ,OAAO,KAAO,IACZ,EAAS,SAAY,CACnB,GAAM,CAAE,eAAgB,EAAI,QACtB,EAAM,GAAQ,MAAM,CAAE,MAAO,EAAI,GAAI,cAAa,KAAM,aAAc,CAAC,EAEvE,EAAM,MAAM,EAAW,SAAS,CAAW,EACjD,GAAI,CAAC,EAAK,CAKR,GAAK,KAAK,qCAAqC,EAC/C,MACF,CAeI,EAAI,SAAW,WACjB,GAAK,KACH,CACE,mBAAoB,EAAI,WACxB,kBAAmB,EAAI,SACzB,EACA,gHACF,EAQF,IAAM,EAAY,EAAW,IAAI,EAAI,aAAa,EAClD,GAAI,CAAC,EAEH,MADA,MAAM,EAAW,EAAY,EAAa,sBAAsB,EAAI,cAAc,EAAE,EAC1E,MAAM,qCAAqC,EAAI,cAAc,EAAE,EAE3E,IAAM,EACJ,EAAI,QAAU,OAAO,EAAI,QAAW,UAAY,CAAC,MAAM,QAAQ,EAAI,MAAM,EACpE,EAAI,OACL,CAAC,EAOD,EACJ,EAAO,MAAQ,OAAO,EAAO,MAAS,UAAY,CAAC,MAAM,QAAQ,EAAO,IAAI,EACxE,EAAO,KACP,CAAC,EACD,EAAa,EAAuB,UAAU,CAAO,EAC3D,GAAI,CAAC,EAAW,QAAS,CACvB,IAAM,EAAS,EAAW,MAAM,OAAO,EAAE,EAAE,SAAW,6BAEtD,MADA,MAAM,EAAW,EAAY,EAAa,wBAAwB,GAAQ,EAChE,MAAM,sCAAsC,EAAY,IAAI,GAAQ,CAChF,CACA,IAAM,EAAa,EAAW,KAE9B,MAAM,EAAW,OAAO,EAAa,CACnC,OAAQ,UACR,WAAY,EAAI,GAChB,UAAW,IAAI,IACjB,CAAC,EAKD,EAAwB,CAAW,EAEnC,IAAI,EACJ,GAAI,CACF,EAAiB,MAAM,EAAS,CAClC,OAAS,EAAK,CAGZ,MADA,MAAM,EAAW,EAAY,EAAa,gCAD3B,aAAe,MAAQ,EAAI,QAAU,OAAO,CAAG,GACoB,EAC5E,CACR,CAQA,GAAI,EACF,GAAI,CACF,MAAM,EAAY,CACpB,OAAS,EAAK,CAGZ,MADA,MAAM,EAAW,EAAY,EAAa,oCAD3B,aAAe,MAAQ,EAAI,QAAU,OAAO,CAAG,GACwB,EAChF,CACR,CAOF,IAAI,EACA,EACJ,GAAI,EACF,GAAI,CACF,IAAM,EAAW,MAAM,EAAgB,EAAI,QAAQ,EACnD,EAAY,EAAS,UACrB,EAAc,EAAS,OACzB,OAAS,EAAK,CAGZ,MADA,MAAM,EAAW,EAAY,EAAa,+BAD3B,aAAe,MAAQ,EAAI,QAAU,OAAO,CAAG,GACmB,EAC3E,CACR,KAKA,GAAY,EAAI,SAYlB,IAAI,EACJ,GAAI,EAAW,SAAW,OACxB,GAAI,CACF,EAAY,MAAM,EAChB,EACA,EAAW,YAAc,IAAA,GAAkD,CAAC,EAAvC,CAAE,UAAW,EAAW,SAAU,CACzE,EACA,GAAK,KAAK,CAAE,cAAa,WAAU,EAAG,6BAA6B,CACrE,OAAS,EAAK,CAGZ,GAAK,KAAK,CAAE,KAAI,EAAG,+DAA+D,CACpF,CAGF,IAAI,EACJ,GAAI,CAKF,IAAM,EAA4B,CAChC,SAAU,EACV,GAAI,EAAW,SAAW,IAAA,IAAa,CAAE,OAAQ,EAAW,MAAO,EACnE,GAAI,EAAW,YAAc,IAAA,IAAa,CAAE,UAAW,EAAW,SAAU,EAC5E,GAAI,EAAW,YAAc,IAAA,IAAa,CAAE,UAAW,EAAW,SAAU,EAC5E,GAAI,EAAW,iBAAmB,IAAA,IAAa,CAC7C,eAAgB,EAAW,cAC7B,EACA,GAAI,EAAW,QAAU,IAAA,IAAa,CAAE,MAAO,EAAW,KAAM,CAClE,EACA,EAAS,MAAM,EAAU,CACvB,cACA,SAAU,EAAI,MACd,SACA,YACA,KAAM,EACN,SAAU,EACV,UACA,WAAa,GACX,EAAI,eAAe,IAAc,IAAA,GAAyC,EAA7B,CAAE,GAAG,EAAU,WAAU,CAAY,EACpF,GAAI,IAAQ,IAAA,IAAa,CAAE,OAAQ,CAAI,CACzC,CAAC,CACH,OAAS,EAAK,CAQZ,MANA,MAAM,EAAW,EAAY,EADd,aAAe,MAAQ,EAAI,QAAU,OAAO,CAAG,CACd,EAC5C,GACF,MAAM,EAAY,CAAC,CAAC,MAAO,GAAwB,CACjD,GAAK,KAAK,CAAE,IAAK,CAAW,EAAG,oDAAoD,CACrF,CAAC,EAEG,CACR,CAqDA,GA/CI,GACF,MAAM,EAAY,CAAC,CAAC,MAAO,GAAwB,CACjD,GAAK,KAAK,CAAE,IAAK,CAAW,EAAG,wDAAwD,CACzF,CAAC,EAGH,MAAM,EAAW,OAAO,EAAa,CACnC,OAAQ,YACR,WAAY,IAAI,KAChB,OAAQ,CACN,SAAU,EAAO,SACjB,cAAe,EAAO,cACtB,WAAY,EAAO,WACnB,YAAa,EAAO,YACpB,WAAY,EAAO,WACnB,iBAAkB,EAAO,iBAIzB,eAAgB,EAAO,eACvB,gBAAiB,EAAO,eAC1B,EAOA,aAAc,KAAK,MAAM,KAAK,UAAU,EAAO,MAAM,CAAC,EAItD,eAAgB,KAAK,MAAM,KAAK,UAAU,EAAO,QAAQ,CAAC,CAC5D,CAAC,EAGD,EAA0B,EAAa,CACrC,SAAU,EAAO,SACjB,cAAe,EAAO,cACtB,WAAY,EAAO,UACrB,CAAC,EAOG,GAAY,EAAO,MAAM,OAAS,EACpC,GAAI,CACF,MAAM,EAAS,EAAO,MAAO,CAC3B,cACA,SACA,GAAI,IAAQ,IAAA,IAAa,CAAE,OAAQ,CAAI,CACzC,CAAC,EAGD,GAAK,KAAK,CAAE,cAAa,MAAO,EAAO,MAAM,MAAO,EAAG,2BAA2B,CACpF,OAAS,EAAK,CACZ,GAAK,KACH,CAAE,cAAa,MAAO,EAAO,MAAM,OAAQ,KAAI,EAC/C,sFACF,CACF,CAGF,GAAK,KACH,CACE,SAAU,EAAO,SACjB,cAAe,EAAO,cACtB,WAAY,EAAO,WACnB,YAAa,EAAO,YACpB,QAAS,EAAO,gBAClB,EACA,sBACF,CACF,CAAC,CACL,CAGA,eAAe,EAAW,EAA6B,EAAY,EAA+B,CAChG,GAAI,CACF,MAAM,EAAW,OAAO,EAAI,CAC1B,OAAQ,SACR,WAAY,IAAI,KAChB,aAAc,CAAE,MAAO,CAAO,CAChC,CAAC,EAMD,EAAuB,EAAI,CAAM,CACnC,MAAQ,CAGR,CACF,CA6CA,SAAgB,EACd,EACuB,CACvB,GAAM,CAAE,aAAY,UAAW,EAE/B,MAAO,CACL,UAAW,KAAO,IAA4C,CACxD,EAAM,UAAY,eAEtB,MAAM,EAAS,SAAY,CACzB,IAAM,EAAM,GAAQ,MAAM,CACxB,MAAO,mBACP,MAAO,EAAM,MACb,KAAM,aACR,CAAC,EAYK,EAAQ,EAAW,SAAS,EAK5B,GAAM,MAJU,EAAW,SAAS,CACxC,MAAO,EAAG,EAAM,WAAY,EAAM,KAAK,EACvC,MAAO,CACT,CAAC,EAAA,CACmB,GACpB,GAAI,CAAC,EAAK,CAKR,GAAK,KAAK,uDAAuD,EACjE,MACF,CAMA,GAAI,EAAI,SAAW,WAAa,EAAI,SAAW,UAAW,CACxD,GAAK,KACH,CAAE,YAAa,EAAI,GAAI,OAAQ,EAAI,MAAO,EAC1C,8EACF,EACA,MACF,CAQI,EAAI,SAAW,WACjB,GAAK,KACH,CAAE,YAAa,EAAI,GAAI,SAAU,EAAM,QAAS,EAChD,gGACF,EAGF,MAAM,EAAW,EAAY,EAAI,GAAI,EAAM,KAAK,EAChD,GAAK,KACH,CAAE,YAAa,EAAI,GAAI,SAAU,EAAM,SAAU,YAAa,EAAI,MAAO,EACzE,2CACF,CACF,CAAC,CACH,CACF,CACF"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import{a as e}from"./runner-
|
|
2
|
-
//# sourceMappingURL=xlsx-reader-
|
|
1
|
+
import{a as e}from"./runner-BOwPUm7W.mjs";function t(e){if(e==null)return``;if(typeof e==`string`)return e;if(typeof e==`number`||typeof e==`boolean`)return String(e);if(e instanceof Date)return e.toISOString();if(`richText`in e&&Array.isArray(e.richText))return e.richText.map(e=>e.text).join(``);if(`text`in e&&typeof e.text==`string`)return e.text;if(`result`in e&&e.result!==void 0&&e.result!==null){let t=e.result;return t instanceof Date?t.toISOString():typeof t==`object`?``:String(t)}return``}function n(e){let n=[];if(Array.isArray(e))for(let r=1;r<e.length;r+=1)n[r-1]=t(e[r]);else for(let[r,i]of Object.entries(e)){let e=Number.parseInt(r,10);!Number.isNaN(e)&&e>=1&&(n[e-1]=t(i))}for(let e=0;e<n.length;e+=1)n[e]===void 0&&(n[e]=``);return n}async function*r(t){let{filePath:r,hasHeader:i=!0,headerRowIndex:a=0,sheet:o}=t,s=new(await(import(`exceljs`))).stream.xlsx.WorkbookReader(r,{worksheets:`emit`,sharedStrings:`cache`,styles:`ignore`,hyperlinks:`ignore`,entries:`ignore`}),c;if(o!==void 0&&(c=Number.parseInt(o,10),Number.isNaN(c)||c<1))throw Error(`Invalid xlsx sheet "${o}" — expected a 1-based worksheet index`);let l=0,u=0;for await(let t of s){if(u+=1,!(c===void 0?u===1:u===c)){for await(let e of t);continue}let r=0,o=null;for await(let s of t){r+=1;let t=n(s.values);if(i&&o===null){if(r<a+1)continue;o=t.map(e=>e.trim()),e(o);continue}l+=1;let c={};if(o)for(let e=0;e<o.length;e+=1){let n=o[e];n&&(c[n]=t[e]??``)}else for(let e=0;e<t.length;e+=1)c[String(e)]=t[e]??``;yield{rowNumber:l,row:c}}return}if(c!==void 0)throw Error(`xlsx worksheet ${c} not found`)}export{r as streamXlsx};
|
|
2
|
+
//# sourceMappingURL=xlsx-reader-CUbaPt3-.mjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xlsx-reader-
|
|
1
|
+
{"version":3,"file":"xlsx-reader-CUbaPt3-.mjs","names":[],"sources":["../src/xlsx-reader.ts"],"sourcesContent":["/**\n * Streaming XLSX reader (commerce-demo phase 02, D005) — the exceljs branch of\n * the feed reader. Yields the SAME `AsyncIterable<{ rowNumber, row }>` contract as\n * the csv-parse `streamFeed`, so the runner / transform layer downstream is\n * format-agnostic.\n *\n * **exceljs over SheetJS** (R02): we parse operator-uploaded spreadsheets — the\n * exact threat model of the unfixed npm `xlsx` prototype-pollution CVE\n * (CVE-2023-30533). exceljs is MIT, ships its own types, has a real streaming\n * `WorkbookReader`, and carries no equivalent advisory.\n *\n * **exceljs is `await import()`-ed**, never statically imported, so it stays out\n * of this package's static module graph — heavy + Node-only, it must not be\n * inlinable into any `*-ui` server-pages bundle (CLAUDE.md bundle budgets). The\n * `read-feed` dispatcher additionally only loads THIS module for `format: 'xlsx'`.\n */\n\nimport type { CellValue } from 'exceljs'\nimport { assertUniqueHeaders, type StreamFeedRow } from './streaming.js'\n\nexport interface XlsxReadOptions {\n /** Path to the .xlsx file on local disk. */\n filePath: string\n /** First row is the header (column names). `false` → positional `{ \"0\": … }` keys. Default `true`. */\n hasHeader?: boolean\n /** 0-based index of the header row (rows before it are skipped). Default `0`. */\n headerRowIndex?: number\n /** Worksheet to read, as a 1-based index string. Omit → the first worksheet. */\n sheet?: string\n}\n\n/** Convert one exceljs cell value to a plain string (feeds are text/number cells). */\nfunction cellToString(v: CellValue): string {\n if (v === null || v === undefined) return ''\n if (typeof v === 'string') return v\n if (typeof v === 'number' || typeof v === 'boolean') return String(v)\n if (v instanceof Date) return v.toISOString()\n // Object cell values: rich text, hyperlink, formula result, error.\n if ('richText' in v && Array.isArray(v.richText)) {\n return v.richText.map((r) => r.text).join('')\n }\n if ('text' in v && typeof v.text === 'string') return v.text\n if ('result' in v && v.result !== undefined && v.result !== null) {\n const r = v.result\n if (r instanceof Date) return r.toISOString()\n // A nested error object (formula error) surfaces as empty; primitives stringify.\n return typeof r === 'object' ? '' : String(r)\n }\n return ''\n}\n\n/** Normalise an exceljs `row.values` (1-indexed array OR object form) to a 0-indexed string[]. */\nfunction rowCells(values: CellValue[] | { [key: string]: CellValue }): string[] {\n const out: string[] = []\n if (Array.isArray(values)) {\n // exceljs row arrays are 1-indexed (`values[0]` is empty); shift to 0-based.\n for (let i = 1; i < values.length; i += 1) out[i - 1] = cellToString(values[i])\n } else {\n for (const [k, v] of Object.entries(values)) {\n const idx = Number.parseInt(k, 10)\n if (!Number.isNaN(idx) && idx >= 1) out[idx - 1] = cellToString(v)\n }\n }\n // Fill any holes (sparse rows) with empty strings — the contract is \"missing\n // cell = ''\", matching streamFeed.\n for (let i = 0; i < out.length; i += 1) if (out[i] === undefined) out[i] = ''\n return out\n}\n\n/**\n * Async-iterable over the data rows of an .xlsx file. `rowNumber` is the 1-based\n * DATA row index (header excluded). Cell values are coerced to strings so the\n * mapping engine's per-column transforms (which expect raw string cells) apply\n * uniformly across csv/tsv/xlsx.\n */\nexport async function* streamXlsx(options: XlsxReadOptions): AsyncIterable<StreamFeedRow> {\n const { filePath, hasHeader = true, headerRowIndex = 0, sheet } = options\n const ExcelJS = await import('exceljs')\n const reader = new ExcelJS.stream.xlsx.WorkbookReader(filePath, {\n worksheets: 'emit',\n sharedStrings: 'cache',\n styles: 'ignore',\n hyperlinks: 'ignore',\n entries: 'ignore',\n })\n\n // `WorksheetReader` doesn't expose its sheet id, so count emitted worksheets\n // (1-based) to match the configured `sheet`. Default: the first worksheet.\n // A provided-but-invalid sheet FAILS FAST rather than silently reading sheet 1.\n let targetSheetIndex: number | undefined\n if (sheet !== undefined) {\n targetSheetIndex = Number.parseInt(sheet, 10)\n if (Number.isNaN(targetSheetIndex) || targetSheetIndex < 1) {\n throw new Error(`Invalid xlsx sheet \"${sheet}\" — expected a 1-based worksheet index`)\n }\n }\n let dataRowNumber = 0\n let sheetIndex = 0\n\n for await (const worksheet of reader) {\n sheetIndex += 1\n const isTarget =\n targetSheetIndex === undefined ? sheetIndex === 1 : sheetIndex === targetSheetIndex\n\n if (!isTarget) {\n // Drain a non-target worksheet so the underlying zip stream advances to the\n // next; exceljs's streaming reader stalls if a worksheet is left unconsumed.\n for await (const _skip of worksheet) {\n void _skip\n }\n continue\n }\n\n let excelRowNum = 0\n let header: string[] | null = null\n for await (const row of worksheet) {\n excelRowNum += 1\n const cells = rowCells(row.values)\n\n if (hasHeader && header === null) {\n // Skip rows before the configured header row, then capture it (trimmed).\n if (excelRowNum < headerRowIndex + 1) continue\n header = cells.map((c) => c.trim())\n assertUniqueHeaders(header)\n continue\n }\n\n dataRowNumber += 1\n const rec: Record<string, string> = {}\n if (header) {\n for (let i = 0; i < header.length; i += 1) {\n const key = header[i]\n if (key) rec[key] = cells[i] ?? ''\n }\n } else {\n for (let i = 0; i < cells.length; i += 1) rec[String(i)] = cells[i] ?? ''\n }\n yield { rowNumber: dataRowNumber, row: rec }\n }\n\n // Target worksheet fully read — done (remaining sheets are not consumed).\n return\n }\n\n // Reached only when a specific sheet was requested but never matched — the\n // first-sheet default always returns inside the loop on a non-empty workbook.\n if (targetSheetIndex !== undefined) {\n throw new Error(`xlsx worksheet ${targetSheetIndex} not found`)\n }\n}\n"],"mappings":"0CAgCA,SAAS,EAAa,EAAsB,CAC1C,GAAI,GAAM,KAAyB,MAAO,GAC1C,GAAI,OAAO,GAAM,SAAU,OAAO,EAClC,GAAI,OAAO,GAAM,UAAY,OAAO,GAAM,UAAW,OAAO,OAAO,CAAC,EACpE,GAAI,aAAa,KAAM,OAAO,EAAE,YAAY,EAE5C,GAAI,aAAc,GAAK,MAAM,QAAQ,EAAE,QAAQ,EAC7C,OAAO,EAAE,SAAS,IAAK,GAAM,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,EAE9C,GAAI,SAAU,GAAK,OAAO,EAAE,MAAS,SAAU,OAAO,EAAE,KACxD,GAAI,WAAY,GAAK,EAAE,SAAW,IAAA,IAAa,EAAE,SAAW,KAAM,CAChE,IAAM,EAAI,EAAE,OAGZ,OAFI,aAAa,KAAa,EAAE,YAAY,EAErC,OAAO,GAAM,SAAW,GAAK,OAAO,CAAC,CAC9C,CACA,MAAO,EACT,CAGA,SAAS,EAAS,EAA8D,CAC9E,IAAM,EAAgB,CAAC,EACvB,GAAI,MAAM,QAAQ,CAAM,EAEtB,IAAK,IAAI,EAAI,EAAG,EAAI,EAAO,OAAQ,GAAK,EAAG,EAAI,EAAI,GAAK,EAAa,EAAO,EAAE,OAE9E,IAAK,GAAM,CAAC,EAAG,KAAM,OAAO,QAAQ,CAAM,EAAG,CAC3C,IAAM,EAAM,OAAO,SAAS,EAAG,EAAE,EAC7B,CAAC,OAAO,MAAM,CAAG,GAAK,GAAO,IAAG,EAAI,EAAM,GAAK,EAAa,CAAC,EACnE,CAIF,IAAK,IAAI,EAAI,EAAG,EAAI,EAAI,OAAQ,GAAK,EAAO,EAAI,KAAO,IAAA,KAAW,EAAI,GAAK,IAC3E,OAAO,CACT,CAQA,eAAuB,EAAW,EAAwD,CACxF,GAAM,CAAE,WAAU,YAAY,GAAM,iBAAiB,EAAG,SAAU,EAE5D,EAAS,IAAI,MADG,OAAO,YAAA,CACF,OAAO,KAAK,eAAe,EAAU,CAC9D,WAAY,OACZ,cAAe,QACf,OAAQ,SACR,WAAY,SACZ,QAAS,QACX,CAAC,EAKG,EACJ,GAAI,IAAU,IAAA,KACZ,EAAmB,OAAO,SAAS,EAAO,EAAE,EACxC,OAAO,MAAM,CAAgB,GAAK,EAAmB,GACvD,MAAU,MAAM,uBAAuB,EAAM,uCAAuC,EAGxF,IAAI,EAAgB,EAChB,EAAa,EAEjB,UAAW,IAAM,KAAa,EAAQ,CAKpC,GAJA,GAAc,EAIV,EAFF,IAAqB,IAAA,GAAY,IAAe,EAAI,IAAe,GAEtD,CAGb,UAAW,IAAM,KAAS,GAG1B,QACF,CAEA,IAAI,EAAc,EACd,EAA0B,KAC9B,UAAW,IAAM,KAAO,EAAW,CACjC,GAAe,EACf,IAAM,EAAQ,EAAS,EAAI,MAAM,EAEjC,GAAI,GAAa,IAAW,KAAM,CAEhC,GAAI,EAAc,EAAiB,EAAG,SACtC,EAAS,EAAM,IAAK,GAAM,EAAE,KAAK,CAAC,EAClC,EAAoB,CAAM,EAC1B,QACF,CAEA,GAAiB,EACjB,IAAM,EAA8B,CAAC,EACrC,GAAI,EACF,IAAK,IAAI,EAAI,EAAG,EAAI,EAAO,OAAQ,GAAK,EAAG,CACzC,IAAM,EAAM,EAAO,GACf,IAAK,EAAI,GAAO,EAAM,IAAM,GAClC,MAEA,IAAK,IAAI,EAAI,EAAG,EAAI,EAAM,OAAQ,GAAK,EAAG,EAAI,OAAO,CAAC,GAAK,EAAM,IAAM,GAEzE,KAAM,CAAE,UAAW,EAAe,IAAK,CAAI,CAC7C,CAGA,MACF,CAIA,GAAI,IAAqB,IAAA,GACvB,MAAU,MAAM,kBAAkB,EAAiB,WAAW,CAElE"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@murumets-ee/imports",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.35.0",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -45,13 +45,13 @@
|
|
|
45
45
|
"drizzle-orm": "^0.45.2",
|
|
46
46
|
"exceljs": "^4.4.0",
|
|
47
47
|
"zod": "^3.24.1",
|
|
48
|
-
"@murumets-ee/core": "0.
|
|
49
|
-
"@murumets-ee/db": "0.
|
|
50
|
-
"@murumets-ee/entity": "0.
|
|
51
|
-
"@murumets-ee/logging": "0.
|
|
52
|
-
"@murumets-ee/queue": "0.
|
|
53
|
-
"@murumets-ee/search-elasticsearch": "0.
|
|
54
|
-
"@murumets-ee/storage": "0.
|
|
48
|
+
"@murumets-ee/core": "0.35.0",
|
|
49
|
+
"@murumets-ee/db": "0.35.0",
|
|
50
|
+
"@murumets-ee/entity": "0.35.0",
|
|
51
|
+
"@murumets-ee/logging": "0.35.0",
|
|
52
|
+
"@murumets-ee/queue": "0.35.0",
|
|
53
|
+
"@murumets-ee/search-elasticsearch": "0.35.0",
|
|
54
|
+
"@murumets-ee/storage": "0.35.0"
|
|
55
55
|
},
|
|
56
56
|
"devDependencies": {
|
|
57
57
|
"@types/node": "^20.19.42",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"parts-schema-Vv0EJCjj.mjs","names":[],"sources":["../src/mapping/number-format.ts","../src/mapping/column-transforms.ts","../src/mapping/engine.ts","../src/mapping/target-schema.ts","../src/mapping/parts-schema.ts"],"sourcesContent":["/**\n * Number parsing for the configurable mapping engine (commerce-demo phase 02).\n *\n * Real supplier feeds (R03) disagree on decimal AND thousands separators:\n * `55.02` (dot decimal), `326,88` (comma decimal), `1.234,56` (EU\n * thousands+decimal), `1,234.56` (US), `1 234,56` (space-grouped). The mapping\n * stores an explicit `{ decimal, thousands }` per profile so there is no guessing\n * — `parseNumber` trusts the configured format.\n *\n * Two parsers live here:\n * - `parseNumber(raw, format)` — the format-driven parser used by the generic\n * `number` per-column transform.\n * - `parseEuropeanNumber(raw)` — the carmaker-feed legacy parser (comma decimal,\n * dot thousands, **bare-dot rejected as ambiguous**). Carries the carmaker feed's\n * documented format with two intentional broadenings vs the old inline parser:\n * it trims surrounding whitespace, and returns `null` (not `NaN`) for an empty /\n * `'NA'` cell so both number ops share absent-vs-malformed semantics. The\n * carmaker doc-shape suite (27 tests) is the regression net.\n */\n\n/** Separator config for {@link parseNumber}. Stored per `import_mapping`. */\nexport interface NumberFormat {\n /** Decimal separator. */\n decimal: '.' | ','\n /** Thousands grouping separator. `''` = none. */\n thousands: '' | '.' | ',' | ' '\n}\n\n/** Post-cleanup numeric form: optional sign, digits, optional single decimal. */\nconst STRICT_NUMBER_RE = /^-?\\d+(\\.\\d+)?$/\n\n/**\n * Parse a feed cell into a number using an EXPLICIT `{ decimal, thousands }`\n * format. No ambiguity: thousands separators are stripped, the decimal separator\n * is normalised to `.`, and the result is strict-validated.\n *\n * Returns:\n * - `null` for an empty / `'NA'` cell (a legitimately-absent value — the caller\n * decides whether that's allowed for a given target field).\n * - `NaN` for a non-empty but unparseable cell (malformed — the caller surfaces\n * a row-level error rather than coercing to 0).\n * - the parsed number otherwise.\n *\n * A character that is neither a digit, the configured decimal, the configured\n * thousands separator, nor a leading `-` makes the cell unparseable (`NaN`) —\n * e.g. a literal `.` when the format's decimal is `,` and thousands is not `.`.\n */\nexport function parseNumber(raw: string, format: NumberFormat): number | null {\n const trimmed = raw.trim()\n if (trimmed === '' || trimmed.toUpperCase() === 'NA') return null\n\n const { decimal, thousands } = format\n let s = trimmed\n // 1. Remove every thousands separator (when one is configured).\n if (thousands !== '') s = s.split(thousands).join('')\n\n // 2. Normalise the decimal separator to `.`. After thousands removal, the\n // \"other\" separator must not appear — if it does, the cell doesn't match\n // the declared format and is unparseable.\n if (decimal === ',') {\n if (s.includes('.')) return Number.NaN\n s = s.split(',').join('.')\n } else {\n // decimal === '.'\n if (s.includes(',')) return Number.NaN\n }\n\n if (!STRICT_NUMBER_RE.test(s)) return Number.NaN\n return Number.parseFloat(s)\n}\n\n/**\n * Convert a major-unit money value (e.g. `55.02` EUR) to integer MINOR units\n * (`5502`). Rounds to the nearest cent to absorb float drift. The commerce-demo\n * PartsDocument money fields (`list_price_minor`, `purchase_price_minor`,\n * `core_charge_minor`) are integer minor units so derived prices (PR4) don't\n * accrue rounding error — see F003 in progress.md.\n */\nexport function toMinorUnits(value: number): number {\n return Math.round(value * 100)\n}\n\n/**\n * Parse a European-formatted decimal: `,` is the decimal separator, `.` is the\n * (optional) thousands separator. Returns `null` for an empty / `'NA'` cell and\n * `NaN` on unparseable (malformed) input.\n *\n * This is the carmaker-feed legacy parser, preserved from the hardcoded transform.\n * Crucially it **rejects bare-dot** input (`1.98`, `1.234`) as ambiguous rather\n * than guessing — the carmaker feeds are documented as comma-decimal, and a `.`\n * without a `,` could mean a US-format decimal OR a thousands marker. The\n * format-driven {@link parseNumber} doesn't need this guard (its separators are\n * explicit); the carmaker path keeps it.\n *\n * The `null` (absent) vs `NaN` (malformed) split mirrors {@link parseNumber} so\n * the mapping engine treats both number ops uniformly: a target field's `buildRow`\n * reads `null` as \"not provided\" and `NaN` as \"invalid\". This is what lets the\n * reimplemented carmaker transform keep its `GrossPrice 'NA' → null` behaviour\n * (optional field) while still erroring on a malformed `NetPrice` (required).\n *\n * Cases: `'1,98'`→`1.98`, `'1.234,56'`→`1234.56`, `'5'`→`5`, `'-5,00'`→`-5`,\n * empty / `'NA'` → `null`. Rejected (`NaN`): bare-dot (`'1.98'`, `'1.234'`),\n * trailing junk (`'1,98EUR'`), multiple commas (`'1,2,3'`), other non-numeric.\n */\nexport function parseEuropeanNumber(s: string): number | null {\n const trimmed = s.trim()\n if (trimmed === '' || trimmed.toUpperCase() === 'NA') return null\n const hasComma = trimmed.includes(',')\n const hasDot = trimmed.includes('.')\n let cleaned: string\n if (hasComma && hasDot) {\n // European thousands+decimal: strip every `.`, then swap the (sole) `,` for\n // `.`. The strict regex below catches multi-comma input.\n cleaned = trimmed.replace(/\\./g, '').replace(',', '.')\n } else if (hasComma) {\n cleaned = trimmed.replace(',', '.')\n } else if (hasDot) {\n return Number.NaN\n } else {\n cleaned = trimmed\n }\n if (!STRICT_NUMBER_RE.test(cleaned)) return Number.NaN\n return Number.parseFloat(cleaned)\n}\n","/**\n * Per-column transforms for the mapping engine (commerce-demo phase 02, D005).\n *\n * A **bounded, enum-keyed** library of cleaning operations applied to a source\n * cell before it lands on a target field. This is the security model from R02:\n * transforms are declarative operations with Zod-validated params, NEVER\n * user-supplied executable code. The carmaker feed's logic decomposes exactly\n * into `stripPrefix` + `europeanNumber` (+ the schema's code-normalize).\n *\n * Number-producing ops (`number`, `europeanNumber`) are TERMINAL by convention —\n * they convert the string cell into `number | null`; string ops after them are\n * no-ops. The engine reads the result as `string | number | null` and hands it to\n * the target schema's `buildRow`.\n */\n\nimport { z } from 'zod'\nimport { type NumberFormat, parseEuropeanNumber, parseNumber } from './number-format.js'\n\n/** Max length for a strip-affix value — bounds the param, no ReDoS surface. */\nconst MAX_AFFIX_LEN = 32\n\n/**\n * Discriminated union of bounded transform ops. The TS type is INFERRED from\n * this schema so there's a single source of truth + the snapshot is validated at\n * the engine boundary.\n */\nexport const columnTransformSchema = z.discriminatedUnion('op', [\n z.object({ op: z.literal('trim') }),\n z.object({ op: z.literal('uppercase') }),\n z.object({ op: z.literal('lowercase') }),\n z.object({ op: z.literal('stripPrefix'), value: z.string().min(1).max(MAX_AFFIX_LEN) }),\n z.object({ op: z.literal('stripSuffix'), value: z.string().min(1).max(MAX_AFFIX_LEN) }),\n z.object({ op: z.literal('defaultIfEmpty'), value: z.string().max(255) }),\n z.object({ op: z.literal('number') }),\n z.object({ op: z.literal('europeanNumber') }),\n])\n\n/** One declarative cleaning operation. */\nexport type ColumnTransform = z.infer<typeof columnTransformSchema>\n\n/** The discriminator literal of a {@link ColumnTransform}. */\nexport type ColumnTransformOp = ColumnTransform['op']\n\n/** UI descriptor for one transform op: its key, a label, and whether it takes a `value` param. */\nexport interface ColumnTransformOpDescriptor {\n op: ColumnTransformOp\n label: string\n /** `true` → the op carries a `value` string param (`stripPrefix`/`stripSuffix`/`defaultIfEmpty`). */\n hasValue: boolean\n}\n\n/**\n * Static catalogue of the available per-column transform ops, for the PR3 wizard's\n * transform picker. The ORDER is the suggested UI order; `number`/`europeanNumber`\n * are terminal (see {@link applyColumnTransforms}) and labelled accordingly.\n */\nexport const COLUMN_TRANSFORM_OPS: readonly ColumnTransformOpDescriptor[] = [\n { op: 'trim', label: 'Trim whitespace', hasValue: false },\n { op: 'uppercase', label: 'Uppercase', hasValue: false },\n { op: 'lowercase', label: 'Lowercase', hasValue: false },\n { op: 'stripPrefix', label: 'Strip prefix', hasValue: true },\n { op: 'stripSuffix', label: 'Strip suffix', hasValue: true },\n { op: 'defaultIfEmpty', label: 'Default if empty', hasValue: true },\n { op: 'number', label: 'Parse number (uses number format)', hasValue: false },\n { op: 'europeanNumber', label: 'Parse European number (1.234,56)', hasValue: false },\n] as const\n\n/** The value a column carries as it moves through the transform pipeline. */\nexport type CellValue = string | number | null\n\n/**\n * Apply a transform pipeline to a raw cell, left-to-right. String ops act on the\n * string value; the `number` / `europeanNumber` ops are terminal (convert to\n * `number | null`, with `NaN` for a malformed numeric — the schema's `buildRow`\n * surfaces that as a row error). Once the value is non-string, later string ops\n * pass through unchanged.\n */\nexport function applyColumnTransforms(\n raw: string,\n transforms: readonly ColumnTransform[],\n format: NumberFormat,\n): CellValue {\n let value: CellValue = raw\n for (const t of transforms) {\n if (typeof value !== 'string') {\n // A prior number op already produced number|null — string ops are no-ops\n // past this point (number ops are terminal by convention).\n continue\n }\n switch (t.op) {\n case 'trim':\n value = value.trim()\n break\n case 'uppercase':\n value = value.toUpperCase()\n break\n case 'lowercase':\n value = value.toLowerCase()\n break\n case 'stripPrefix':\n value = value.startsWith(t.value) ? value.slice(t.value.length) : value\n break\n case 'stripSuffix':\n value = value.endsWith(t.value) ? value.slice(0, value.length - t.value.length) : value\n break\n case 'defaultIfEmpty':\n value = value.trim() === '' ? t.value : value\n break\n case 'number':\n value = parseNumber(value, format)\n break\n case 'europeanNumber':\n value = parseEuropeanNumber(value)\n break\n }\n }\n return value\n}\n","/**\n * Mapping engine (commerce-demo phase 02, D005).\n *\n * Turns a declarative `MappingConfig` (which source column → which whitelisted\n * target field, plus per-column transforms + a number format) into a\n * `RowTransform` the existing runner drives. The pipeline per row is:\n *\n * raw cells → applyColumnTransforms(per column) → MappedRow → schema.buildRow\n *\n * Two whitelists guard the boundary (R02 injection model): the `target_schema`\n * name resolves to a registered `TargetSchema`, and every `column_map[].target`\n * must be one of that schema's fields — enforced here at build time (throws),\n * before any row is processed.\n *\n * `resolveMapping` implements the D002 most-specific ladder used at enqueue time\n * to pick the `(supplier, brand)` mapping over the `(supplier, NULL)` default.\n */\n\nimport { z } from 'zod'\nimport type { RowTransform } from '../transform.js'\nimport { applyColumnTransforms, columnTransformSchema } from './column-transforms.js'\nimport type { MappedRow, TargetSchema } from './target-schema.js'\n\n/** One column-map entry. `target` is checked against the schema whitelist separately. */\nexport const mappingColumnSchema = z.object({\n /** Source header name (or positional index string for headerless feeds). */\n source: z.string().min(1).max(256),\n /** Target field key — must be in the chosen schema's whitelist (enforced in `buildMappingTransform`). */\n target: z.string().min(1).max(64),\n /** Ordered per-column cleaning ops. Defaults to none. */\n transforms: z.array(columnTransformSchema).default([]),\n})\nexport type MappingColumn = z.infer<typeof mappingColumnSchema>\n\n/**\n * Decimal + thousands separators for the `number` transform.\n *\n * The two MUST differ: `parseNumber` strips every thousands separator FIRST, so a\n * config where `decimal === thousands` (e.g. both `,`) would strip the decimal\n * separator too — `326,88` → `32688`, a silent 100× error rather than a row error.\n * Rejected at save (`assertValidMapping`) AND at run (`mappingConfigSchema`).\n */\nexport const numberFormatSchema = z\n .object({\n decimal: z.enum(['.', ',']),\n thousands: z.enum(['', '.', ',', ' ']),\n })\n .refine((f) => f.decimal !== f.thousands, {\n message: 'decimal and thousands separators must differ',\n path: ['thousands'],\n })\n\n/**\n * The portable mapping config — the subset of an `import_mapping` row the engine\n * needs. The enqueuer SNAPSHOTS this into `import_run.params.mapping` so a run\n * records the exact mapping it used (reproducible) and the transform stays a pure\n * function of `(row, ctx)` with no per-row DB I/O.\n */\nexport const mappingConfigSchema = z.object({\n targetSchema: z.string().min(1).max(64),\n numberFormat: numberFormatSchema,\n columnMap: z.array(mappingColumnSchema),\n})\nexport type MappingConfig = z.infer<typeof mappingConfigSchema>\n\n/**\n * Build a `RowTransform` from a validated mapping config + its target schema.\n *\n * Throws at BUILD time if any `column_map` target is outside the schema's\n * whitelist (deny-by-default, R02) — so a misconfigured mapping fails loudly\n * before a single row is written, rather than silently dropping a column.\n *\n * Generic over the schema's output type, so passing the concrete `partsSchema`\n * yields a `RowTransform<PartsDocument>` (the carmaker path keeps its typing); the\n * registry-resolved `TargetSchema<unknown>` yields `RowTransform<unknown>` (the\n * `commerce:mapping` path, where the output type isn't known until runtime).\n */\nexport function buildMappingTransform<TOut>(\n config: MappingConfig,\n schema: TargetSchema<TOut>,\n): RowTransform<TOut> {\n const allowed = new Set(schema.fields.map((f) => f.key))\n for (const col of config.columnMap) {\n if (!allowed.has(col.target)) {\n throw new Error(\n `Mapping target \"${col.target}\" is not a field of schema \"${schema.name}\" ` +\n `(allowed: ${[...allowed].join(', ')})`,\n )\n }\n }\n\n const format = config.numberFormat\n const columnMap = config.columnMap\n\n return async (row, ctx) => {\n const mapped: MappedRow = {}\n for (const col of columnMap) {\n // `source` is read as data from the parsed row map — never used as a key\n // into framework objects (R02). A missing source column yields `''`.\n const raw = row[col.source] ?? ''\n mapped[col.target] = applyColumnTransforms(raw, col.transforms, format)\n }\n return schema.buildRow(mapped, {\n importRunId: ctx.importRunId,\n rowNumber: ctx.rowNumber,\n params: ctx.params,\n })\n }\n}\n\n/** The minimal shape `resolveMapping` needs — satisfied by an `import_mapping` row or a test fixture. */\nexport interface ResolvableMapping {\n /** `null` = supplier default (applies to all brands). */\n brandId: string | null\n active: boolean\n}\n\n/**\n * Most-specific resolution (D002 ladder): given the candidate mappings for one\n * `(supplier, target_schema)` and the requested brand, return the brand-specific\n * mapping if one exists, else the supplier-default (`brandId === null`). Inactive\n * mappings are skipped. Returns `undefined` when nothing matches.\n */\nexport function resolveMapping<M extends ResolvableMapping>(\n mappings: readonly M[],\n brandId: string | null,\n): M | undefined {\n const active = mappings.filter((m) => m.active)\n if (brandId !== null) {\n const specific = active.find((m) => m.brandId === brandId)\n if (specific) return specific\n }\n return active.find((m) => m.brandId === null)\n}\n","/**\n * TargetSchema — the contract a mapping writes INTO (commerce-demo phase 02, D005).\n *\n * A mapping's `column_map` points source columns at a schema's whitelisted target\n * fields; the engine applies the per-column transforms to build a `MappedRow`, then\n * hands it to `schema.buildRow` to produce the typed output document (an ES\n * `PartsDocument` for `parts`, a discount row for `discount_table`).\n *\n * The `fields` list is the SECOND injection-boundary whitelist (the first is the\n * registered schema name): a `column_map` entry whose `target` is not a field key\n * here is rejected at validation time (R02). Two schemas ship in PR2 — `parts` and\n * `discount_table`.\n */\n\nimport type { RowResult } from '../transform.js'\nimport type { CellValue } from './column-transforms.js'\n\n/** One whitelisted target field on a schema. */\nexport interface TargetField {\n /** Stable target key referenced by `column_map[].target`. */\n key: string\n /** Human label for the wizard dropdown (PR3). */\n label: string\n /** Whether a mapping for this schema MUST include this target. */\n required: boolean\n /** Coarse type — drives the wizard's transform suggestions; `buildRow` does the real coercion. */\n type: 'string' | 'number'\n}\n\n/** The per-row product of the engine: target field key → cleaned value. */\nexport type MappedRow = Record<string, CellValue>\n\n/** Per-row context handed to `buildRow`. `params` is the opaque per-run params object. */\nexport interface BuildRowContext {\n importRunId: string\n rowNumber: number\n /**\n * Opaque per-run params (the parts schema reads brand/supplier identity from\n * it; `discount_table` ignores it). The schema validates the shape it needs.\n */\n params: Record<string, unknown>\n}\n\n/** A registered target schema: a whitelist + a row builder. */\nexport interface TargetSchema<TOut> {\n /** Registered name — also the `import_mapping.target_schema` value. */\n name: string\n /** The target-field whitelist (injection boundary). */\n fields: readonly TargetField[]\n /** Build a typed output (success) / skip / error from a mapped row. */\n buildRow(mapped: MappedRow, ctx: BuildRowContext): RowResult<TOut>\n}\n\n// ---------------------------------------------------------------------------\n// Shared `buildRow` helpers (used by every concrete schema).\n// ---------------------------------------------------------------------------\n\n/**\n * A nullable text cell → trimmed string, or `null` for empty / non-string. Accepts\n * `undefined` (an UNMAPPED target — indexed access under `noUncheckedIndexedAccess`)\n * and treats it as absent.\n */\nexport function textOrNull(v: CellValue | undefined): string | null {\n if (typeof v !== 'string') return null\n const t = v.trim()\n return t === '' ? null : t\n}\n\n/** The three outcomes a `number` / `europeanNumber` op produces for a cell. */\nexport type NumberOutcome =\n | { kind: 'value'; value: number }\n | { kind: 'absent' }\n | { kind: 'invalid' }\n\n/**\n * Classify a cell that should hold a number. `undefined` (unmapped) / `null` →\n * absent; a real number → value (`NaN` → invalid); a non-empty raw string (no\n * number op declared) → invalid; a blank raw string → absent.\n */\nexport function numberCell(v: CellValue | undefined): NumberOutcome {\n if (v === undefined || v === null) return { kind: 'absent' }\n if (typeof v === 'number') {\n return Number.isNaN(v) ? { kind: 'invalid' } : { kind: 'value', value: v }\n }\n return v.trim() === '' ? { kind: 'absent' } : { kind: 'invalid' }\n}\n\n/** Build a `RowResult` error variant (assignable to any `TargetSchema<TOut>`). */\nexport function rowError(errorType: string, message: string, field: string) {\n return { kind: 'error' as const, error: { errorType, message, field } }\n}\n","/**\n * `parts` target schema — maps a cleaned feed row into the ES `PartsDocument`\n * (commerce-demo phase 02, D005). This is the SHARED assembly + validation logic\n * the reimplemented carmaker transform and every operator-built `parts` mapping\n * run through, so the row taxonomy (code normalization, currency gate, price\n * validation) is uniform across all supplier feeds.\n *\n * **Field coverage.** This schema populates the direct passthroughs (the existing\n * `PartsDocument` columns plus `list_price_minor`, `discount_group`,\n * `superseding_code`) AND the DERIVED `purchase_price_minor` (PR4 — direct net, or\n * two-file `list × (1 − groupDiscount)` resolved from `params.discountGroups`). All\n * `core_charge_*` fields stay null until PR7; `core_charge` is deliberately NOT in\n * the whitelist (deny-by-default — PR7 adds it with its `*S`-row + column ingestion).\n * Replacement-pair SIDE EFFECTS (separate entity) are PR6.\n *\n * **Money.** Every money field is integer minor units via `toMinorUnits` (F003), and\n * the two-file derivation stays in minor units end-to-end (no major↔minor\n * round-trip). The legacy `net_price_eur` stays a major-unit float for back-compat:\n * it holds the direct net price, or — for a two-file LIST feed with no direct net —\n * the list price as a display placeholder (the true cost is `purchase_price_minor`).\n *\n * **Cost-pending warning.** A list-only row whose `discount_group` doesn't resolve\n * against `params.discountGroups` is still indexed (list set, `purchase_price_minor`\n * null) and carries a `no_purchase_price` WARNING — not a row error (phase 03 §5).\n */\n\nimport type { PartsDocument } from '@murumets-ee/search-elasticsearch'\nimport { z } from 'zod'\nimport type { RowWarning } from '../transform.js'\nimport { toMinorUnits } from './number-format.js'\nimport {\n type BuildRowContext,\n type MappedRow,\n numberCell,\n rowError,\n type TargetField,\n type TargetSchema,\n textOrNull,\n} from './target-schema.js'\n\n/**\n * Params the `parts` schema reads from the run — the per-(supplier, brand)\n * identity resolved at enqueue time. Identical to the carmaker transform's params\n * (the carmaker path re-exports this), so brand/supplier injection + the optional\n * supplier code-prefix strip are validated the same way everywhere.\n */\nexport const partsParamsSchema = z.object({\n brandId: z.string().uuid(),\n brandSlug: z.string().min(1).max(64),\n supplierId: z.string().uuid(),\n supplierDisplayName: z.string().min(1).max(255),\n importBatchId: z.string().uuid(),\n /** Supplier code prefix to strip before normalization (e.g. `'ME-'`). Optional. */\n supplierCodePrefix: z\n .string()\n .max(8)\n .regex(/^[A-Z0-9-]+$/, 'must be uppercase alphanumeric or \"-\"')\n .optional(),\n /**\n * Purchase-discount map for this supplier (commerce-demo PR4, phase 03 §2):\n * `{ groupCode → discountPct }`, resolved from `supplier_discount_group` at\n * enqueue time and snapshotted onto the run so `buildRow` stays a pure function\n * (no per-row DB I/O; `@murumets-ee/imports` stays a leaf). Absent for direct-net\n * suppliers + the carmaker path. Percentages are 0–100 (the entity constrains them).\n */\n discountGroups: z.record(z.string(), z.number().min(0).max(100)).optional(),\n})\nexport type PartsParams = z.infer<typeof partsParamsSchema>\n\n/** Per-run memoization of the parsed params (keyed on the `ctx.params` reference). */\nconst paramsCache = new WeakMap<object, PartsParams>()\n\nfunction resolveParams(raw: Record<string, unknown>): PartsParams {\n const cached = paramsCache.get(raw)\n if (cached) return cached\n const parsed = partsParamsSchema.parse(raw)\n paramsCache.set(raw, parsed)\n return parsed\n}\n\n/** Per-locale display-name target fields (8 locales). */\nconst NAME_LOCALES = ['de', 'en', 'es', 'fr', 'it', 'nl', 'pt', 'ro'] as const\ntype NameLocale = (typeof NAME_LOCALES)[number]\nconst nameKey = (l: NameLocale): `name_${NameLocale}` => `name_${l}`\n\n/** The `parts` target-field whitelist (PR2 subset — `core_charge`/`purchase_price` are later PRs). */\nconst PARTS_FIELDS: readonly TargetField[] = [\n { key: 'code', label: 'Code', required: true, type: 'string' },\n { key: 'net_price', label: 'Net price', required: false, type: 'number' },\n { key: 'gross_price', label: 'Gross price', required: false, type: 'number' },\n { key: 'list_price', label: 'List price', required: false, type: 'number' },\n { key: 'currency', label: 'Currency', required: false, type: 'string' },\n { key: 'barcode', label: 'Barcode', required: false, type: 'string' },\n { key: 'description1', label: 'Description 1', required: false, type: 'string' },\n { key: 'description2', label: 'Description 2', required: false, type: 'string' },\n { key: 'discount_group', label: 'Discount group', required: false, type: 'string' },\n { key: 'superseding_code', label: 'Superseding code', required: false, type: 'string' },\n ...NAME_LOCALES.map(\n (l): TargetField => ({\n key: nameKey(l),\n label: `Name (${l})`,\n required: false,\n type: 'string',\n }),\n ),\n] as const\n\nconst NORMALIZE_CODE_RE = /[\\s-]/g\n/** Uppercase + dash/whitespace-stripped (D2). `*`/other chars are preserved (e.g. `*S` markers). */\nfunction normalizeCode(s: string): string {\n return s.toUpperCase().replace(NORMALIZE_CODE_RE, '')\n}\n\n/**\n * Two-file purchase price (phase 03 §2): `list × (1 − discountPct/100)`, computed\n * ENTIRELY in integer minor units so no major↔minor round-trip can introduce drift\n * (F003). `listMinor` is already minor units; the result is rounded to the nearest\n * minor unit. Worked (R03 A+B): `5502 × 0.74 = 4071.48 → 4071` (€40.71).\n */\nfunction deriveTwoFilePurchaseMinor(listMinor: number, discountPct: number): number {\n return Math.round(listMinor * (1 - discountPct / 100))\n}\n\n/**\n * The `parts` schema. `buildRow` THROWS on invalid run params (so a misconfigured\n * run surfaces as a transform error, matching the carmaker contract) and returns a\n * typed `RowResult` per row otherwise.\n */\nexport const partsSchema: TargetSchema<PartsDocument> = {\n name: 'parts',\n fields: PARTS_FIELDS,\n buildRow(mapped: MappedRow, ctx: BuildRowContext) {\n const params = resolveParams(ctx.params)\n\n // 1. Code (required) → raw `code` + normalized `code_normalized` + doc id.\n const codeRaw = typeof mapped.code === 'string' ? mapped.code.trim() : ''\n if (!codeRaw) return rowError('missing_code', 'code is empty', 'code')\n const stripped =\n params.supplierCodePrefix && codeRaw.startsWith(params.supplierCodePrefix)\n ? codeRaw.slice(params.supplierCodePrefix.length)\n : codeRaw\n const codeNormalized = normalizeCode(stripped)\n if (!codeNormalized) {\n return rowError('empty_normalized_code', `Code \"${codeRaw}\" is empty after normalization`, 'code')\n }\n\n // 2. Currency gate (D9 EUR-only). Unmapped/blank defaults to EUR.\n const currencyText = textOrNull(mapped.currency)\n const currency = currencyText ?? 'EUR'\n if (currency !== 'EUR') {\n return rowError('unsupported_currency', `Currency \"${currency}\" not supported (EUR-only)`, 'currency')\n }\n\n // 3. Prices. net/gross/list each: absent → null, invalid → error, value → number.\n const net = numberCell(mapped.net_price)\n if (net.kind === 'invalid') return rowError('invalid_net_price', 'net_price is not a number', 'net_price')\n const gross = numberCell(mapped.gross_price)\n if (gross.kind === 'invalid') {\n return rowError('invalid_gross_price', 'gross_price is not a number', 'gross_price')\n }\n const list = numberCell(mapped.list_price)\n if (list.kind === 'invalid') return rowError('invalid_list_price', 'list_price is not a number', 'list_price')\n\n const netValue = net.kind === 'value' ? net.value : null\n const listValue = list.kind === 'value' ? list.value : null\n // At least one price must resolve. `net_price_eur` is non-nullable on the doc,\n // so it takes the direct net or — for a list-only two-file feed — the list\n // price as a display placeholder (the true cost is `purchase_price_minor`).\n const displayPrice = netValue ?? listValue\n if (displayPrice === null) {\n return rowError('missing_price', 'no net_price or list_price provided', 'price')\n }\n\n // 4. Purchase-price derivation (phase 03 §2, PR4). All money stays in integer\n // minor units (F003 — never round-trip back to major).\n // - direct net column (R03 C/D) → purchase = net\n // - two-file list × (1 − groupDiscount) → purchase = round(list × (1 − d/100))\n // - neither resolvable → cost pending: index list-only + WARN\n // (a `no_purchase_price` warning, NOT a row error — the part is still indexed).\n const discountGroup = textOrNull(mapped.discount_group)\n const listMinor = listValue === null ? null : toMinorUnits(listValue)\n let purchasePriceMinor: number | null = null\n const warnings: RowWarning[] = []\n if (netValue !== null) {\n purchasePriceMinor = toMinorUnits(netValue)\n } else if (listMinor !== null) {\n // List-only row: resolve the supplier's discount group → derive cost.\n const discountPct = discountGroup !== null ? params.discountGroups?.[discountGroup] : undefined\n if (discountPct !== undefined) {\n purchasePriceMinor = deriveTwoFilePurchaseMinor(listMinor, discountPct)\n } else {\n warnings.push({\n warningType: 'no_purchase_price',\n message:\n discountGroup === null\n ? 'list price present but no discount group on the row — cost pending'\n : `no discount group \"${discountGroup}\" configured for this supplier — cost pending`,\n field: 'discount_group',\n })\n }\n }\n\n const docId = `${codeNormalized}__${params.supplierId}`\n const doc: PartsDocument = {\n doc_id: docId,\n code: codeRaw,\n code_normalized: codeNormalized,\n brand_id: params.brandId,\n brand_slug: params.brandSlug,\n supplier_id: params.supplierId,\n supplier_display_name: params.supplierDisplayName,\n net_price_eur: displayPrice,\n gross_price_eur: gross.kind === 'value' ? gross.value : null,\n currency: 'EUR',\n barcode: textOrNull(mapped.barcode),\n name_de: textOrNull(mapped.name_de),\n name_en: textOrNull(mapped.name_en),\n name_es: textOrNull(mapped.name_es),\n name_fr: textOrNull(mapped.name_fr),\n name_it: textOrNull(mapped.name_it),\n name_nl: textOrNull(mapped.name_nl),\n name_pt: textOrNull(mapped.name_pt),\n name_ro: textOrNull(mapped.name_ro),\n description1: textOrNull(mapped.description1),\n description2: textOrNull(mapped.description2),\n // commerce-demo: list price → minor units; group + code passthroughs.\n list_price_minor: listMinor,\n discount_group: discountGroup,\n superseding_code: textOrNull(mapped.superseding_code),\n // Derived purchase cost (PR4 — direct net or two-file list × (1 − d)); null = cost pending.\n purchase_price_minor: purchasePriceMinor,\n // core_charge fields stay null until PR7.\n core_charge_minor: null,\n core_charge_currency: null,\n core_charge_refundable: null,\n core_charge_tax_class: null,\n core_charge_tax_inclusive: null,\n import_batch_id: params.importBatchId,\n imported_at: new Date().toISOString(),\n }\n\n return warnings.length > 0\n ? { kind: 'success', id: docId, doc, warnings }\n : { kind: 'success', id: docId, doc }\n },\n}\n"],"mappings":"wBA6BA,MAAM,EAAmB,kBAkBzB,SAAgB,EAAY,EAAa,EAAqC,CAC5E,IAAM,EAAU,EAAI,KAAK,EACzB,GAAI,IAAY,IAAM,EAAQ,YAAY,IAAM,KAAM,OAAO,KAE7D,GAAM,CAAE,UAAS,aAAc,EAC3B,EAAI,EAOR,GALI,IAAc,KAAI,EAAI,EAAE,MAAM,CAAS,CAAC,CAAC,KAAK,EAAE,GAKhD,IAAY,IAAK,CACnB,GAAI,EAAE,SAAS,GAAG,EAAG,MAAO,KAC5B,EAAI,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,CAC3B,MAEE,GAAI,EAAE,SAAS,GAAG,EAAG,MAAO,KAI9B,OADK,EAAiB,KAAK,CAAC,EACrB,OAAO,WAAW,CAAC,EADY,GAExC,CASA,SAAgB,EAAa,EAAuB,CAClD,OAAO,KAAK,MAAM,EAAQ,GAAG,CAC/B,CAwBA,SAAgB,EAAoB,EAA0B,CAC5D,IAAM,EAAU,EAAE,KAAK,EACvB,GAAI,IAAY,IAAM,EAAQ,YAAY,IAAM,KAAM,OAAO,KAC7D,IAAM,EAAW,EAAQ,SAAS,GAAG,EAC/B,EAAS,EAAQ,SAAS,GAAG,EAC/B,EACJ,GAAI,GAAY,EAGd,EAAU,EAAQ,QAAQ,MAAO,EAAE,CAAC,CAAC,QAAQ,IAAK,GAAG,OAChD,GAAI,EACT,EAAU,EAAQ,QAAQ,IAAK,GAAG,OAC7B,GAAI,EACT,MAAO,SAEP,GAAU,EAGZ,OADK,EAAiB,KAAK,CAAO,EAC3B,OAAO,WAAW,CAAO,EADY,GAE9C,CCxGA,MAOa,EAAwB,EAAE,mBAAmB,KAAM,CAC9D,EAAE,OAAO,CAAE,GAAI,EAAE,QAAQ,MAAM,CAAE,CAAC,EAClC,EAAE,OAAO,CAAE,GAAI,EAAE,QAAQ,WAAW,CAAE,CAAC,EACvC,EAAE,OAAO,CAAE,GAAI,EAAE,QAAQ,WAAW,CAAE,CAAC,EACvC,EAAE,OAAO,CAAE,GAAI,EAAE,QAAQ,aAAa,EAAG,MAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAa,CAAE,CAAC,EACtF,EAAE,OAAO,CAAE,GAAI,EAAE,QAAQ,aAAa,EAAG,MAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAa,CAAE,CAAC,EACtF,EAAE,OAAO,CAAE,GAAI,EAAE,QAAQ,gBAAgB,EAAG,MAAO,EAAE,OAAO,CAAC,CAAC,IAAI,GAAG,CAAE,CAAC,EACxE,EAAE,OAAO,CAAE,GAAI,EAAE,QAAQ,QAAQ,CAAE,CAAC,EACpC,EAAE,OAAO,CAAE,GAAI,EAAE,QAAQ,gBAAgB,CAAE,CAAC,CAC9C,CAAC,EAqBY,EAA+D,CAC1E,CAAE,GAAI,OAAQ,MAAO,kBAAmB,SAAU,EAAM,EACxD,CAAE,GAAI,YAAa,MAAO,YAAa,SAAU,EAAM,EACvD,CAAE,GAAI,YAAa,MAAO,YAAa,SAAU,EAAM,EACvD,CAAE,GAAI,cAAe,MAAO,eAAgB,SAAU,EAAK,EAC3D,CAAE,GAAI,cAAe,MAAO,eAAgB,SAAU,EAAK,EAC3D,CAAE,GAAI,iBAAkB,MAAO,mBAAoB,SAAU,EAAK,EAClE,CAAE,GAAI,SAAU,MAAO,oCAAqC,SAAU,EAAM,EAC5E,CAAE,GAAI,iBAAkB,MAAO,mCAAoC,SAAU,EAAM,CACrF,EAYA,SAAgB,EACd,EACA,EACA,EACW,CACX,IAAI,EAAmB,EACvB,IAAK,IAAM,KAAK,EACV,UAAO,GAAU,SAKrB,OAAQ,EAAE,GAAV,CACE,IAAK,OACH,EAAQ,EAAM,KAAK,EACnB,MACF,IAAK,YACH,EAAQ,EAAM,YAAY,EAC1B,MACF,IAAK,YACH,EAAQ,EAAM,YAAY,EAC1B,MACF,IAAK,cACH,EAAQ,EAAM,WAAW,EAAE,KAAK,EAAI,EAAM,MAAM,EAAE,MAAM,MAAM,EAAI,EAClE,MACF,IAAK,cACH,EAAQ,EAAM,SAAS,EAAE,KAAK,EAAI,EAAM,MAAM,EAAG,EAAM,OAAS,EAAE,MAAM,MAAM,EAAI,EAClF,MACF,IAAK,iBACH,EAAQ,EAAM,KAAK,IAAM,GAAK,EAAE,MAAQ,EACxC,MACF,IAAK,SACH,EAAQ,EAAY,EAAO,CAAM,EACjC,MACF,IAAK,iBACH,EAAQ,EAAoB,CAAK,EACjC,KACJ,CAEF,OAAO,CACT,CC7FA,MAAa,EAAsB,EAAE,OAAO,CAE1C,OAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,EAEjC,OAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAEhC,WAAY,EAAE,MAAM,CAAqB,CAAC,CAAC,QAAQ,CAAC,CAAC,CACvD,CAAC,EAWY,EAAqB,EAC/B,OAAO,CACN,QAAS,EAAE,KAAK,CAAC,IAAK,GAAG,CAAC,EAC1B,UAAW,EAAE,KAAK,CAAC,GAAI,IAAK,IAAK,GAAG,CAAC,CACvC,CAAC,CAAC,CACD,OAAQ,GAAM,EAAE,UAAY,EAAE,UAAW,CACxC,QAAS,+CACT,KAAM,CAAC,WAAW,CACpB,CAAC,EAQU,EAAsB,EAAE,OAAO,CAC1C,aAAc,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EACtC,aAAc,EACd,UAAW,EAAE,MAAM,CAAmB,CACxC,CAAC,EAeD,SAAgB,EACd,EACA,EACoB,CACpB,IAAM,EAAU,IAAI,IAAI,EAAO,OAAO,IAAK,GAAM,EAAE,GAAG,CAAC,EACvD,IAAK,IAAM,KAAO,EAAO,UACvB,GAAI,CAAC,EAAQ,IAAI,EAAI,MAAM,EACzB,MAAU,MACR,mBAAmB,EAAI,OAAO,8BAA8B,EAAO,KAAK,cACzD,CAAC,GAAG,CAAO,CAAC,CAAC,KAAK,IAAI,EAAE,EACzC,EAIJ,IAAM,EAAS,EAAO,aAChB,EAAY,EAAO,UAEzB,OAAO,MAAO,EAAK,IAAQ,CACzB,IAAM,EAAoB,CAAC,EAC3B,IAAK,IAAM,KAAO,EAAW,CAG3B,IAAM,EAAM,EAAI,EAAI,SAAW,GAC/B,EAAO,EAAI,QAAU,EAAsB,EAAK,EAAI,WAAY,CAAM,CACxE,CACA,OAAO,EAAO,SAAS,EAAQ,CAC7B,YAAa,EAAI,YACjB,UAAW,EAAI,UACf,OAAQ,EAAI,MACd,CAAC,CACH,CACF,CAeA,SAAgB,EACd,EACA,EACe,CACf,IAAM,EAAS,EAAS,OAAQ,GAAM,EAAE,MAAM,EAC9C,GAAI,IAAY,KAAM,CACpB,IAAM,EAAW,EAAO,KAAM,GAAM,EAAE,UAAY,CAAO,EACzD,GAAI,EAAU,OAAO,CACvB,CACA,OAAO,EAAO,KAAM,GAAM,EAAE,UAAY,IAAI,CAC9C,CCvEA,SAAgB,EAAW,EAAyC,CAClE,GAAI,OAAO,GAAM,SAAU,OAAO,KAClC,IAAM,EAAI,EAAE,KAAK,EACjB,OAAO,IAAM,GAAK,KAAO,CAC3B,CAaA,SAAgB,EAAW,EAAyC,CAKlE,OAJI,GAAyB,KAAa,CAAE,KAAM,QAAS,EACvD,OAAO,GAAM,SACR,OAAO,MAAM,CAAC,EAAI,CAAE,KAAM,SAAU,EAAI,CAAE,KAAM,QAAS,MAAO,CAAE,EAEpE,EAAE,KAAK,IAAM,GAAK,CAAE,KAAM,QAAS,EAAI,CAAE,KAAM,SAAU,CAClE,CAGA,SAAgB,EAAS,EAAmB,EAAiB,EAAe,CAC1E,MAAO,CAAE,KAAM,QAAkB,MAAO,CAAE,YAAW,UAAS,OAAM,CAAE,CACxE,CC5CA,MAAa,EAAoB,EAAE,OAAO,CACxC,QAAS,EAAE,OAAO,CAAC,CAAC,KAAK,EACzB,UAAW,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EACnC,WAAY,EAAE,OAAO,CAAC,CAAC,KAAK,EAC5B,oBAAqB,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,EAC9C,cAAe,EAAE,OAAO,CAAC,CAAC,KAAK,EAE/B,mBAAoB,EACjB,OAAO,CAAC,CACR,IAAI,CAAC,CAAC,CACN,MAAM,eAAgB,uCAAuC,CAAC,CAC9D,SAAS,EAQZ,eAAgB,EAAE,OAAO,EAAE,OAAO,EAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAC5E,CAAC,EAIK,EAAc,IAAI,QAExB,SAAS,EAAc,EAA2C,CAChE,IAAM,EAAS,EAAY,IAAI,CAAG,EAClC,GAAI,EAAQ,OAAO,EACnB,IAAM,EAAS,EAAkB,MAAM,CAAG,EAE1C,OADA,EAAY,IAAI,EAAK,CAAM,EACpB,CACT,CAGA,MAAM,EAAe,CAAC,KAAM,KAAM,KAAM,KAAM,KAAM,KAAM,KAAM,IAAI,EAE9D,EAAW,GAAwC,QAAQ,IAG3D,EAAuC,CAC3C,CAAE,IAAK,OAAQ,MAAO,OAAQ,SAAU,GAAM,KAAM,QAAS,EAC7D,CAAE,IAAK,YAAa,MAAO,YAAa,SAAU,GAAO,KAAM,QAAS,EACxE,CAAE,IAAK,cAAe,MAAO,cAAe,SAAU,GAAO,KAAM,QAAS,EAC5E,CAAE,IAAK,aAAc,MAAO,aAAc,SAAU,GAAO,KAAM,QAAS,EAC1E,CAAE,IAAK,WAAY,MAAO,WAAY,SAAU,GAAO,KAAM,QAAS,EACtE,CAAE,IAAK,UAAW,MAAO,UAAW,SAAU,GAAO,KAAM,QAAS,EACpE,CAAE,IAAK,eAAgB,MAAO,gBAAiB,SAAU,GAAO,KAAM,QAAS,EAC/E,CAAE,IAAK,eAAgB,MAAO,gBAAiB,SAAU,GAAO,KAAM,QAAS,EAC/E,CAAE,IAAK,iBAAkB,MAAO,iBAAkB,SAAU,GAAO,KAAM,QAAS,EAClF,CAAE,IAAK,mBAAoB,MAAO,mBAAoB,SAAU,GAAO,KAAM,QAAS,EACtF,GAAG,EAAa,IACb,IAAoB,CACnB,IAAK,EAAQ,CAAC,EACd,MAAO,SAAS,EAAE,GAClB,SAAU,GACV,KAAM,QACR,EACF,CACF,EAEM,EAAoB,SAE1B,SAAS,EAAc,EAAmB,CACxC,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,EAAmB,EAAE,CACtD,CAQA,SAAS,EAA2B,EAAmB,EAA6B,CAClF,OAAO,KAAK,MAAM,GAAa,EAAI,EAAc,IAAI,CACvD,CAOA,MAAa,EAA2C,CACtD,KAAM,QACN,OAAQ,EACR,SAAS,EAAmB,EAAsB,CAChD,IAAM,EAAS,EAAc,EAAI,MAAM,EAGjC,EAAU,OAAO,EAAO,MAAS,SAAW,EAAO,KAAK,KAAK,EAAI,GACvE,GAAI,CAAC,EAAS,OAAO,EAAS,eAAgB,gBAAiB,MAAM,EAKrE,IAAM,EAAiB,EAHrB,EAAO,oBAAsB,EAAQ,WAAW,EAAO,kBAAkB,EACrE,EAAQ,MAAM,EAAO,mBAAmB,MAAM,EAC9C,CACuC,EAC7C,GAAI,CAAC,EACH,OAAO,EAAS,wBAAyB,SAAS,EAAQ,gCAAiC,MAAM,EAKnG,IAAM,EADe,EAAW,EAAO,QACX,GAAK,MACjC,GAAI,IAAa,MACf,OAAO,EAAS,uBAAwB,aAAa,EAAS,4BAA6B,UAAU,EAIvG,IAAM,EAAM,EAAW,EAAO,SAAS,EACvC,GAAI,EAAI,OAAS,UAAW,OAAO,EAAS,oBAAqB,4BAA6B,WAAW,EACzG,IAAM,EAAQ,EAAW,EAAO,WAAW,EAC3C,GAAI,EAAM,OAAS,UACjB,OAAO,EAAS,sBAAuB,8BAA+B,aAAa,EAErF,IAAM,EAAO,EAAW,EAAO,UAAU,EACzC,GAAI,EAAK,OAAS,UAAW,OAAO,EAAS,qBAAsB,6BAA8B,YAAY,EAE7G,IAAM,EAAW,EAAI,OAAS,QAAU,EAAI,MAAQ,KAC9C,EAAY,EAAK,OAAS,QAAU,EAAK,MAAQ,KAIjD,EAAe,GAAY,EACjC,GAAI,IAAiB,KACnB,OAAO,EAAS,gBAAiB,sCAAuC,OAAO,EASjF,IAAM,EAAgB,EAAW,EAAO,cAAc,EAChD,EAAY,IAAc,KAAO,KAAO,EAAa,CAAS,EAChE,EAAoC,KAClC,EAAyB,CAAC,EAChC,GAAI,IAAa,KACf,EAAqB,EAAa,CAAQ,OACrC,GAAI,IAAc,KAAM,CAE7B,IAAM,EAAc,IAAkB,KAAgD,IAAA,GAAzC,EAAO,iBAAiB,GACjE,IAAgB,IAAA,GAGlB,EAAS,KAAK,CACZ,YAAa,oBACb,QACE,IAAkB,KACd,qEACA,sBAAsB,EAAc,+CAC1C,MAAO,gBACT,CAAC,EATD,EAAqB,EAA2B,EAAW,CAAW,CAW1E,CAEA,IAAM,EAAQ,GAAG,EAAe,IAAI,EAAO,aACrC,EAAqB,CACzB,OAAQ,EACR,KAAM,EACN,gBAAiB,EACjB,SAAU,EAAO,QACjB,WAAY,EAAO,UACnB,YAAa,EAAO,WACpB,sBAAuB,EAAO,oBAC9B,cAAe,EACf,gBAAiB,EAAM,OAAS,QAAU,EAAM,MAAQ,KACxD,SAAU,MACV,QAAS,EAAW,EAAO,OAAO,EAClC,QAAS,EAAW,EAAO,OAAO,EAClC,QAAS,EAAW,EAAO,OAAO,EAClC,QAAS,EAAW,EAAO,OAAO,EAClC,QAAS,EAAW,EAAO,OAAO,EAClC,QAAS,EAAW,EAAO,OAAO,EAClC,QAAS,EAAW,EAAO,OAAO,EAClC,QAAS,EAAW,EAAO,OAAO,EAClC,QAAS,EAAW,EAAO,OAAO,EAClC,aAAc,EAAW,EAAO,YAAY,EAC5C,aAAc,EAAW,EAAO,YAAY,EAE5C,iBAAkB,EAClB,eAAgB,EAChB,iBAAkB,EAAW,EAAO,gBAAgB,EAEpD,qBAAsB,EAEtB,kBAAmB,KACnB,qBAAsB,KACtB,uBAAwB,KACxB,sBAAuB,KACvB,0BAA2B,KAC3B,gBAAiB,EAAO,cACxB,YAAa,IAAI,KAAK,CAAA,CAAE,YAAY,CACtC,EAEA,OAAO,EAAS,OAAS,EACrB,CAAE,KAAM,UAAW,GAAI,EAAO,MAAK,UAAS,EAC5C,CAAE,KAAM,UAAW,GAAI,EAAO,KAAI,CACxC,CACF"}
|
package/dist/runner-0r6UICjo.mjs
DELETED
|
@@ -1,2 +0,0 @@
|
|
|
1
|
-
import{z as e}from"zod";import{createReadStream as t}from"node:fs";import{parse as n}from"csv-parse";import{bulkUpsert as r}from"@murumets-ee/search-elasticsearch";const i=50,a=5;var o=class{patterns=new Map;maxPatterns;maxSamplesPerPattern;droppedSignatures=0;constructor(e={}){this.maxPatterns=e.maxPatterns??50,this.maxSamplesPerPattern=e.maxSamplesPerPattern??5}addError(e,t,n,r,i){let a=`${t}:${r??`GENERAL`}:${n}`,o=this.patterns.get(a);if(!o){if(this.patterns.size>=this.maxPatterns){this.droppedSignatures+=1;return}o={errorType:t,field:r??null,message:n,count:0,firstOccurrence:e,lastOccurrence:e,samples:[]},this.patterns.set(a,o)}o.count+=1,o.lastOccurrence=e,o.samples.length<this.maxSamplesPerPattern&&o.samples.push({rowNumber:e,rowData:i})}getTotalErrorCount(){let e=0;for(let t of this.patterns.values())e+=t.count;return e}getDistinctPatternCount(){return this.patterns.size}getDroppedSignatureCount(){return this.droppedSignatures}getTopPatterns(){let e=Array.from(this.patterns.values()).sort((e,t)=>t.count-e.count),t=e.reduce((e,t)=>e+t.count,0);return e.map(e=>({errorType:e.errorType,field:e.field,message:e.message,count:e.count,firstOccurrence:e.firstOccurrence,lastOccurrence:e.lastOccurrence,samples:e.samples.slice(),percentage:t>0?e.count/t*100:0}))}snapshot(){return{totalErrors:this.getTotalErrorCount(),distinctPatterns:this.getDistinctPatternCount(),droppedSignatures:this.droppedSignatures,patterns:this.getTopPatterns()}}};function s(e){let t=new Set;for(let n of e)if(n!==``){if(t.has(n))throw Error(`Duplicate header "${n}" after trimming — column names must be unique`);t.add(n)}}async function c(e,n={}){let{hasHeader:r=!0}=n,i=t(e),a=0,o=0;for await(let e of i){for(let t=0;t<e.length;t+=1)e[t]===10&&(a+=1);e.length>0&&(o=e[e.length-1]??0)}return o!==0&&o!==10&&(a+=1),r?Math.max(0,a-1):a}async function*l(e){let{filePath:r,delimiter:i=` `,hasHeader:a=!0,columns:o,relaxColumnCount:c=!1,quote:l=!1}=e,u=o?Array.from(o):a?e=>{let t=e.map(e=>e.trim());return s(t),t}:!1,d=t(r),f=d.pipe(n({delimiter:i,columns:u,bom:!0,skip_empty_lines:!0,relax_column_count:c,quote:l})),p=0;try{for await(let e of f){p+=1;let t={};if(Array.isArray(e)){let n=e;for(let e=0;e<n.length;e+=1)t[String(e)]=n[e]??``}else for(let[n,r]of Object.entries(e))t[n]=r??``;yield{rowNumber:p,row:t}}}finally{d.destroy()}}const u=e.object({format:e.enum([`csv`,`tsv`,`xlsx`]).optional(),delimiter:e.string().min(1).max(4).optional(),hasHeader:e.boolean().optional(),headerRowIndex:e.number().int().min(0).optional(),sheet:e.string().min(1).max(128).optional()}).strict();function d(e){let t=e.delimiter??(e.format===`csv`?`,`:` `);return{filePath:e.filePath,delimiter:t,...e.hasHeader!==void 0&&{hasHeader:e.hasHeader},...e.columns!==void 0&&{columns:e.columns},...e.relaxColumnCount!==void 0&&{relaxColumnCount:e.relaxColumnCount},...e.quote!==void 0&&{quote:e.quote}}}async function*f(e){if(e.format===`xlsx`){let{streamXlsx:t}=await import(`./xlsx-reader-oApgZRJd.mjs`);yield*t({filePath:e.filePath,...e.hasHeader!==void 0&&{hasHeader:e.hasHeader},...e.headerRowIndex!==void 0&&{headerRowIndex:e.headerRowIndex},...e.sheet!==void 0&&{sheet:e.sheet}});return}yield*l(d(e))}const p=1e3;async function m(e){let{importRunId:t,runLabel:n,params:i,transform:a,feed:s,esClient:c,esIndex:l,batchSize:u=p,onProgress:d,rowLimit:m,signal:h,errorTracker:g=new o,warningTracker:_=new o,logger:v}=e;if(u<1)throw Error(`batchSize must be >= 1 (got ${u})`);let y=Date.now();v?.info({importRunId:t,runLabel:n,batchSize:u,esIndex:l},`imports.runner: started`);let b=0,x=0,S=0,C=0,w=0,T=0,E=[],D=async()=>{if(E.length===0)return;let e=E;E=[];let n=Date.now(),i;try{i=await r(c,{index:l,docs:e.map(({id:e,doc:t})=>({id:e,doc:t})),...h!==void 0&&{signal:h}})}catch(t){let n=h?.aborted??(t instanceof Error&&(t.name===`AbortError`||/abort/i.test(t.message)))?`aborted`:`bulk_request_failed`,r=t instanceof Error?t.message:String(t);for(let{rowNumber:t}of e)g.addError(t,n,r,void 0,null);throw S+=e.length,t}x+=i.succeeded,S+=i.failures.length,T+=1;let a=Date.now()-n,o=(Date.now()-y)/1e3;if(v?.info({importRunId:t,batch:T,flushMs:a,submitted:e.length,succeeded:i.succeeded,failed:i.failures.length,rowsRead:b,rowsPerSecond:o>0?Math.round(b/o):0},`imports.runner: batch flushed`),i.failures.length>0){let t=new Map(e.map(e=>[e.id,e.rowNumber]));for(let e of i.failures){let n=t.get(e.id)??-1;g.addError(n,e.type,e.reason,void 0,{id:e.id})}}let s=new Set(i.failures.map(e=>e.id));for(let t of e)if(!(!t.warnings||t.warnings.length===0)&&!s.has(t.id)){w+=1;for(let e of t.warnings)_.addError(t.rowNumber,e.warningType,e.message,e.field,t.rowData)}if(d){let e=(Date.now()-y)/1e3;d({rowsRead:b,rowsSucceeded:x,rowsFailed:S,rowsSkipped:C,rowsWarned:w,batchesCompleted:T,elapsedSeconds:e,rowsPerSecond:e>0?b/e:0,distinctErrorPatterns:g.getDistinctPatternCount()})}},O=u,k=new Map;for await(let{rowNumber:e,row:r}of f(s)){if(h?.aborted||m!==void 0&&b>=m)break;if(b+=1,b===1&&v?.info({importRunId:t,rowNumber:e,sampleKeys:Object.keys(r)},`imports.runner: STREAM_FIRST_ROW`),b%O===0){let e=(Date.now()-y)/1e3;v?.info({importRunId:t,rowsRead:b,rowsSucceeded:x,rowsFailed:S,rowsSkipped:C,pending:E.length,elapsedSec:Math.round(e*10)/10,rowsPerSecond:e>0?Math.round(b/e):0,distinctFirstErrorTypes:k.size},`imports.runner: STREAM_TICK`)}let o={importRunId:t,params:i,runLabel:n,rowNumber:e},s;try{s=await a(r,o)}catch(n){let i=n instanceof Error?n.message:String(n);g.addError(e,`transform_threw`,i,void 0,r),k.has(`transform_threw`)||(k.set(`transform_threw`,{rowNumber:e,message:i}),v?.warn({importRunId:t,rowNumber:e,errorType:`transform_threw`,message:i},`imports.runner: FIRST_ERROR transform_threw`)),S+=1;continue}if(s.kind===`skip`){C+=1;continue}if(s.kind===`error`){g.addError(e,s.error.errorType,s.error.message,s.error.field,r),k.has(s.error.errorType)||(k.set(s.error.errorType,{rowNumber:e,message:s.error.message}),v?.warn({importRunId:t,rowNumber:e,errorType:s.error.errorType,field:s.error.field,message:s.error.message},`imports.runner: FIRST_ERROR ${s.error.errorType}`)),S+=1;continue}E.push({id:s.id,doc:s.doc,rowNumber:e,rowData:r,...s.warnings&&s.warnings.length>0&&{warnings:s.warnings}}),E.length>=u&&await D()}await D();let A=(Date.now()-y)/1e3;return v?.info({importRunId:t,rowsRead:b,rowsSucceeded:x,rowsFailed:S,rowsSkipped:C,batchesCompleted:T,totalSec:Math.round(A*10)/10,rowsPerSecond:A>0?Math.round(b/A):0},`imports.runner: finished`),{rowsRead:b,rowsSucceeded:x,rowsFailed:S,rowsSkipped:C,rowsWarned:w,batchesCompleted:T,errors:g.snapshot(),warnings:_.snapshot()}}export{s as a,i as c,f as i,a as l,m as n,c as o,u as r,l as s,p as t,o as u};
|
|
2
|
-
//# sourceMappingURL=runner-0r6UICjo.mjs.map
|