@murumets-ee/imports 0.49.0 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/carmaker-feed-C_MnDaOC.mjs +2 -0
- package/dist/{carmaker-feed-CxhZBrkO.mjs.map → carmaker-feed-C_MnDaOC.mjs.map} +1 -1
- package/dist/carmaker-feed-DlD2xPs0.d.mts +58 -0
- package/dist/carmaker-feed-DlD2xPs0.d.mts.map +1 -0
- package/dist/{cleanup-CHjDgvKT.mjs → cleanup-DdILFoAP.mjs} +2 -2
- package/dist/{cleanup-CHjDgvKT.mjs.map → cleanup-DdILFoAP.mjs.map} +1 -1
- package/dist/index.d.mts +7 -5
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/index.mjs.map +1 -1
- package/dist/mapping/index.d.mts +2 -359
- package/dist/mapping/index.d.mts.map +1 -1
- package/dist/mapping/index.mjs +1 -1
- package/dist/{mapping-transform-D76b-P-p.mjs → mapping-transform-AMFEg1rS.mjs} +2 -2
- package/dist/{mapping-transform-D76b-P-p.mjs.map → mapping-transform-AMFEg1rS.mjs.map} +1 -1
- package/dist/{parts-schema-D0gMhNWj.mjs → parts-schema-BBUINSdN.mjs} +2 -2
- package/dist/{parts-schema-D0gMhNWj.mjs.map → parts-schema-BBUINSdN.mjs.map} +1 -1
- package/dist/{parts-schema-x8_GRYWc.d.mts → parts-schema-u5kr5zFg.d.mts} +359 -3
- package/dist/parts-schema-u5kr5zFg.d.mts.map +1 -0
- package/dist/plugin.d.mts +1 -1
- package/dist/plugin.mjs +1 -1
- package/dist/plugin.mjs.map +1 -1
- package/dist/runner-wPnCZlB4.mjs +2 -0
- package/dist/runner-wPnCZlB4.mjs.map +1 -0
- package/dist/{schema-registry-C_lQje-K.mjs → schema-registry-B_7AweTa.mjs} +2 -2
- package/dist/{schema-registry-C_lQje-K.mjs.map → schema-registry-B_7AweTa.mjs.map} +1 -1
- package/dist/storage-resolver.d.mts +1 -1
- package/dist/transform-DwAQ4b-j.mjs +2 -0
- package/dist/transform-DwAQ4b-j.mjs.map +1 -0
- package/dist/{transform-CABBaYmF.d.mts → transform-WeiiLdYP.d.mts} +12 -1
- package/dist/transform-WeiiLdYP.d.mts.map +1 -0
- package/dist/transforms.d.mts +2 -46
- package/dist/transforms.mjs +1 -1
- package/dist/{worker-DRTgbtT7.d.mts → worker-DH3SN-91.d.mts} +189 -4
- package/dist/worker-DH3SN-91.d.mts.map +1 -0
- package/dist/worker.d.mts +1 -1
- package/dist/worker.mjs +1 -1
- package/dist/worker.mjs.map +1 -1
- package/dist/{xlsx-reader-Ccesszbc.mjs → xlsx-reader-Dsq_ERiB.mjs} +2 -2
- package/dist/{xlsx-reader-Ccesszbc.mjs.map → xlsx-reader-Dsq_ERiB.mjs.map} +1 -1
- package/package.json +8 -8
- package/dist/carmaker-feed-CxhZBrkO.mjs +0 -2
- package/dist/parts-schema-x8_GRYWc.d.mts.map +0 -1
- package/dist/runner-C-s6SD7t.mjs +0 -2
- package/dist/runner-C-s6SD7t.mjs.map +0 -1
- package/dist/transform-CABBaYmF.d.mts.map +0 -1
- package/dist/transform-DIzWppgJ.mjs +0 -2
- package/dist/transform-DIzWppgJ.mjs.map +0 -1
- package/dist/transforms.d.mts.map +0 -1
- package/dist/worker-DRTgbtT7.d.mts.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transform-DwAQ4b-j.mjs","names":[],"sources":["../src/entities/import-run.ts","../src/entities/import-mapping-validation.ts","../src/entities/import-mapping.ts","../src/transform.ts"],"sourcesContent":["/**\n * ImportRun — one feed-import attempt. Tracks file, transform, status,\n * row totals, and the top error patterns surfaced by `ErrorTracker`.\n *\n * Per PLAN-ECOMMERCE.md PR 7 (PoC scope): generic plumbing entity. The\n * commerce-specific transform parameters (brandId, supplierId, supplier\n * code prefix) live in the opaque `params` JSONB so the imports package\n * stays free of commerce-domain dependencies. PR 8 wires a concrete\n * carmaker-feed transform that reads those params; PR 8a renders a\n * bespoke `/admin/commerce/imports` page over this entity's history.\n *\n * `hideFromMenu: true` because PR 7 ships only the entity + plumbing —\n * the operator-facing surface is PR 8a. The auto-EntityListPage at\n * `/admin/import_run` still resolves for direct lookups, but no sidebar\n * entry points to it.\n */\n\nimport { behavior, defineEntity, field, searchable } from '@murumets-ee/entity'\nimport type { AdminClient } from '@murumets-ee/entity/admin'\n\n/**\n * Lifecycle stages.\n *\n * - `pending` — row exists, queue job has been enqueued, worker hasn't picked it up yet.\n * - `running` — worker is streaming + bulk-writing.\n * - `succeeded` — worker finished cleanly (note: per-row failures still possible — see `totals.failed`).\n * - `failed` — worker threw and the queue marked the job dead. `errorSummary.fatal` carries the cause.\n * - `cancelled` — manual operator action via PR 8a (out of scope for PR 7).\n */\nexport const IMPORT_RUN_STATUSES = [\n 'pending',\n 'running',\n 'succeeded',\n 'failed',\n 'cancelled',\n] as const\nexport type ImportRunStatus = (typeof IMPORT_RUN_STATUSES)[number]\n\nexport const ImportRun = defineEntity({\n name: 'import_run',\n fields: {\n /** Operator-visible label, e.g. `\"MERCEDES — ME_20251027075918.txt\"`. Free-form. */\n label: field.text({ required: true, maxLength: 255 }),\n /** See {@link IMPORT_RUN_STATUSES}. */\n status: field.select({\n options: IMPORT_RUN_STATUSES,\n default: 'pending',\n indexed: true,\n }),\n /**\n * Path or storage key of the uploaded feed file. Generic string —\n * could be `/var/lumi/uploads/<id>.txt` for the local-disk PoC or an\n * S3 object key once storage adapter integration lands. The worker\n * reads this with the configured `readFeed` resolver.\n */\n filePath: field.text({ required: true, maxLength: 2048 }),\n /**\n * Name of the registered transform applied to each row. Resolved at\n * worker-dispatch time against the transform registry contributed by\n * the consumer. PoC: `'commerce:carmaker-feed'`.\n */\n transformName: field.text({ required: true, maxLength: 128, indexed: true }),\n /**\n * Opaque per-transform parameters. The carmaker transform expects\n * `{ brandId, supplierId, codePrefix? }`. Validation happens inside\n * the transform — the imports package never inspects this shape so\n * a new transform with different params doesn't require a schema\n * migration.\n */\n params: field.json(),\n /**\n * Row counters: `{ submitted, succeeded, failed, skipped, batches }`.\n * Updated by the worker as each batch completes. Final values are\n * what the operator reads; intermediate progress comes from\n * `toolkit_jobs.progress` via the queue UI.\n */\n totals: field.json(),\n /**\n * Output of `ErrorTracker.getTopPatterns(totalRows)` — the top-50\n * error signatures with up to 5 sample rows each. Empty `[]` until\n * the worker writes it on completion.\n */\n errorSummary: field.json(),\n /**\n * NON-FATAL warning patterns (commerce-demo PR4) — same\n * `ErrorTrackerSnapshot` shape as {@link errorSummary} but for advisories\n * on rows that STILL succeeded (their doc was indexed), e.g.\n * `no_purchase_price` (a part imported list-only, cost pending). Kept\n * separate from `errorSummary` so a warning never reads as a failed row.\n * Null/absent until a SUCCESSFUL run that produced warnings writes it — a\n * fatal failure (`markFailed`) abandons the run mid-stream and only sets\n * `errorSummary.fatal`, so the two summaries are not co-present.\n *\n * NOTE: because the warning aggregator reuses `ErrorTracker`, the persisted\n * JSON reuses its field names — the `totalErrors` key inside this blob is\n * really the WARNING count (and `patterns[].errorType` the warning type).\n */\n warningSummary: field.json(),\n /**\n * Price-distribution snapshot (F010 / issue #602) — `{ count, nullRate,\n * min, median, max }`, computed over ES-accepted rows' derived purchase\n * price (`PartsDocument.purchase_price_minor`, integer minor units) in\n * the SAME streaming pass that builds `totals`. See\n * `@murumets-ee/imports`' `price-stats.ts` for the exact shape and the\n * bounded-sample cap.\n *\n * The pipeline's existing 6b header-drift check (commerce's\n * register-handler.ts) only catches STRUCTURAL drift — a mapped column\n * vanishing from the header row. It does NOT catch a supplier silently\n * flipping their decimal/thousands separator, which changes no header\n * at all and produces numerically valid but silently WRONG prices. The\n * worker compares THIS run's `priceStats.median` against the most\n * recent prior SUCCEEDED run for the same (transformName, brandId,\n * supplierId) and, on a magnitude-scale shift (>=10x or <=0.1x — see\n * `PRICE_ANOMALY_THRESHOLD_MULTIPLIER`), appends a NON-FATAL\n * `price_anomaly` entry to {@link warningSummary}. Unlike the 6b 409,\n * this never blocks the run — a real repricing is a legitimate reason\n * prices would shift, so it's an advisory only.\n *\n * `null`/absent for runs written before this field existed, or for a\n * schema/transform whose rows never carry a price sample.\n */\n priceStats: field.json(),\n /** Set when the worker picks up the job. */\n startedAt: field.date(),\n /** Set when the worker finishes (success OR fatal failure). */\n finishedAt: field.date(),\n /**\n * `toolkit_jobs.id` of the queue job processing this run — link\n * back so PR 8a can show live progress without a second lookup.\n */\n queueJobId: field.text({ maxLength: 64, indexed: true }),\n },\n behaviors: [\n behavior.auditable(),\n // Operator-facing search: by label (e.g. \"MERCEDES — ME_*.txt\"),\n // transform name, or queue job id. No FTS — labels are short\n // alphanumeric tags, not prose.\n searchable({\n fields: ['label', 'transformName', 'queueJobId', 'status', 'filePath'],\n projection: (row) => ({\n id: row.id as string,\n label: row.label as string,\n description: `${row.transformName} · ${row.status}`,\n }),\n }),\n ],\n scope: 'global',\n admin: {\n group: 'imports',\n label: 'Import runs',\n labelSingular: 'Import run',\n icon: 'upload',\n hideFromMenu: true,\n },\n})\n\nexport type ImportRunClient = AdminClient<typeof ImportRun.allFields>\n","/**\n * Save-time validation for `import_mapping` rows (commerce-demo phase 02 §3/§5).\n *\n * This is the R02 injection boundary realised as an entity behavior: a mapping's\n * `columnMap` may only point at fields the chosen `TargetSchema` whitelists, and a\n * mapping is incomplete unless every REQUIRED target of that schema is mapped. Both\n * checks run on EVERY write path (the PR3 wizard's generic-CRUD `POST`, a future\n * seed, a direct `AdminClient` call) — \"validate at the boundary\", not \"validate in\n * the one UI we happen to ship\".\n *\n * The transform compiler (`buildMappingTransform`) re-checks the whitelist at run\n * time, so this is defense-in-depth — but failing at SAVE gives the operator the\n * error in the wizard (where they can fix it) instead of as a per-row run failure.\n *\n * Lives next to the entity (same package) and imports the mapping engine + schema\n * registry one-directionally (engine/registry never import the entity), so no cycle.\n */\n\nimport type { Behavior } from '@murumets-ee/entity'\nimport { z } from 'zod'\nimport { mappingColumnSchema, numberFormatSchema } from '../mapping/engine.js'\nimport { getTargetSchema, listTargetSchemas } from '../mapping/schema-registry.js'\n\n/** Default applied by the `numberFormat` field — used when a create omits it (defaults run after hooks). */\nconst DEFAULT_NUMBER_FORMAT = { decimal: '.', thousands: '' } as const\n\n// Bounded, like every JSONB the entity accepts: a legitimate column map has at\n// most one entry per source column (same order of magnitude as the snapshot's\n// 256-column cap), so an arbitrarily long array is a bloat vector, not data.\nconst columnMapSchema = z.array(mappingColumnSchema).max(512)\n\n/**\n * File-less edit snapshot (#667) — header columns + a few sample rows, same\n * `{ columns, rows: [{column → cell}] }` shape as the preview endpoint's\n * response. Bounded on every axis (these are DISPLAY samples, not data):\n * a snapshot that exceeds the caps should have been trimmed/omitted by the\n * wizard, so an oversized one here is a bypass attempt or a bug — reject, and\n * additionally hard-cap the serialized size so 256-wide × max-cell rows can't\n * park ~600 KB of JSONB per mapping row.\n *\n * Strictness matters: the behavior hook persists the RAW input, so a\n * non-strict parse would let unknown top-level keys ride into JSONB unmeasured\n * (zod strips them from `snap.data`, but that stripped copy isn't what's\n * stored).\n */\nconst headerSnapshotSchema = z.strictObject({\n columns: z.array(z.string().min(1).max(256)).min(1).max(256),\n rows: z.array(z.record(z.string().max(500))).max(5),\n})\n\n/**\n * Serialized-size ceiling for a stored snapshot (defense-in-depth over the\n * per-axis caps). Measured in UTF-16 code units (`JSON.stringify(...).length`)\n * — multi-byte content can serialize up to ~3× larger on disk; the bound holds\n * either way. Mirrored client-side by `toHeaderSnapshot` in commerce-ui\n * (pinned by its round-trip test).\n */\nexport const HEADER_SNAPSHOT_MAX_CHARS = 100_000\n\n/**\n * Serialized-size ceiling for the RAW `columnMap` input. Same rationale + same\n * raw-not-parsed measurement as the snapshot ceiling: `mappingColumnSchema` is\n * shared with the RUN-TIME params parser (engine.ts), so it can't be made\n * `.strict()` without risking previously-persisted `import_run.params` — this\n * save-path-only ceiling bounds what unknown entry keys / unbounded per-entry\n * `transforms` arrays can park in JSONB without touching run-time parsing. A\n * legitimate 256-column map with transform chains serializes well under this.\n */\nexport const COLUMN_MAP_MAX_CHARS = 100_000\n\n/**\n * Throw a descriptive error unless the mapping is structurally valid AND honours its\n * schema's field whitelist + required targets. The thrown message is operator-facing\n * (surfaced verbatim by the admin CRUD route as a 4xx body).\n */\nexport function assertValidMapping(input: {\n targetSchema: unknown\n numberFormat: unknown\n columnMap: unknown\n headerSnapshot?: unknown\n}): void {\n // Snapshot is optional metadata — validate only when present (absent/null =\n // \"no file-less edit support for this row\", a valid state for pre-#667 rows).\n if (input.headerSnapshot !== undefined && input.headerSnapshot !== null) {\n const snap = headerSnapshotSchema.safeParse(input.headerSnapshot)\n if (!snap.success) {\n throw new Error(`Invalid headerSnapshot: ${snap.error.issues[0]?.message ?? 'bad shape'}`)\n }\n // Measure the RAW input, not `snap.data` — the raw value is what the write\n // path persists, and it's identical to the parse only because the schema\n // is `.strict()`. Measuring the raw keeps that true even if strictness is\n // ever relaxed.\n const serialized = JSON.stringify(input.headerSnapshot).length\n if (serialized > HEADER_SNAPSHOT_MAX_CHARS) {\n throw new Error(\n `headerSnapshot too large (${serialized} > ${HEADER_SNAPSHOT_MAX_CHARS} serialized chars) — trim sample rows/cells`,\n )\n }\n }\n const targetSchema = typeof input.targetSchema === 'string' ? input.targetSchema : ''\n const schema = getTargetSchema(targetSchema)\n if (!schema) {\n throw new Error(\n `Unknown target schema \"${targetSchema}\" (registered: ${listTargetSchemas().join(', ')})`,\n )\n }\n\n const nf = numberFormatSchema.safeParse(input.numberFormat)\n if (!nf.success) {\n throw new Error(`Invalid numberFormat: ${nf.error.issues[0]?.message ?? 'bad shape'}`)\n }\n\n const cm = columnMapSchema.safeParse(input.columnMap)\n if (!cm.success) {\n throw new Error(`Invalid columnMap: ${cm.error.issues[0]?.message ?? 'bad shape'}`)\n }\n const columnMapSerialized = JSON.stringify(input.columnMap).length\n if (columnMapSerialized > COLUMN_MAP_MAX_CHARS) {\n throw new Error(\n `columnMap too large (${columnMapSerialized} > ${COLUMN_MAP_MAX_CHARS} serialized chars)`,\n )\n }\n\n const allowed = new Set(schema.fields.map((f) => f.key))\n const seenTargets = new Set<string>()\n const dupeTargets = new Set<string>()\n for (const col of cm.data) {\n if (!allowed.has(col.target)) {\n throw new Error(\n `Mapping target \"${col.target}\" is not a field of schema \"${schema.name}\" ` +\n `(allowed: ${[...allowed].join(', ')})`,\n )\n }\n // Two source columns mapping to the SAME target collapse last-write-wins at run\n // time (`mapped[col.target] = …`), silently importing wrong data. The wizard\n // blocks this client-side; enforce it here too so a direct API call can't bypass\n // it (validate at the boundary).\n if (seenTargets.has(col.target)) dupeTargets.add(col.target)\n seenTargets.add(col.target)\n }\n if (dupeTargets.size > 0) {\n throw new Error(\n `Mapping for schema \"${schema.name}\" maps a target more than once: ${[...dupeTargets].join(', ')}`,\n )\n }\n\n const mappedTargets = new Set(cm.data.map((c) => c.target))\n const missing = schema.fields\n .filter((f) => f.required && !mappedTargets.has(f.key))\n .map((f) => f.key)\n if (missing.length > 0) {\n throw new Error(\n `Mapping for schema \"${schema.name}\" is missing required target(s): ${missing.join(', ')}`,\n )\n }\n}\n\n/**\n * Behavior wrapping {@link assertValidMapping}. Typed `Behavior<Record<never,\n * never>, …>` (NOT plain `Behavior`, whose wide `fields?` would collapse the\n * entity's inferred field types — the documented no-field-behavior gotcha).\n * Contributes a hook, no fields.\n */\nexport const importMappingValidation: Behavior<\n Record<never, never>,\n 'imports:import-mapping-validation'\n> = {\n name: 'imports:import-mapping-validation',\n hooks: {\n beforeCreate: async (data) => {\n // Field defaults run AFTER hooks, so an omitted numberFormat/columnMap is the\n // field default here — mirror those defaults so a minimal create validates\n // against the same effective config the row will hold.\n assertValidMapping({\n targetSchema: data.targetSchema,\n numberFormat: data.numberFormat ?? DEFAULT_NUMBER_FORMAT,\n columnMap: data.columnMap ?? [],\n headerSnapshot: data.headerSnapshot,\n })\n return data\n },\n beforeUpdate: async (_id, data, ctx) => {\n // A PATCH is partial — validate the EFFECTIVE config (patch overlaid on the\n // current row), so changing only `name` doesn't escape validation and changing\n // only `columnMap` is still checked against the persisted `targetSchema`.\n const current = (await ctx?.loadCurrent?.()) ?? {}\n assertValidMapping({\n targetSchema: 'targetSchema' in data ? data.targetSchema : current.targetSchema,\n numberFormat:\n 'numberFormat' in data\n ? data.numberFormat\n : (current.numberFormat ?? DEFAULT_NUMBER_FORMAT),\n columnMap: 'columnMap' in data ? data.columnMap : (current.columnMap ?? []),\n // Only the incoming value needs checking — the persisted snapshot was\n // validated when it was written.\n headerSnapshot: 'headerSnapshot' in data ? data.headerSnapshot : undefined,\n })\n return data\n },\n },\n}\n","/**\n * ImportMapping — a reusable, per-(supplier, brand) column-mapping profile that\n * drives the configurable feed-import engine (commerce-demo D005, phase 02).\n *\n * Replaces the hardcoded 14/15-column carmaker transform with operator-editable\n * mappings: which source column → which whitelisted target field, per-column\n * cleaning transforms, the file format (csv/tsv/xlsx), and the number format\n * (decimal + thousands separators — real supplier feeds vary, see R03).\n *\n * **Why this lives in `@murumets-ee/imports`, not `@murumets-ee/commerce`**\n * (D005 cycle constraint): the engine + the concrete target schemas need the\n * commerce `PartsDocument` shape, and `imports` already depends on\n * `@murumets-ee/search-elasticsearch` for it. Putting the mapping in commerce\n * would force `commerce → imports` and a forbidden import cycle. The entity\n * references the commerce `supplier` / `brand` entities **by name string**\n * (`field.reference({ entity: 'supplier' })`) — no Entity-object import, so the\n * leaf-package boundary holds.\n *\n * **Resolution** (most-specific, D002 ladder): for an upload tagged\n * `(supplier, brand)`, pick the `(supplier, brand)` mapping if present, else the\n * `(supplier, NULL)` supplier-default. See `resolveMapping()` in the engine.\n *\n * `hideFromMenu: true` — the mapping CRUD wizard lands in commerce-demo PR3. The\n * auto-EntityListPage at `/admin/import_mapping` still resolves for direct lookups\n * and for the admin-only AdminClient writes this engine relies on.\n */\n\nimport type { Behavior, SchemaFragment, SchemaTarget } from '@murumets-ee/entity'\nimport { behavior, defineEntity, field, searchable } from '@murumets-ee/entity'\nimport type { AdminClient } from '@murumets-ee/entity/admin'\nimport { sql } from 'drizzle-orm'\nimport type { PgColumn } from 'drizzle-orm/pg-core'\nimport { importMappingValidation } from './import-mapping-validation.js'\n\n/**\n * Registered target schemas a mapping may write into. This is the FIRST of two\n * whitelists (the second is each schema's target-field set) — a mapping whose\n * `targetSchema` is outside this list is rejected (deny-by-default, R02).\n */\nexport const IMPORT_MAPPING_TARGET_SCHEMAS = ['parts', 'discount_table'] as const\nexport type ImportMappingTargetSchema = (typeof IMPORT_MAPPING_TARGET_SCHEMAS)[number]\n\n/** File formats the reader dispatches on. `csv`/`tsv` → csv-parse; `xlsx` → exceljs. */\nexport const IMPORT_MAPPING_FORMATS = ['csv', 'tsv', 'xlsx'] as const\nexport type ImportMappingFormat = (typeof IMPORT_MAPPING_FORMATS)[number]\n\n/**\n * Declarative DDL: one mapping per `(supplier, brand, target_schema)`, enforced by\n * TWO PARTIAL unique indexes rather than one plain composite unique.\n *\n * Why partial: a plain `UNIQUE (supplierId, brandId, targetSchema)` treats NULL\n * `brandId` as distinct (Postgres NULL-distinct semantics), so it would allow MANY\n * supplier-default rows (`brandId IS NULL`) for the same `(supplier, schema)` —\n * ambiguous, since the engine resolves `brandId === null` as THE supplier default\n * (`resolveMapping`). Splitting it:\n * - brand-specific rows (`brandId IS NOT NULL`): unique on `(supplier, brand, schema)`;\n * - supplier-default rows (`brandId IS NULL`): unique on `(supplier, schema)`.\n *\n * The `where` predicates use raw `sql` with the live Drizzle column reference (so the\n * emitted identifier is correct regardless of casing) — `IS NULL` / `IS NOT NULL`,\n * NOT `eq(col, …)` which would serialize as an invalid bound `$1` in DDL.\n *\n * Typed `Behavior<Record<never, never>, …>` (NOT plain `Behavior`, whose wide\n * `fields?` would collapse the entity's inferred field types to `any` — the\n * documented no-field-behavior gotcha). Contributes constraints, no fields.\n */\nconst importMappingUnique: Behavior<Record<never, never>, 'imports:import-mapping-unique'> = {\n name: 'imports:import-mapping-unique',\n constraints: (table: Record<string, PgColumn>, target: SchemaTarget): SchemaFragment[] => {\n if (target !== 'main') return []\n const supplierId = table.supplierId\n const brandId = table.brandId\n const targetSchema = table.targetSchema\n // Defensive: only emit when all three columns exist.\n if (!supplierId || !brandId || !targetSchema) return []\n return [\n {\n kind: 'uniqueIndex',\n name: 'uq_import_mapping_supplier_brand_schema',\n on: [supplierId, brandId, targetSchema],\n where: sql`${brandId} IS NOT NULL`,\n },\n {\n kind: 'uniqueIndex',\n name: 'uq_import_mapping_supplier_schema_default',\n on: [supplierId, targetSchema],\n where: sql`${brandId} IS NULL`,\n },\n ]\n },\n}\n\nexport const ImportMapping = defineEntity({\n name: 'import_mapping',\n scope: 'global',\n fields: {\n /** FK to commerce `supplier` (by name string — no cross-package import). Required. */\n supplierId: field.reference({ entity: 'supplier', required: true, indexed: true }),\n /** FK to commerce `brand`. NULL = supplier default (applies to all that supplier's brands). */\n brandId: field.reference({ entity: 'brand', indexed: true }),\n /** Which registered target schema this mapping feeds — `parts` or `discount_table`. */\n targetSchema: field.select({\n options: IMPORT_MAPPING_TARGET_SCHEMAS,\n required: true,\n indexed: true,\n }),\n /** Operator-facing profile name (a supplier can have e.g. \"monthly full\" vs \"weekly delta\"). */\n name: field.text({ required: true, maxLength: 255 }),\n /** File format — drives reader dispatch + the default delimiter. */\n format: field.select({ options: IMPORT_MAPPING_FORMATS, required: true }),\n /**\n * Explicit single-char field delimiter override. When omitted, the reader\n * defaults from `format` (`csv` → `,`, `tsv` → `\\t`). Ignored for `xlsx`.\n */\n delimiter: field.text({ maxLength: 4 }),\n /** First row is the header. `false` → positional `{ \"0\": …, \"1\": … }` source keys. */\n hasHeader: field.boolean({ default: true }),\n /** 0-based header row index (header hygiene — R03 feeds have leading/trailing-space headers). */\n headerRowIndex: field.number({ integer: true, min: 0, default: 0 }),\n /** xlsx sheet name (or 1-based index as a string). Ignored for csv/tsv. */\n sheet: field.text({ maxLength: 128 }),\n /**\n * Number format for the `number` per-column transform:\n * `{ decimal: '.'|',', thousands: ''|'.'|','|' ' }`. Opaque json; validated\n * by the engine's Zod schema at consume time (matches `import_run.params`).\n */\n numberFormat: field.json({ default: { decimal: '.', thousands: '' } }),\n /**\n * Ordered column map: `[{ source, target, transforms? }]`. `source` is a\n * header name (or positional index string); `target` MUST be a field in the\n * chosen schema's whitelist (validated by the engine's Zod schema, R02\n * injection boundary); `transforms` is a bounded, enum-keyed list. Opaque\n * json on the entity; typed + validated by the engine.\n */\n columnMap: field.json({ default: [] }),\n /**\n * File-less edit snapshot (#667): `{ columns: string[], rows: [{column →\n * cell}] }` — the sample feed's header + a few sample rows, captured from\n * the wizard's preview at save time. Lets the editor re-open a saved\n * mapping WITHOUT re-uploading the (potentially huge) feed just to tweak a\n * transform. NULL on rows saved before this field existed (and when the\n * preview exceeded the size caps) — the editor falls back to the\n * upload-first flow. Staleness is benign: a source column that later\n * vanishes from the real feed is caught at run time by the 6b drift check\n * (409) / row errors, not by this snapshot. Size-capped by\n * `import-mapping-validation`.\n */\n headerSnapshot: field.json(),\n /** Inactive mappings are skipped by resolution. */\n active: field.boolean({ default: true, indexed: true }),\n },\n behaviors: [\n behavior.auditable(),\n // Operator-facing search (PR3 wizard): by profile name, schema, or format.\n // No `fts` → IlikeProvider, so this adds NO DDL (the 0036 migration is\n // unchanged). Names are short tags, not prose.\n searchable({\n fields: ['name', 'targetSchema', 'format'],\n projection: (row) => ({\n id: row.id as string,\n label: row.name as string,\n description: `${row.targetSchema} · ${row.format}`,\n }),\n }),\n importMappingUnique,\n // Save-time whitelist + required-target validation (R02 injection boundary).\n importMappingValidation,\n ],\n admin: {\n group: 'imports',\n label: 'Import mappings',\n labelSingular: 'Import mapping',\n icon: 'table',\n hideFromMenu: true,\n },\n})\n\nexport type ImportMappingClient = AdminClient<typeof ImportMapping.allFields>\n","/**\n * Per-feed transform plugin interface (PLAN-ECOMMERCE.md PR 7 / D14).\n *\n * The streaming reader reads raw rows out of the file (column key →\n * string). The transform turns one row into a typed `OutputDoc` ready\n * for the bulk-write surface (PoC: `PartsDocument` for the parts ES\n * index). The transform is what makes a generic feed-importer\n * commerce-aware — without it, the importer has no opinion on what a\n * row \"means\".\n *\n * Why an interface, not a function:\n * - The transform may need to reject a row (`RowSkip`) without that\n * counting as an error in `ErrorTracker` — e.g. blank lines, header\n * rows mistakenly retained, or \"this is a replacement-code marker\n * row, handled out-of-band\" decisions.\n * - The transform may need to fail a row (`RowError`) with a typed\n * error class so `ErrorTracker` collapses them by `errorType`.\n * - The transform may need access to the run-level params (brand /\n * supplier IDs, code prefix, batch ID) without those being\n * re-derived per row.\n *\n * PR 7 ships only the interface + a registry. PR 8 ships the first\n * concrete carmaker-feed transform.\n */\n\n/** Identifies which transform a registered handler implements. Stable across deploys. */\nexport type TransformName = string\n\n/** Per-run context provided to every row call. Kept narrow on purpose. */\nexport interface TransformContext {\n /** UUID of the `import_run` row. Forward to the output doc as `import_batch_id` for source attribution (D20). */\n importRunId: string\n /** Opaque per-run params copied from `import_run.params`. The transform validates the shape it expects. */\n params: Record<string, unknown>\n /** Operator-supplied label, useful for logging / debugging. */\n runLabel: string\n /** 1-based row number across the whole feed (header counts as row 0). Forward to `RowError.rowNumber`. */\n rowNumber: number\n}\n\n/**\n * A directional supersession pair harvested from a row as a SIDE-OUTPUT\n * (commerce-demo PR6, SD007). The edge ALWAYS runs `from` (older) → `to` (newer/current).\n *\n * Which end the emitting row occupies depends on which supersession column the mapping used,\n * so neither end can be described as \"the row's own code\": a `superseding_code` column names\n * the newer code and puts the row at `from` (the row is the obsolete one), while a\n * `replaces_code` column names the older code and puts the row at `to` (the row is current).\n * Both codes are already NORMALIZED (same normalization as the doc's `code_normalized`) so a\n * downstream equivalence-group merge collides codes correctly.\n *\n * This is PURE DATA. The runner buffers pairs in memory during the single ES pass\n * (an array append — it never blocks the `bulkUpsert` loop) and hands the buffer to\n * an INJECTED post-run sink (the commerce replacement-merge engine). The\n * `@murumets-ee/imports` leaf never writes the consumer's Postgres tables itself —\n * the boundary holds because the sink is injected, not imported.\n */\nexport interface DirectionalPair {\n /** The superseded (older) normalized code. */\n from: string\n /** The superseding (newer / current) normalized code. */\n to: string\n}\n\n/**\n * A partial-document PATCH emitted as a side-output (commerce-demo PR7): the row\n * itself produces no saleable doc, but it carries a partial update that must land\n * on a DIFFERENT doc — identified by `id` (the same `<code_normalized>__<supplier>`\n * key the parts sink uses). The runner buffers these during the single stream and,\n * AFTER the ES pass, applies them via a phase-2 `bulkPartialUpdate` (an ES `update`\n * merge, realtime by `_id` → order-independent). A patch whose target doesn't exist\n * surfaces as an `orphan_*` warning, never a phantom doc.\n *\n * Canonical use: a `<code>*S` core-fee row (D004) attaches its deposit to the parent\n * part's `core_charge_minor`. Kept GENERIC (`Partial<TDoc>`) so the runner never\n * imports a schema-specific doc type — the same leaf-clean shape as {@link DirectionalPair}.\n */\nexport interface DocPatch<TDoc> {\n /** The `_id` of the doc to merge into (a different row's doc than the one that emitted this). */\n id: string\n /** The partial fields to merge onto that doc. */\n doc: Partial<TDoc>\n /**\n * The warning to record if the target `id` isn't in the index (the phase-2\n * update 404s). The runner detects the orphan (it can't be known at buildRow\n * time — the parent may stream later) but the SCHEMA owns the bucket name +\n * message, so this rides on the patch. Absent → a generic `orphan_patch`\n * fallback. For `*S` core-fee rows the parts schema sets `orphan_core_charge`\n * (D004: warn, never silently drop).\n */\n orphanWarning?: RowWarning\n}\n\n/**\n * One of: a successful `OutputDoc` keyed by a stable `id`, an\n * intentional `skip` (counted in `totals.skipped`), or a row-level\n * error (counted in `totals.failed` AND aggregated by `ErrorTracker`).\n *\n * The `id` field on `success` is used as the bulk-upsert primary key\n * (`_id` in ES). Per D4 the parts index uses\n * `<code_normalized>__<supplier_id>` so a re-run of the same feed\n * idempotently overwrites instead of duplicating.\n *\n * **`warnings` on `success`** (commerce-demo PR4): a row can be a valid,\n * indexable doc AND still carry a non-fatal advisory — e.g. a part imported\n * with `list_price` but no resolvable purchase cost (`no_purchase_price`,\n * phase 03 §5). The runner indexes the doc (it's a success, NOT a failure)\n * and separately aggregates the warnings into a `warningSummary`, so the\n * operator sees \"120 rows cost-pending\" without those rows being dropped or\n * mis-counted as failures. Empty/absent `warnings` is the common case.\n *\n * **`pairs` on `success`** (commerce-demo PR6, SD007): a row can ALSO carry\n * directional supersession {@link DirectionalPair}s as a side-output — the `parts`\n * schema emits one when a row's `superseding_code` is present (a SPARSE column on a\n * full manufacturer catalogue, so most rows emit none). The runner buffers these\n * from ES-accepted rows and returns them for the injected `pairSink`. Empty/absent\n * `pairs` is the common case.\n *\n * **`patches` on `skip`** (commerce-demo PR7): a `<code>*S` core-fee row is NOT a\n * saleable doc (so it `skip`s the main index), but it carries a {@link DocPatch}\n * side-output that attaches its deposit to the PARENT part's `core_charge_minor`.\n * The runner buffers these (sparse — only reman parts carry `*S`) and applies them\n * in a phase-2 `bulkPartialUpdate` AFTER the stream; an unresolved parent becomes an\n * `orphan_core_charge` warning (D004). Empty/absent `patches` is the common case.\n *\n * **`priceSample` on `success`** (F010 / #602): an optional numeric price value,\n * in whatever unit scale the schema itself uses (the `parts` schema emits\n * `purchase_price_minor`, integer minor units), sampled for RUN-LEVEL value-drift\n * detection — see `price-stats.ts`. The runner accumulates these from ES-accepted\n * rows into a `PriceStats` distribution snapshot (min/median/max/nullRate) via the\n * SAME streaming pass that builds `totals`, no second pass over the file. A schema\n * that never emits `priceSample` (or a row with no resolvable price) simply omits\n * it — the collector counts that toward `nullRate` rather than treating it as an\n * error. Absent is the common case for schemas with no price notion at all.\n */\nexport type RowResult<TDoc> =\n | {\n kind: 'success'\n id: string\n doc: TDoc\n warnings?: readonly RowWarning[]\n pairs?: readonly DirectionalPair[]\n priceSample?: number\n }\n | { kind: 'skip'; reason: string; patches?: readonly DocPatch<TDoc>[] }\n | { kind: 'error'; error: RowError }\n\n/** Row-level error. The aggregator uses `(errorType, field, message)` as the dedup signature. */\nexport interface RowError {\n errorType: string\n message: string\n field?: string | undefined\n}\n\n/**\n * A non-fatal advisory attached to a SUCCESSFUL row (the doc is still\n * indexed). Same `(warningType, field, message)` dedup signature shape as\n * {@link RowError} so the runner can aggregate warnings through an\n * `ErrorTracker` instance into `import_run.warningSummary`.\n */\nexport interface RowWarning {\n warningType: string\n message: string\n field?: string | undefined\n}\n\n/**\n * The contract every per-feed transform implements. Pure function of\n * `(row, ctx) → RowResult` — no I/O. Async only because the future\n * carmaker variant might consult an in-memory taxonomy lookup.\n */\nexport type RowTransform<TDoc> = (\n rawRow: Record<string, string>,\n ctx: TransformContext,\n) => Promise<RowResult<TDoc>>\n\n/**\n * Holds the registered transforms for the current process. Each plugin\n * that ships a transform calls {@link TransformRegistry.register} from\n * its `init` hook.\n *\n * Per CLAUDE.md \"Whitelist, don't blacklist\": the worker dispatches\n * by `import_run.transformName` against this registry. An unregistered\n * name fails the run rather than running with a default transform.\n *\n * **Production callers MUST go through {@link getTransformRegistry}**, not\n * `new TransformRegistry()`. The constructor is exposed only so tests\n * (and rare custom-worker embeds) can build an isolated instance and\n * inject it into {@link createRunImportHandler} directly. Two `new`\n * instances do NOT share state — registering a transform on one will\n * NOT make it visible to a worker resolving against the other. The\n * package's queue handler (registered by `imports()` plugin) always\n * resolves against the singleton.\n */\nexport class TransformRegistry {\n private readonly entries = new Map<TransformName, RowTransform<unknown>>()\n\n /**\n * Register a transform under a stable name. Throws on duplicate\n * names so two plugins can't silently overwrite each other (same\n * fail-loud collision protection that `combineAdminRoutes` enforces\n * for duplicate `(prefix, method, path)` route registrations).\n */\n register<TDoc>(name: TransformName, transform: RowTransform<TDoc>): void {\n if (this.entries.has(name)) {\n throw new Error(\n `Transform \"${name}\" is already registered — two plugins cannot register the same transform name.`,\n )\n }\n this.entries.set(name, transform as RowTransform<unknown>)\n }\n\n /**\n * Last-wins registration — overwrites an existing entry instead of\n * throwing. The {@link registerImportTransform} consumer-app path uses\n * this (same semantics as `@murumets-ee/queue/client`'s `registerJob`):\n * under dev HMR a re-evaluated consumer module re-registers against the\n * surviving globalThis registry, and an overwrite picks up the edited\n * function where a skip-or-throw would keep dispatching the stale one\n * (the #186 staleness class). Plugin built-ins keep using\n * {@link register} behind `has()` guards so genuine cross-plugin name\n * collisions still fail loud.\n */\n set<TDoc>(name: TransformName, transform: RowTransform<TDoc>): void {\n this.entries.set(name, transform as RowTransform<unknown>)\n }\n\n /** Look up a transform by name. Returns `undefined` if no plugin has registered one. */\n get(name: TransformName): RowTransform<unknown> | undefined {\n return this.entries.get(name)\n }\n\n /** True iff a transform is registered for the given name. */\n has(name: TransformName): boolean {\n return this.entries.has(name)\n }\n\n /** All registered names. Useful for the admin Catalog tab and for tests. */\n list(): TransformName[] {\n return Array.from(this.entries.keys()).sort()\n }\n\n /** Drop all registrations. Tests only. */\n clear(): void {\n this.entries.clear()\n }\n}\n\n// ---------------------------------------------------------------------------\n// Process-global singleton (mirrors `@murumets-ee/queue`'s handler registry).\n// ---------------------------------------------------------------------------\n//\n// Why a singleton: registrations (the plugin's built-ins at init, plus\n// any consumer-app custom transforms) and the worker's `get` happen in\n// different modules that Next.js HMR / Turbopack can re-evaluate\n// independently. Holding the registry on `globalThis` via `Symbol.for`\n// means the same `Map` instance is shared across module evaluations —\n// `register` and `get` see each other regardless of evaluation order.\n// Same fix shape as the queue's handler-registry HMR bug (#186).\n\nconst REGISTRY_KEY = Symbol.for('@murumets-ee/imports:transforms')\n\ninterface GlobalThisWithTransforms {\n [REGISTRY_KEY]?: TransformRegistry\n}\n\n/** Returns the singleton registry, creating it on first access. */\nexport function getTransformRegistry(): TransformRegistry {\n const g = globalThis as GlobalThisWithTransforms\n let reg = g[REGISTRY_KEY]\n if (!reg) {\n reg = new TransformRegistry()\n g[REGISTRY_KEY] = reg\n }\n return reg\n}\n\n/**\n * Convenience wrapper around the singleton registry for CONSUMER apps that\n * ship their own custom transforms — same semantics as\n * `@murumets-ee/queue/client`'s `registerJob`: **last-wins overwrite, never\n * throws on a duplicate name.** Two reasons, both load-bearing:\n *\n * 1. **Upgrade safety.** The `imports()` plugin registers the built-in\n * transforms (`commerce:mapping`, `commerce:carmaker-feed`) during\n * `createApp`, and older hand-wired apps — following this module's\n * previous docs — call this helper UNGUARDED from their bootstrap right\n * after. A duplicate-name throw here would crash boot on the very\n * `pnpm up` upgrade that ships plugin-owned registration; re-registering\n * the identical stock function is a harmless overwrite instead.\n * 2. **HMR freshness.** The registry lives on globalThis and survives dev\n * module re-evaluation; an edited consumer transform re-registers on the\n * next eval and the overwrite picks up the edit — a skip would keep\n * dispatching the stale function forever (the #186 staleness class).\n *\n * Side effect worth knowing: an app can deliberately override a built-in\n * name (its bootstrap runs after plugin init, so last-wins = app wins).\n * Genuine collision protection (two plugins claiming one name) stays on the\n * raw `TransformRegistry.register`.\n */\nexport function registerImportTransform<TDoc>(\n name: TransformName,\n transform: RowTransform<TDoc>,\n): void {\n getTransformRegistry().set(name, transform)\n}\n"],"mappings":"gXA6BA,MAAa,EAAsB,CACjC,UACA,UACA,YACA,SACA,WACF,EAGa,EAAY,EAAa,CACpC,KAAM,aACN,OAAQ,CAEN,MAAO,EAAM,KAAK,CAAE,SAAU,GAAM,UAAW,GAAI,CAAC,EAEpD,OAAQ,EAAM,OAAO,CACnB,QAAS,EACT,QAAS,UACT,QAAS,EACX,CAAC,EAOD,SAAU,EAAM,KAAK,CAAE,SAAU,GAAM,UAAW,IAAK,CAAC,EAMxD,cAAe,EAAM,KAAK,CAAE,SAAU,GAAM,UAAW,IAAK,QAAS,EAAK,CAAC,EAQ3E,OAAQ,EAAM,KAAK,EAOnB,OAAQ,EAAM,KAAK,EAMnB,aAAc,EAAM,KAAK,EAezB,eAAgB,EAAM,KAAK,EAyB3B,WAAY,EAAM,KAAK,EAEvB,UAAW,EAAM,KAAK,EAEtB,WAAY,EAAM,KAAK,EAKvB,WAAY,EAAM,KAAK,CAAE,UAAW,GAAI,QAAS,EAAK,CAAC,CACzD,EACA,UAAW,CACT,EAAS,UAAU,EAInB,EAAW,CACT,OAAQ,CAAC,QAAS,gBAAiB,aAAc,SAAU,UAAU,EACrE,WAAa,IAAS,CACpB,GAAI,EAAI,GACR,MAAO,EAAI,MACX,YAAa,GAAG,EAAI,cAAc,KAAK,EAAI,QAC7C,EACF,CAAC,CACH,EACA,MAAO,SACP,MAAO,CACL,MAAO,UACP,MAAO,cACP,cAAe,aACf,KAAM,SACN,aAAc,EAChB,CACF,CAAC,ECnIK,EAAwB,CAAE,QAAS,IAAK,UAAW,EAAG,EAKtD,EAAkB,EAAE,MAAM,CAAmB,CAAC,CAAC,IAAI,GAAG,EAgBtD,EAAuB,EAAE,aAAa,CAC1C,QAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,EAC3D,KAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CACpD,CAAC,EA2BD,SAAgB,EAAmB,EAK1B,CAGP,GAAI,EAAM,iBAAmB,IAAA,IAAa,EAAM,iBAAmB,KAAM,CACvE,IAAM,EAAO,EAAqB,UAAU,EAAM,cAAc,EAChE,GAAI,CAAC,EAAK,QACR,MAAU,MAAM,2BAA2B,EAAK,MAAM,OAAO,EAAE,EAAE,SAAW,aAAa,EAM3F,IAAM,EAAa,KAAK,UAAU,EAAM,cAAc,CAAC,CAAC,OACxD,GAAI,EAAA,IACF,MAAU,MACR,6BAA6B,EAAW,qDAC1C,CAEJ,CACA,IAAM,EAAe,OAAO,EAAM,cAAiB,SAAW,EAAM,aAAe,GAC7E,EAAS,EAAgB,CAAY,EAC3C,GAAI,CAAC,EACH,MAAU,MACR,0BAA0B,EAAa,iBAAiB,EAAkB,CAAC,CAAC,KAAK,IAAI,EAAE,EACzF,EAGF,IAAM,EAAK,EAAmB,UAAU,EAAM,YAAY,EAC1D,GAAI,CAAC,EAAG,QACN,MAAU,MAAM,yBAAyB,EAAG,MAAM,OAAO,EAAE,EAAE,SAAW,aAAa,EAGvF,IAAM,EAAK,EAAgB,UAAU,EAAM,SAAS,EACpD,GAAI,CAAC,EAAG,QACN,MAAU,MAAM,sBAAsB,EAAG,MAAM,OAAO,EAAE,EAAE,SAAW,aAAa,EAEpF,IAAM,EAAsB,KAAK,UAAU,EAAM,SAAS,CAAC,CAAC,OAC5D,GAAI,EAAA,IACF,MAAU,MACR,wBAAwB,EAAoB,4BAC9C,EAGF,IAAM,EAAU,IAAI,IAAI,EAAO,OAAO,IAAK,GAAM,EAAE,GAAG,CAAC,EACjD,EAAc,IAAI,IAClB,EAAc,IAAI,IACxB,IAAK,IAAM,KAAO,EAAG,KAAM,CACzB,GAAI,CAAC,EAAQ,IAAI,EAAI,MAAM,EACzB,MAAU,MACR,mBAAmB,EAAI,OAAO,8BAA8B,EAAO,KAAK,cACzD,CAAC,GAAG,CAAO,CAAC,CAAC,KAAK,IAAI,EAAE,EACzC,EAME,EAAY,IAAI,EAAI,MAAM,GAAG,EAAY,IAAI,EAAI,MAAM,EAC3D,EAAY,IAAI,EAAI,MAAM,CAC5B,CACA,GAAI,EAAY,KAAO,EACrB,MAAU,MACR,uBAAuB,EAAO,KAAK,kCAAkC,CAAC,GAAG,CAAW,CAAC,CAAC,KAAK,IAAI,GACjG,EAGF,IAAM,EAAgB,IAAI,IAAI,EAAG,KAAK,IAAK,GAAM,EAAE,MAAM,CAAC,EACpD,EAAU,EAAO,OACpB,OAAQ,GAAM,EAAE,UAAY,CAAC,EAAc,IAAI,EAAE,GAAG,CAAC,CAAC,CACtD,IAAK,GAAM,EAAE,GAAG,EACnB,GAAI,EAAQ,OAAS,EACnB,MAAU,MACR,uBAAuB,EAAO,KAAK,mCAAmC,EAAQ,KAAK,IAAI,GACzF,CAEJ,CAQA,MAAa,EAGT,CACF,KAAM,oCACN,MAAO,CACL,aAAc,KAAO,KAInB,EAAmB,CACjB,aAAc,EAAK,aACnB,aAAc,EAAK,cAAgB,EACnC,UAAW,EAAK,WAAa,CAAC,EAC9B,eAAgB,EAAK,cACvB,CAAC,EACM,GAET,aAAc,MAAO,EAAK,EAAM,IAAQ,CAItC,IAAM,EAAW,MAAM,GAAK,cAAc,GAAM,CAAC,EAYjD,OAXA,EAAmB,CACjB,aAAc,iBAAkB,EAAO,EAAK,aAAe,EAAQ,aACnE,aACE,iBAAkB,EACd,EAAK,aACJ,EAAQ,cAAgB,EAC/B,UAAW,cAAe,EAAO,EAAK,UAAa,EAAQ,WAAa,CAAC,EAGzE,eAAgB,mBAAoB,EAAO,EAAK,eAAiB,IAAA,EACnE,CAAC,EACM,CACT,CACF,CACF,ECjKa,EAAgC,CAAC,QAAS,gBAAgB,EAI1D,EAAyB,CAAC,MAAO,MAAO,MAAM,EAiD9C,EAAgB,EAAa,CACxC,KAAM,iBACN,MAAO,SACP,OAAQ,CAEN,WAAY,EAAM,UAAU,CAAE,OAAQ,WAAY,SAAU,GAAM,QAAS,EAAK,CAAC,EAEjF,QAAS,EAAM,UAAU,CAAE,OAAQ,QAAS,QAAS,EAAK,CAAC,EAE3D,aAAc,EAAM,OAAO,CACzB,QAAS,EACT,SAAU,GACV,QAAS,EACX,CAAC,EAED,KAAM,EAAM,KAAK,CAAE,SAAU,GAAM,UAAW,GAAI,CAAC,EAEnD,OAAQ,EAAM,OAAO,CAAE,QAAS,EAAwB,SAAU,EAAK,CAAC,EAKxE,UAAW,EAAM,KAAK,CAAE,UAAW,CAAE,CAAC,EAEtC,UAAW,EAAM,QAAQ,CAAE,QAAS,EAAK,CAAC,EAE1C,eAAgB,EAAM,OAAO,CAAE,QAAS,GAAM,IAAK,EAAG,QAAS,CAAE,CAAC,EAElE,MAAO,EAAM,KAAK,CAAE,UAAW,GAAI,CAAC,EAMpC,aAAc,EAAM,KAAK,CAAE,QAAS,CAAE,QAAS,IAAK,UAAW,EAAG,CAAE,CAAC,EAQrE,UAAW,EAAM,KAAK,CAAE,QAAS,CAAC,CAAE,CAAC,EAarC,eAAgB,EAAM,KAAK,EAE3B,OAAQ,EAAM,QAAQ,CAAE,QAAS,GAAM,QAAS,EAAK,CAAC,CACxD,EACA,UAAW,CACT,EAAS,UAAU,EAInB,EAAW,CACT,OAAQ,CAAC,OAAQ,eAAgB,QAAQ,EACzC,WAAa,IAAS,CACpB,GAAI,EAAI,GACR,MAAO,EAAI,KACX,YAAa,GAAG,EAAI,aAAa,KAAK,EAAI,QAC5C,EACF,CAAC,EACD,CAjGF,KAAM,gCACN,aAAc,EAAiC,IAA2C,CACxF,GAAI,IAAW,OAAQ,MAAO,CAAC,EAC/B,IAAM,EAAa,EAAM,WACnB,EAAU,EAAM,QAChB,EAAe,EAAM,aAG3B,MADI,CAAC,GAAc,CAAC,GAAW,CAAC,EAAqB,CAAC,EAC/C,CACL,CACE,KAAM,cACN,KAAM,0CACN,GAAI,CAAC,EAAY,EAAS,CAAY,EACtC,MAAO,CAAG,GAAG,EAAQ,aACvB,EACA,CACE,KAAM,cACN,KAAM,4CACN,GAAI,CAAC,EAAY,CAAY,EAC7B,MAAO,CAAG,GAAG,EAAQ,SACvB,CACF,CACF,CA2EE,EAEA,CACF,EACA,MAAO,CACL,MAAO,UACP,MAAO,kBACP,cAAe,iBACf,KAAM,QACN,aAAc,EAChB,CACF,CAAC,ECmBD,IAAa,EAAb,KAA+B,CAC7B,QAA2B,IAAI,IAQ/B,SAAe,EAAqB,EAAqC,CACvE,GAAI,KAAK,QAAQ,IAAI,CAAI,EACvB,MAAU,MACR,cAAc,EAAK,+EACrB,EAEF,KAAK,QAAQ,IAAI,EAAM,CAAkC,CAC3D,CAaA,IAAU,EAAqB,EAAqC,CAClE,KAAK,QAAQ,IAAI,EAAM,CAAkC,CAC3D,CAGA,IAAI,EAAwD,CAC1D,OAAO,KAAK,QAAQ,IAAI,CAAI,CAC9B,CAGA,IAAI,EAA8B,CAChC,OAAO,KAAK,QAAQ,IAAI,CAAI,CAC9B,CAGA,MAAwB,CACtB,OAAO,MAAM,KAAK,KAAK,QAAQ,KAAK,CAAC,CAAC,CAAC,KAAK,CAC9C,CAGA,OAAc,CACZ,KAAK,QAAQ,MAAM,CACrB,CACF,EAcA,MAAM,EAAe,OAAO,IAAI,iCAAiC,EAOjE,SAAgB,GAA0C,CACxD,IAAM,EAAI,WACN,EAAM,EAAE,GAKZ,OAJK,IACH,EAAM,IAAI,EACV,EAAE,GAAgB,GAEb,CACT,CAyBA,SAAgB,EACd,EACA,EACM,CACN,EAAqB,CAAC,CAAC,IAAI,EAAM,CAAS,CAC5C"}
|
|
@@ -118,6 +118,16 @@ interface DocPatch<TDoc> {
|
|
|
118
118
|
* The runner buffers these (sparse — only reman parts carry `*S`) and applies them
|
|
119
119
|
* in a phase-2 `bulkPartialUpdate` AFTER the stream; an unresolved parent becomes an
|
|
120
120
|
* `orphan_core_charge` warning (D004). Empty/absent `patches` is the common case.
|
|
121
|
+
*
|
|
122
|
+
* **`priceSample` on `success`** (F010 / #602): an optional numeric price value,
|
|
123
|
+
* in whatever unit scale the schema itself uses (the `parts` schema emits
|
|
124
|
+
* `purchase_price_minor`, integer minor units), sampled for RUN-LEVEL value-drift
|
|
125
|
+
* detection — see `price-stats.ts`. The runner accumulates these from ES-accepted
|
|
126
|
+
* rows into a `PriceStats` distribution snapshot (min/median/max/nullRate) via the
|
|
127
|
+
* SAME streaming pass that builds `totals`, no second pass over the file. A schema
|
|
128
|
+
* that never emits `priceSample` (or a row with no resolvable price) simply omits
|
|
129
|
+
* it — the collector counts that toward `nullRate` rather than treating it as an
|
|
130
|
+
* error. Absent is the common case for schemas with no price notion at all.
|
|
121
131
|
*/
|
|
122
132
|
type RowResult<TDoc> = {
|
|
123
133
|
kind: 'success';
|
|
@@ -125,6 +135,7 @@ type RowResult<TDoc> = {
|
|
|
125
135
|
doc: TDoc;
|
|
126
136
|
warnings?: readonly RowWarning[];
|
|
127
137
|
pairs?: readonly DirectionalPair[];
|
|
138
|
+
priceSample?: number;
|
|
128
139
|
} | {
|
|
129
140
|
kind: 'skip';
|
|
130
141
|
reason: string;
|
|
@@ -232,4 +243,4 @@ declare function getTransformRegistry(): TransformRegistry;
|
|
|
232
243
|
declare function registerImportTransform<TDoc>(name: TransformName, transform: RowTransform<TDoc>): void;
|
|
233
244
|
//#endregion
|
|
234
245
|
export { RowWarning as a, TransformRegistry as c, RowTransform as i, getTransformRegistry as l, RowError as n, TransformContext as o, RowResult as r, TransformName as s, DirectionalPair as t, registerImportTransform as u };
|
|
235
|
-
//# sourceMappingURL=transform-
|
|
246
|
+
//# sourceMappingURL=transform-WeiiLdYP.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transform-WeiiLdYP.d.mts","names":[],"sources":["../src/transform.ts"],"mappings":";;AA0BA;;;;AAAyB;AAGzB;;;;;;;;;;AAQW;AAoBX;;;;AAII;AAgBJ;;;KAnDY,aAAA;;UAGK,gBAAA;EA6DW;EA3D1B,WAAA;EA8CwB;EA5CxB,MAAA,EAAQ,MAAM;EAgDd;EA9CA,QAAA;EA8Ca;EA5Cb,SAAA;AAAA;;AAqD0B;AA6C5B;;;;;;;;;;;;;;;UA9EiB,eAAA;EAmFX;EAjFJ,IAAA;EAkFI;EAhFJ,EAAE;AAAA;;;;;;;;;;AAoFgC;AAGpC;;;UAvEiB,QAAA;EAwEf;EAtEA,EAAA;EAwEA;EAtEA,GAAA,EAAK,OAAA,CAAQ,IAAA;EAsER;AASP;;;;;;;EAtEE,aAAA,GAAgB,UAAA;AAAA;AAiFlB;;;;;;;;;;;;;;;;;;;AAG2B;AAoB3B;;;;;;;;;;;;;;;;;;;;;;AAvBA,KApCY,SAAA;EAEN,IAAA;EACA,EAAA;EACA,GAAA,EAAK,IAAA;EACL,QAAA,YAAoB,UAAA;EACpB,KAAA,YAAiB,eAAA;EACjB,WAAA;AAAA;EAEA,IAAA;EAAc,MAAA;EAAgB,OAAA,YAAmB,QAAA,CAAS,IAAA;AAAA;EAC1D,IAAA;EAAe,KAAA,EAAO,QAAA;AAAA;;UAGX,QAAA;EACf,SAAA;EACA,OAAA;EACA,KAAA;AAAA;;;;AAoHuD;AAiCzD;;UA5IiB,UAAA;EACf,WAAA;EACA,OAAA;EACA,KAAA;AAAA;;;;;;KAQU,YAAA,UACV,MAAA,EAAQ,MAAA,kBACR,GAAA,EAAK,gBAAA,KACF,OAAA,CAAQ,SAAA,CAAU,IAAA;;;AAgIQ;;;;;;;;;;;;;;;;cA5GlB,iBAAA;EAAA,iBACM,OAAA;;;;;;;EAQjB,QAAA,MAAA,CAAe,IAAA,EAAM,aAAA,EAAe,SAAA,EAAW,YAAA,CAAa,IAAA;;;;;;;;;;;;EAoB5D,GAAA,MAAA,CAAU,IAAA,EAAM,aAAA,EAAe,SAAA,EAAW,YAAA,CAAa,IAAA;;EAKvD,GAAA,CAAI,IAAA,EAAM,aAAA,GAAgB,YAAA;;EAK1B,GAAA,CAAI,IAAA,EAAM,aAAA;;EAKV,IAAA,CAAA,GAAQ,aAAA;;EAKR,KAAA,CAAA;AAAA;;iBAwBc,oBAAA,CAAA,GAAwB,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;iBAiCzC,uBAAA,MAAA,CACd,IAAA,EAAM,aAAA,EACN,SAAA,EAAW,YAAA,CAAa,IAAA"}
|
package/dist/transforms.d.mts
CHANGED
|
@@ -1,46 +1,2 @@
|
|
|
1
|
-
import { i as
|
|
2
|
-
|
|
3
|
-
import { PartsDocument } from "@murumets-ee/search-elasticsearch";
|
|
4
|
-
|
|
5
|
-
//#region src/transforms/carmaker-feed.d.ts
|
|
6
|
-
/** Stable name under which this transform is registered. */
|
|
7
|
-
declare const CARMAKER_FEED_TRANSFORM_NAME = "commerce:carmaker-feed";
|
|
8
|
-
/**
|
|
9
|
-
* Per-job params — the per-(supplier, brand) identity resolved at enqueue time.
|
|
10
|
-
* This IS the parts schema's params (re-exported), so brand/supplier injection +
|
|
11
|
-
* the optional `supplierCodePrefix` strip validate identically everywhere.
|
|
12
|
-
*/
|
|
13
|
-
declare const carmakerFeedParamsSchema: import("zod").ZodObject<{
|
|
14
|
-
brandId: import("zod").ZodString;
|
|
15
|
-
brandSlug: import("zod").ZodString;
|
|
16
|
-
supplierId: import("zod").ZodString;
|
|
17
|
-
supplierDisplayName: import("zod").ZodString;
|
|
18
|
-
importBatchId: import("zod").ZodString;
|
|
19
|
-
supplierCodePrefix: import("zod").ZodOptional<import("zod").ZodString>;
|
|
20
|
-
discountGroups: import("zod").ZodOptional<import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodNumber>>;
|
|
21
|
-
}, "strip", import("zod").ZodTypeAny, {
|
|
22
|
-
supplierId: string;
|
|
23
|
-
brandId: string;
|
|
24
|
-
brandSlug: string;
|
|
25
|
-
supplierDisplayName: string;
|
|
26
|
-
importBatchId: string;
|
|
27
|
-
supplierCodePrefix?: string | undefined;
|
|
28
|
-
discountGroups?: Record<string, number> | undefined;
|
|
29
|
-
}, {
|
|
30
|
-
supplierId: string;
|
|
31
|
-
brandId: string;
|
|
32
|
-
brandSlug: string;
|
|
33
|
-
supplierDisplayName: string;
|
|
34
|
-
importBatchId: string;
|
|
35
|
-
supplierCodePrefix?: string | undefined;
|
|
36
|
-
discountGroups?: Record<string, number> | undefined;
|
|
37
|
-
}>;
|
|
38
|
-
type CarmakerFeedParams = PartsParams;
|
|
39
|
-
/**
|
|
40
|
-
* The carmaker transform — built once at module load from the static mapping. The
|
|
41
|
-
* build validates every target against the `parts` whitelist (throws on a typo).
|
|
42
|
-
*/
|
|
43
|
-
declare const carmakerFeedTransform: RowTransform<PartsDocument>;
|
|
44
|
-
//#endregion
|
|
45
|
-
export { CARMAKER_FEED_TRANSFORM_NAME, type CarmakerFeedParams, carmakerFeedParamsSchema, carmakerFeedTransform };
|
|
46
|
-
//# sourceMappingURL=transforms.d.mts.map
|
|
1
|
+
import { a as carmakerFeedTransform, i as carmakerFeedParamsSchema, r as CarmakerFeedParams, t as CARMAKER_FEED_TRANSFORM_NAME } from "./carmaker-feed-DlD2xPs0.mjs";
|
|
2
|
+
export { CARMAKER_FEED_TRANSFORM_NAME, type CarmakerFeedParams, carmakerFeedParamsSchema, carmakerFeedTransform };
|
package/dist/transforms.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{i as e,r as t,t as n}from"./carmaker-feed-C_MnDaOC.mjs";export{n as CARMAKER_FEED_TRANSFORM_NAME,t as carmakerFeedParamsSchema,e as carmakerFeedTransform};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { c as TransformRegistry, i as RowTransform, t as DirectionalPair } from "./transform-
|
|
1
|
+
import { c as TransformRegistry, i as RowTransform, t as DirectionalPair } from "./transform-WeiiLdYP.mjs";
|
|
2
2
|
import { Logger } from "@murumets-ee/core";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { EsClientLike } from "@murumets-ee/search-elasticsearch";
|
|
@@ -86,7 +86,32 @@ declare const ImportRun: import("@murumets-ee/entity").Entity<{
|
|
|
86
86
|
* JSON reuses its field names — the `totalErrors` key inside this blob is
|
|
87
87
|
* really the WARNING count (and `patterns[].errorType` the warning type).
|
|
88
88
|
*/
|
|
89
|
-
warningSummary: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>;
|
|
89
|
+
warningSummary: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>;
|
|
90
|
+
/**
|
|
91
|
+
* Price-distribution snapshot (F010 / issue #602) — `{ count, nullRate,
|
|
92
|
+
* min, median, max }`, computed over ES-accepted rows' derived purchase
|
|
93
|
+
* price (`PartsDocument.purchase_price_minor`, integer minor units) in
|
|
94
|
+
* the SAME streaming pass that builds `totals`. See
|
|
95
|
+
* `@murumets-ee/imports`' `price-stats.ts` for the exact shape and the
|
|
96
|
+
* bounded-sample cap.
|
|
97
|
+
*
|
|
98
|
+
* The pipeline's existing 6b header-drift check (commerce's
|
|
99
|
+
* register-handler.ts) only catches STRUCTURAL drift — a mapped column
|
|
100
|
+
* vanishing from the header row. It does NOT catch a supplier silently
|
|
101
|
+
* flipping their decimal/thousands separator, which changes no header
|
|
102
|
+
* at all and produces numerically valid but silently WRONG prices. The
|
|
103
|
+
* worker compares THIS run's `priceStats.median` against the most
|
|
104
|
+
* recent prior SUCCEEDED run for the same (transformName, brandId,
|
|
105
|
+
* supplierId) and, on a magnitude-scale shift (>=10x or <=0.1x — see
|
|
106
|
+
* `PRICE_ANOMALY_THRESHOLD_MULTIPLIER`), appends a NON-FATAL
|
|
107
|
+
* `price_anomaly` entry to {@link warningSummary}. Unlike the 6b 409,
|
|
108
|
+
* this never blocks the run — a real repricing is a legitimate reason
|
|
109
|
+
* prices would shift, so it's an advisory only.
|
|
110
|
+
*
|
|
111
|
+
* `null`/absent for runs written before this field existed, or for a
|
|
112
|
+
* schema/transform whose rows never carry a price sample.
|
|
113
|
+
*/
|
|
114
|
+
priceStats: import("@murumets-ee/entity").JsonField & Partial<import("@murumets-ee/entity").JsonField>; /** Set when the worker picks up the job. */
|
|
90
115
|
startedAt: import("@murumets-ee/entity").DateField & Partial<import("@murumets-ee/entity").DateField>; /** Set when the worker finishes (success OR fatal failure). */
|
|
91
116
|
finishedAt: import("@murumets-ee/entity").DateField & Partial<import("@murumets-ee/entity").DateField>;
|
|
92
117
|
/**
|
|
@@ -371,6 +396,151 @@ interface ErrorTrackerSnapshot {
|
|
|
371
396
|
droppedSignatures: number;
|
|
372
397
|
patterns: ErrorPattern[];
|
|
373
398
|
}
|
|
399
|
+
/**
|
|
400
|
+
* Merge two independently-produced `ErrorTrackerSnapshot`s into one — e.g.
|
|
401
|
+
* the runner's per-row `warningSummary` plus a worker-level signal computed
|
|
402
|
+
* AFTER the run completes (F010 / #602: the price-drift anomaly check runs
|
|
403
|
+
* in `worker.ts`, after `runImport` returns, using a fresh single-pattern
|
|
404
|
+
* tracker of its own). Patterns concatenate rather than re-dedup: the two
|
|
405
|
+
* snapshots are assumed to use disjoint `errorType` signature spaces (a
|
|
406
|
+
* worker-level tracker uses a type — `price_anomaly` — the runner's own
|
|
407
|
+
* warning aggregation never emits), so counts sum directly with no risk of
|
|
408
|
+
* double-counting one bucket as two.
|
|
409
|
+
*/
|
|
410
|
+
declare function mergeErrorTrackerSnapshots(a: ErrorTrackerSnapshot, b: ErrorTrackerSnapshot): ErrorTrackerSnapshot;
|
|
411
|
+
//#endregion
|
|
412
|
+
//#region src/price-stats.d.ts
|
|
413
|
+
/**
|
|
414
|
+
* Price-distribution stats + prior-run anomaly detection (F010 / issue #602).
|
|
415
|
+
*
|
|
416
|
+
* The import pipeline's existing anomaly detection — register-handler.ts's
|
|
417
|
+
* "6b. Header-presence drift check" — only catches STRUCTURAL drift: a
|
|
418
|
+
* mapped column vanishing (or getting renamed) from the header row. It does
|
|
419
|
+
* NOT catch VALUE drift: a supplier silently flipping their decimal
|
|
420
|
+
* separator (e.g. `1.234,56` → `1234.56`) or thousands separator WITHOUT
|
|
421
|
+
* touching a single column header. Since the saved
|
|
422
|
+
* `import_mapping.numberFormat` (`{decimal, thousands}`) is unchanged,
|
|
423
|
+
* every price in that run parses as a NUMERICALLY VALID but WRONG value —
|
|
424
|
+
* silently corrupting the catalog's prices. Nothing upstream of this module
|
|
425
|
+
* flags it.
|
|
426
|
+
*
|
|
427
|
+
* This module computes a per-run price-distribution snapshot (`PriceStats`)
|
|
428
|
+
* over `PartsDocument.purchase_price_minor` — the DERIVED cost field,
|
|
429
|
+
* rather than the raw net/list price columns, because both the direct-net
|
|
430
|
+
* AND two-file derivation paths in `mapping/parts-schema.ts` (step 4) funnel
|
|
431
|
+
* through it, so a decimal-format flip on EITHER source column surfaces
|
|
432
|
+
* here regardless of which pricing path a supplier's feed uses.
|
|
433
|
+
*
|
|
434
|
+
* Collection (`PriceStatsCollector`) happens in the SAME streaming pass
|
|
435
|
+
* `runner.ts` already uses to build `totals` — one `.record()` call per
|
|
436
|
+
* ES-accepted row, no second pass over the file, no extra DB round-trip.
|
|
437
|
+
* The prior-run comparison (`detectPriceAnomaly`) happens in `worker.ts`
|
|
438
|
+
* AFTER the run completes, because it needs `AdminClient` to look the
|
|
439
|
+
* prior run up — this module stays DB-agnostic, pure data + pure functions.
|
|
440
|
+
*/
|
|
441
|
+
/** A run's price-distribution snapshot. Values are in MINOR units (cents), matching `PartsDocument.purchase_price_minor`'s own scale. */
|
|
442
|
+
interface PriceStats {
|
|
443
|
+
/** ES-accepted rows considered — same accounting as `RunImportResult.rowsSucceeded`. */
|
|
444
|
+
count: number;
|
|
445
|
+
/** Fraction (0..1) of `count` whose price sample was absent (e.g. a list-only row with no resolvable purchase cost — the same rows that carry a `no_purchase_price` warning). `0` when `count` is `0`. */
|
|
446
|
+
nullRate: number;
|
|
447
|
+
/** Minimum of the non-null samples, in minor units. `null` when there are zero non-null samples. */
|
|
448
|
+
min: number | null;
|
|
449
|
+
/** Median of the non-null samples, in minor units. `null` when there are zero non-null samples. */
|
|
450
|
+
median: number | null;
|
|
451
|
+
/** Maximum of the non-null samples, in minor units. `null` when there are zero non-null samples. */
|
|
452
|
+
max: number | null;
|
|
453
|
+
}
|
|
454
|
+
/** The all-zero/empty snapshot — a run that produced zero ES-accepted rows. */
|
|
455
|
+
declare const EMPTY_PRICE_STATS: PriceStats;
|
|
456
|
+
/**
|
|
457
|
+
* Bounded sample cap for the min/median/max computation — the same
|
|
458
|
+
* "bounded, uniform sample" philosophy `ErrorTracker` uses top-N patterns
|
|
459
|
+
* for. 50,000 floats is ~400KB, trivial next to the ES bulk-write buffers
|
|
460
|
+
* already in flight for a run this size, and far more than needed for a
|
|
461
|
+
* stable median estimate. A feed with more ES-accepted rows than this cap
|
|
462
|
+
* still gets an EXACT `count` / `nullRate` (those are running counters, not
|
|
463
|
+
* sample-bounded) — only min/median/max become an approximation over a
|
|
464
|
+
* UNIFORM RANDOM sample of `PRICE_SAMPLE_CAP` non-null values, not "the
|
|
465
|
+
* first N encountered" (see `record`'s reservoir-sampling note for why
|
|
466
|
+
* order-independence matters specifically for this module's job).
|
|
467
|
+
*/
|
|
468
|
+
declare const PRICE_SAMPLE_CAP = 50000;
|
|
469
|
+
/**
|
|
470
|
+
* Accumulates price samples during a run's streaming pass. `record` is O(1)
|
|
471
|
+
* (amortized array push below the cap, one RNG draw above it); `snapshot`
|
|
472
|
+
* sorts the retained samples once at the end — cheap relative to the ES
|
|
473
|
+
* bulk-write the run already performs every batch.
|
|
474
|
+
*/
|
|
475
|
+
declare class PriceStatsCollector {
|
|
476
|
+
private count;
|
|
477
|
+
private nullCount;
|
|
478
|
+
private nonNullSeen;
|
|
479
|
+
private readonly samples;
|
|
480
|
+
/**
|
|
481
|
+
* Record one ES-accepted row's price sample. Call exactly once per
|
|
482
|
+
* accepted row (mirrors the `rowsWarned` / `pairs` accounting in
|
|
483
|
+
* `runner.ts`'s flush loop) — `undefined` means "this row had no price to
|
|
484
|
+
* sample" and counts toward `nullRate` rather than being skipped.
|
|
485
|
+
*
|
|
486
|
+
* Reservoir sampling (Algorithm R) past the cap, not "first N wins": each
|
|
487
|
+
* new non-null value replaces a uniformly-random existing slot with
|
|
488
|
+
* probability `PRICE_SAMPLE_CAP / nonNullSeen`, giving every non-null
|
|
489
|
+
* sample seen so far an equal chance of surviving into the final
|
|
490
|
+
* reservoir. Order-independence matters here specifically — the anomaly
|
|
491
|
+
* this module exists to catch (a supplier's decimal-format flip) could
|
|
492
|
+
* start partway through a feed larger than the cap (e.g. a corrupted
|
|
493
|
+
* re-export appended to an otherwise-normal catalogue), and a first-N
|
|
494
|
+
* sample would never see a single row past the cutover point, silently
|
|
495
|
+
* missing exactly the drift F010 was built to detect.
|
|
496
|
+
*/
|
|
497
|
+
record(value: number | undefined): void;
|
|
498
|
+
snapshot(): PriceStats;
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Anomaly threshold: a run whose median purchase price is >= 10x higher OR
|
|
502
|
+
* <= 1/10 (0.1x) of the prior successful run's median (for the same
|
|
503
|
+
* transform + supplier + brand) is flagged.
|
|
504
|
+
*
|
|
505
|
+
* Chosen generously above the range a real repricing plausibly covers
|
|
506
|
+
* (even an aggressive markup/discount renegotiation is a low-single-digit
|
|
507
|
+
* multiplier — 2x-3x would already be an extreme outlier for a legitimate
|
|
508
|
+
* price change) but comfortably BELOW the ~100x-1000x magnitude a
|
|
509
|
+
* decimal/thousands-separator flip produces for typical EUR part prices:
|
|
510
|
+
* `"1.234,56"` (a European-format €1234.56) misparsed with dot-decimal
|
|
511
|
+
* conventions reads as `1.234,56` → `1234.56`, a ~1000x inflation if the
|
|
512
|
+
* true value was e.g. €1.23; a thousands-separator flip on `"1234,56"`
|
|
513
|
+
* (true €1234.56) misparsed as `123456` (dropping the decimal entirely) is
|
|
514
|
+
* a ~100x inflation. 10x leaves a wide safety margin against false
|
|
515
|
+
* positives on genuine price changes while still comfortably catching
|
|
516
|
+
* every realistic decimal/thousands-separator flip.
|
|
517
|
+
*/
|
|
518
|
+
declare const PRICE_ANOMALY_THRESHOLD_MULTIPLIER = 10;
|
|
519
|
+
interface PriceAnomaly {
|
|
520
|
+
/** `curr.median / prev.median`. */
|
|
521
|
+
ratio: number;
|
|
522
|
+
/** The prior run's median (minor units). */
|
|
523
|
+
prevMedian: number;
|
|
524
|
+
/** This run's median (minor units). */
|
|
525
|
+
currMedian: number;
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* Compare two runs' price stats for a magnitude-scale anomaly. Returns
|
|
529
|
+
* `null` when there's nothing to compare (either side has no median — zero
|
|
530
|
+
* non-null price samples, e.g. a schema/transform that never emits a price
|
|
531
|
+
* sample, or a run with zero ES-accepted rows) or the ratio falls within
|
|
532
|
+
* the safe band.
|
|
533
|
+
*/
|
|
534
|
+
declare function detectPriceAnomaly(prev: PriceStats, curr: PriceStats): PriceAnomaly | null;
|
|
535
|
+
/**
|
|
536
|
+
* Narrow an `unknown` value (a `field.json()` column read back through
|
|
537
|
+
* `AdminClient`, typed `JsonValue | undefined` on the DTO) to `PriceStats`,
|
|
538
|
+
* or `null` when it doesn't structurally match — e.g. a run written before
|
|
539
|
+
* this field existed, or a run whose transform never emitted a price
|
|
540
|
+
* sample so `min`/`median`/`max` serialised as `null`. Defensive, matching
|
|
541
|
+
* the `params` narrowing pattern already used in `worker.ts`.
|
|
542
|
+
*/
|
|
543
|
+
declare function parsePriceStats(value: unknown): PriceStats | null;
|
|
374
544
|
//#endregion
|
|
375
545
|
//#region src/runner.d.ts
|
|
376
546
|
/** Soft default; chosen to match giga-test. ES bulk requests over ~5MB get split server-side anyway. */
|
|
@@ -412,6 +582,14 @@ interface RunImportOptions<TDoc> {
|
|
|
412
582
|
* doc is indexed. Default: a fresh tracker with the same caps.
|
|
413
583
|
*/
|
|
414
584
|
warningTracker?: ErrorTracker;
|
|
585
|
+
/**
|
|
586
|
+
* Optional price-stats collector (F010 / #602) — accumulates a
|
|
587
|
+
* min/median/max/nullRate distribution over ES-accepted rows' optional
|
|
588
|
+
* `priceSample`s in the SAME streaming pass that builds `totals`. Default:
|
|
589
|
+
* a fresh collector. Tests can inject their own to inspect it directly —
|
|
590
|
+
* mirrors the {@link errorTracker} / {@link warningTracker} DI shape.
|
|
591
|
+
*/
|
|
592
|
+
priceStats?: PriceStatsCollector;
|
|
415
593
|
/**
|
|
416
594
|
* Optional structural logger for per-batch telemetry. Receives
|
|
417
595
|
* `{ batch, ms, succeeded, failed, rowsRead, rowsPerSecond }` after
|
|
@@ -511,6 +689,13 @@ interface RunImportResult {
|
|
|
511
689
|
*/
|
|
512
690
|
patchesApplied: number;
|
|
513
691
|
patchesOrphaned: number;
|
|
692
|
+
/**
|
|
693
|
+
* Price-distribution snapshot (F010 / #602) accumulated from ES-accepted
|
|
694
|
+
* rows' `priceSample`s during this same streaming pass. See `price-stats.ts`.
|
|
695
|
+
* `count` is `0` (all fields `null`/`0`) for a run whose schema never emits
|
|
696
|
+
* a price sample.
|
|
697
|
+
*/
|
|
698
|
+
priceStats: PriceStats;
|
|
514
699
|
}
|
|
515
700
|
/**
|
|
516
701
|
* Apply the runner against a feed file. Stops on rowLimit OR end-of-file
|
|
@@ -735,5 +920,5 @@ interface ImportsRunDeadListenerConfig {
|
|
|
735
920
|
*/
|
|
736
921
|
declare function createImportsRunDeadListener(config: ImportsRunDeadListenerConfig): QueueTerminalNotifier;
|
|
737
922
|
//#endregion
|
|
738
|
-
export {
|
|
739
|
-
//# sourceMappingURL=worker-
|
|
923
|
+
export { ErrorSample as A, StreamFeedOptions as B, PriceStats as C, DEFAULT_MAX_PATTERNS as D, parsePriceStats as E, FeedReaderConfig as F, ImportRun as G, countDataRows as H, ReadFeedOptions as I, ImportRunClient as K, feedReaderConfigSchema as L, ErrorTrackerConfig as M, ErrorTrackerSnapshot as N, DEFAULT_MAX_SAMPLES_PER_PATTERN as O, mergeErrorTrackerSnapshots as P, readFeed as R, PriceAnomaly as S, detectPriceAnomaly as T, streamFeed as U, StreamFeedRow as V, IMPORT_RUN_STATUSES as W, RunImportResult as _, ImportsRunJobContext as a, PRICE_ANOMALY_THRESHOLD_MULTIPLIER as b, PairSinkContext as c, createRunImportHandler as d, importsRunJob as f, RunImportOptions as g, ImportRunProgress as h, ImportsRunDeadListenerConfig as i, ErrorTracker as j, ErrorPattern as k, RunImportHandlerConfig as l, DEFAULT_BATCH_SIZE as m, EsClientResolver as n, ImportsRunJobPayload as o, importsRunJobPayloadSchema as p, ImportRunStatus as q, FilePathResolver as r, PairSink as s, EnsureIndexResolver as t, createImportsRunDeadListener as u, runImport as v, PriceStatsCollector as w, PRICE_SAMPLE_CAP as x, EMPTY_PRICE_STATS as y, readPersistedFeedConfig as z };
|
|
924
|
+
//# sourceMappingURL=worker-DH3SN-91.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-DH3SN-91.d.mts","names":[],"sources":["../src/entities/import-run.ts","../src/streaming.ts","../src/read-feed.ts","../src/error-tracker.ts","../src/price-stats.ts","../src/runner.ts","../src/worker.ts"],"mappings":";;;;;;;;AAoCyD;AAEzD;;;;;;;;AAFyD,cAP5C,mBAAA;AAAA,KAOD,eAAA,WAA0B,mBAAmB;AAAA,cAE5C,SAAA,gCAAS,MAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuHtB;;;;;;;;AAAoE;;;;AC9FpE;;;;;;;;;AAGU;AAkBV;;;;;;;;;;;;0GA6CiB;;;EAEf;;;;;;;;;KD0BU,eAAA,GAAkB,WAAW,QAAQ,SAAA,CAAU,SAAA;;;;;;;;;;;;;;;;;;;;;;;;iBC9FrC,aAAA,CACpB,QAAA,UACA,OAAA;EAAW,SAAA;AAAA,IACV,OAAO;AAAA,UAkBO,iBAAA;;EAEf,QAAA;;;;;EAKA,SAAA;;;;;;;EAOA,SAAA;;;;;;;EAOA,OAAA,GAAU,aAAa;;;;;;EAMvB,gBAAA;;;;;;;;;;;AD8CF;;;;EC/BE,KAAA;AAAA;AAAA,UAGe,aAAA;ED4BmD;EC1BlE,SAAA;;;;AApEF;;;;;;;;;AAGU;AAkBV;EA8DE,GAAA,EAAK,MAAM;AAAA;;;;;;;;;iBAWU,UAAA,CAAW,OAAA,EAAS,iBAAA,GAAoB,aAAA,CAAc,aAAA;;;;ADzHpB;AAEzD;;;;;;cEZa,sBAAA,EAAsB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;KASvB,gBAAA,GAAmB,CAAA,CAAE,KAAK,QAAQ,sBAAA;;;;;;;;;;;;;;;iBAgB9B,uBAAA,CACd,MAAA,YACC,UAAU,QAAQ,sBAAA,CAAuB,SAAA;AAAA,UAY3B,eAAA;;EAEf,QAAA;;EAEA,MAAA;;EAEA,SAAA;;EAEA,SAAA;;EAEA,cAAA;;EAEA,KAAA;;EAEA,OAAA,GAAU,aAAa;;EAEvB,gBAAA;;EAEA,KAAA;AAAA;;iBAiBqB,QAAA,CAAS,OAAA,EAAS,eAAA,GAAkB,aAAA,CAAc,aAAA;;;;;;;;;;;AFvEzE;;;;AAMU;AACV;;;;AAAyD;AAEzD;;;;;;;;;;;;;KGNY,eAAA,sCAKR,eAAA;EAAA,CACG,GAAA,WAAc,eAAe;AAAA;;UAGnB,WAAA;EACf,SAAA;EACA,OAAA,EAAS,eAAe;AAAA;;UAIT,YAAA;EACf,SAAA;EACA,KAAA;EACA,OAAA;EACA,KAAA;EACA,eAAA;EACA,cAAA;EACA,OAAA,EAAS,aAAa,CAAC,WAAA;;EAEvB,UAAA;AAAA;AAAA,UAGe,kBAAA;;EAEf,WAAA;;EAEA,oBAAoB;AAAA;;cAIT,oBAAA;AAAA,cACA,+BAAA;AAAA,cAYA,YAAA;EAAA,iBACM,QAAA;EAAA,iBACA,WAAA;EAAA,iBACA,oBAAA;EAAA,QACT,iBAAA;cAEI,MAAA,GAAQ,kBAAA;;;;;;;;;;EAcpB,QAAA,CACE,SAAA,UACA,SAAA,UACA,OAAA,UACA,KAAA,sBACA,OAAA,EAAS,eAAA;;EA+BX,kBAAA,CAAA;;EAOA,uBAAA,CAAA;;;;;;EASA,wBAAA,CAAA;;;;;AHKF;EGIE,cAAA,CAAA,GAAkB,YAAA;;;;;;EAoBlB,QAAA,CAAA,GAAY,oBAAA;AAAA;AAAA,UAUG,oBAAA;EACf,WAAA;EACA,gBAAA;EACA,iBAAA;EACA,QAAA,EAAU,YAAY;AAAA;;;;;;;AFjId;AAkBV;;;;iBE6HgB,0BAAA,CACd,CAAA,EAAG,oBAAA,EACH,CAAA,EAAG,oBAAA,GACF,oBAAA;;;;;;;;;;;AHvLH;;;;AAMU;AACV;;;;AAAyD;AAEzD;;;;;;;;;;;UIRiB,UAAA;EJQK;EINpB,KAAA;;EAEA,QAAA;;EAEA,GAAA;;EAEA,MAAA;;EAEA,GAAA;AAAA;;cAIW,iBAAA,EAAmB,UAM/B;;;;;;;;;;;;;cAcY,gBAAA;;;;;;;cAQA,mBAAA;EAAA,QACH,KAAA;EAAA,QACA,SAAA;EAAA,QACA,WAAA;EAAA,iBACS,OAAA;;;;;;;;;;;;;;;;;;EAmBjB,MAAA,CAAO,KAAA;EAeP,QAAA,CAAA,GAAY,UAAU;AAAA;;;;AJ+CxB;;;;;;;;AAAoE;;;;AC9FpE;;;cGuFa,kCAAA;AAAA,UAEI,YAAA;EHvFJ;EGyFX,KAAA;EHxFC;EG0FD,UAAA;EH1FQ;EG4FR,UAAA;AAAA;;;;;;;;iBAUc,kBAAA,CAAmB,IAAA,EAAM,UAAA,EAAY,IAAA,EAAM,UAAA,GAAa,YAAA;;;AH1CjE;AAGP;;;;;iBG+DgB,eAAA,CAAgB,KAAA,YAAiB,UAAU;;;;cCrJ9C,kBAAA;AAAA,UAEI,gBAAA;;EAEf,WAAA;;EAEA,QAAA;;EAEA,MAAA,EAAQ,MAAA;;EAER,SAAA,EAAW,YAAA,CAAa,IAAA;;EAExB,IAAA,EAAM,eAAA;;EAEN,QAAA,EAAU,YAAA;;EAEV,OAAA;;EAEA,SAAA;;;;;;;EAOA,UAAA,IAAc,QAAA,EAAU,iBAAA;;EAExB,QAAA;;EAEA,MAAA,GAAS,WAAA;;EAET,YAAA,GAAe,YAAA;;;;;;;EAOf,cAAA,GAAiB,YAAA;;;;;;;;EAQjB,UAAA,GAAa,mBAAA;;;;;;;;;;;;;;;ALoEf;EKpDE,MAAA;IACE,IAAA,GAAO,IAAA,EAAM,MAAA,mBAAyB,GAAA;IACtC,IAAA,GAAO,IAAA,EAAM,MAAA,mBAAyB,GAAA;EAAA;AAAA;;;ALkD0B;;;UKzCnD,iBAAA;EACf,QAAA;EACA,aAAA;EACA,UAAA;EACA,WAAA;EJxDA;EI0DA,UAAA;EACA,gBAAA;EJzDC;EI2DD,cAAA;EJ3DQ;EI6DR,aAAA;EJ3CgC;EI6ChC,qBAAA;EJxBuB;;;;;;EI+BvB,SAAA;AAAA;;AJVK;AAGP;;UIciB,eAAA;EJGJ;EIDX,QAAA;EJCA;EICA,aAAA;EJDW;AAAA;AAWb;;;EIJE,UAAA;EJI2E;EIF3E,WAAA;EJE0E;;;;;EII1E,UAAA;EJJwF;EIMxF,gBAAA;;EAEA,MAAA,EAAQ,UAAA,CAAW,YAAA;;EAEnB,QAAA,EAAU,UAAA,CAAW,YAAA;EHrIZ;;;;;;;;EG8IT,KAAA,WAAgB,eAAA;;;;;;;;;;;;;EAahB,cAAA;EACA,eAAA;;;;;;;EAOA,UAAA,EAAY,UAAA;AAAA;;;;;;;;iBAUQ,SAAA,MAAA,CAAgB,OAAA,EAAS,gBAAA,CAAiB,IAAA,IAAQ,OAAA,CAAQ,eAAA;;;;;;;;;;;;;UC9J/D,oBAAA;EACf,EAAA;EACA,OAAA,EAAS,oBAAA;EACT,cAAA,CAAe,IAAA,EAAM,iBAAiB;AAAA;;;;;;;cAS3B,0BAAA,EAA0B,CAAA,CAAA,SAAA;;;;;;;KAG3B,oBAAA,GAAuB,CAAA,CAAE,KAAK,QAAQ,0BAAA;;;;;;;;;;;;cAarC,aAAA,EAAe,aAAa,CAAC,oBAAA;;;;;;;KAa9B,gBAAA,SAAyB,YAAA,GAAe,OAAA,CAAQ,YAAA;;;;;;;;;;;;;;;KAgBhD,mBAAA,SAA4B,OAAO;;;;;;;ANmD/C;;;;;;;;AAAoE;;;;AC9FpE;;;;;;;KKsEY,gBAAA,IACV,UAAA,aACG,OAAO;EAAG,SAAA;EAAmB,OAAA,SAAgB,OAAA;AAAA;;;;;;UAOjC,eAAA;EACf,WAAA;ELtCU;EKwCV,MAAA,EAAQ,MAAA;ELnBR;;AAAK;AAGP;;;EKuBE,MAAA,GAAS,MAAM;AAAA;;;;ALNJ;AAWb;;;;;;;;;;;;;KKeY,QAAA,IAAY,KAAA,WAAgB,eAAA,IAAmB,GAAA,EAAK,eAAA,KAAoB,OAAA;AAAA,UAEnE,sBAAA;;EAEf,UAAA,EAAY,eAAA;EJtJD;EIwJX,UAAA,EAAY,iBAAA;;EAEZ,QAAA,EAAU,gBAAA;;EAEV,OAAA;;;;;;EAMA,WAAA,GAAc,mBAAA;;;;;;EAMd,eAAA,GAAkB,gBAAA;EJxKe;;;;;;EI+KjC,QAAA,GAAW,QAAA;;EAEX,MAAA,GAAS,MAAA;AAAA;;;;;;;;;;;iBAaK,sBAAA,CACd,MAAA,EAAQ,sBAAA,IACN,GAAA,EAAK,oBAAA,KAAyB,OAAA;AAAA,UAqZjB,4BAAA;EACf,UAAA,EAAY,eAAA;;EAEZ,MAAA,GAAS,MAAM;AAAA;;;;;;;;;;;;;;;AJ/kBjB;;;;;;;;AAAoE;AAgBpE;;;;;;;;;AAEqD;AAYrD;;;;iBIylBgB,4BAAA,CACd,MAAA,EAAQ,4BAAA,GACP,qBAAqB"}
|
package/dist/worker.d.mts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { a as ImportsRunJobContext, c as PairSinkContext, d as createRunImportHandler, f as importsRunJob, i as ImportsRunDeadListenerConfig, l as RunImportHandlerConfig, n as EsClientResolver, o as ImportsRunJobPayload, p as importsRunJobPayloadSchema, r as FilePathResolver, s as PairSink, t as EnsureIndexResolver, u as createImportsRunDeadListener } from "./worker-
|
|
1
|
+
import { a as ImportsRunJobContext, c as PairSinkContext, d as createRunImportHandler, f as importsRunJob, i as ImportsRunDeadListenerConfig, l as RunImportHandlerConfig, n as EsClientResolver, o as ImportsRunJobPayload, p as importsRunJobPayloadSchema, r as FilePathResolver, s as PairSink, t as EnsureIndexResolver, u as createImportsRunDeadListener } from "./worker-DH3SN-91.mjs";
|
|
2
2
|
export { EnsureIndexResolver, EsClientResolver, FilePathResolver, ImportsRunDeadListenerConfig, ImportsRunJobContext, ImportsRunJobPayload, PairSink, PairSinkContext, RunImportHandlerConfig, createImportsRunDeadListener, createRunImportHandler, importsRunJob, importsRunJobPayloadSchema };
|
package/dist/worker.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import{_ as e,c as t,d as n,f as r,h as i,n as a,s as o}from"./runner-wPnCZlB4.mjs";import{a as s,i as c,r as l}from"./publish-DCXIPwTE.mjs";import{runAsCli as u}from"@murumets-ee/core";import{z as d}from"zod";import{and as f,desc as p,eq as m}from"drizzle-orm";import{defineJob as h}from"@murumets-ee/queue/client";const g=d.object({importRunId:d.string().uuid()}),_=h({name:`imports:run`,description:`Stream a feed file, transform rows, bulk-write to Elasticsearch.`,schema:g,defaultRetries:0});function v(l){let{importRuns:d,transforms:h,esClient:g,esIndex:_,resolveFilePath:v,ensureIndex:b,pairSink:x,logger:S}=l;return async l=>u(async()=>{let{importRunId:u}=l.payload,C=S?.child({jobId:l.id,importRunId:u,type:`imports:run`}),w=await d.findById(u);if(!w){C?.warn(`import_run row not found — skipping`);return}w.status===`running`&&C?.warn({previousQueueJobId:w.queueJobId,previousStartedAt:w.startedAt},`imports: re-claiming a row left in running by a prior worker — likely a crash recovered by queue lock-recovery`);let T=h.get(w.transformName);if(!T)throw await y(d,u,`Unknown transform "${w.transformName}"`),Error(`No transform registered for name "${w.transformName}"`);let E=w.params&&typeof w.params==`object`&&!Array.isArray(w.params)?w.params:{},D=i(E);if(!D.success){let e=D.error.issues[0]?.message??`invalid feed reader config`;throw await y(d,u,`Invalid params.feed: ${e}`),Error(`Invalid params.feed for import_run ${u}: ${e}`)}let O=D.data;await d.update(u,{status:`running`,queueJobId:l.id,startedAt:new Date}),c(u);let k;try{k=await g()}catch(e){throw await y(d,u,`Could not resolve ES client: ${e instanceof Error?e.message:String(e)}`),e}if(b)try{await b()}catch(e){throw await y(d,u,`Could not ensure ES index/alias: ${e instanceof Error?e.message:String(e)}`),e}let A,j;if(v)try{let e=await v(w.filePath);A=e.localPath,j=e.cleanup}catch(e){throw await y(d,u,`Could not resolve filePath: ${e instanceof Error?e.message:String(e)}`),e}else A=w.filePath;let M;if(O.format!==`xlsx`)try{M=await e(A,O.hasHeader===void 0?{}:{hasHeader:O.hasHeader}),C?.info({importRunId:u,totalRows:M},`imports: pre-pass row count`)}catch(e){C?.warn({err:e},`imports: pre-pass row count failed — continuing without total`)}let N;try{let e={filePath:A,...O.format!==void 0&&{format:O.format},...O.delimiter!==void 0&&{delimiter:O.delimiter},...O.hasHeader!==void 0&&{hasHeader:O.hasHeader},...O.headerRowIndex!==void 0&&{headerRowIndex:O.headerRowIndex},...O.sheet!==void 0&&{sheet:O.sheet}};N=await a({importRunId:u,runLabel:w.label,params:E,transform:T,feed:e,esClient:k,esIndex:_,onProgress:e=>l.updateProgress(M===void 0?e:{...e,totalRows:M}),...C!==void 0&&{logger:C}})}catch(e){throw await y(d,u,e instanceof Error?e.message:String(e)),j&&await j().catch(e=>{C?.warn({err:e},`filePath cleanup after failed run threw — ignoring`)}),e}j&&await j().catch(e=>{C?.warn({err:e},`filePath cleanup after successful run threw — ignoring`)});let P=N.warnings,F=typeof E.brandId==`string`?E.brandId:void 0,I=typeof E.supplierId==`string`?E.supplierId:void 0;if(F&&I&&N.priceStats.median!==null)try{let e=d.getTable(),i=await d.findMany({where:f(m(e.transformName,w.transformName),m(e.status,`succeeded`)),orderBy:p(e.createdAt),limit:50}),a=i.find(e=>{if(e.id===u)return!1;let t=e.params;if(!t||typeof t!=`object`||Array.isArray(t))return!1;let n=t;return n.brandId===F&&n.supplierId===I});!a&&i.length>=50&&C?.warn({brandId:F,supplierId:I,transformName:w.transformName},`imports: price-drift check found no prior run for this (brand, supplier) within the 50 most recent succeeded runs for this transform — a real prior run may exist beyond the bound, not necessarily absent`);let s=a?t(a.priceStats):null,c=s?o(s,N.priceStats):null;if(c){let e=c.ratio>=1,t=new n;t.addError(-1,`price_anomaly`,`Median purchase price is ${(e?c.ratio:1/c.ratio).toFixed(1)}x ${e?`higher`:`lower`} than the previous successful run for this supplier/brand (previous run ${a?.id??`unknown`}: median ${c.prevMedian}; this run: median ${c.currMedian}, minor units). Check for a decimal or thousands-separator format change in the source feed.`,`priceStats.median`,{previousImportRunId:a?.id??null,previousMedian:c.prevMedian,currentMedian:c.currMedian,ratio:c.ratio}),P=r(N.warnings,t.snapshot()),C?.warn({previousImportRunId:a?.id,ratio:c.ratio},`imports: price_anomaly — median purchase price shifted by a magnitude vs the prior run`)}}catch(e){C?.warn({err:e},`imports: price-drift comparison failed — continuing without it`)}if(await d.update(u,{status:`succeeded`,finishedAt:new Date,totals:{rowsRead:N.rowsRead,rowsSucceeded:N.rowsSucceeded,rowsFailed:N.rowsFailed,rowsSkipped:N.rowsSkipped,rowsWarned:N.rowsWarned,batchesCompleted:N.batchesCompleted,patchesApplied:N.patchesApplied,patchesOrphaned:N.patchesOrphaned},errorSummary:JSON.parse(JSON.stringify(N.errors)),warningSummary:JSON.parse(JSON.stringify(P)),priceStats:JSON.parse(JSON.stringify(N.priceStats))}),s(u,{rowsRead:N.rowsRead,rowsSucceeded:N.rowsSucceeded,rowsFailed:N.rowsFailed}),x&&N.pairs.length>0)try{await x(N.pairs,{importRunId:u,params:E,...C!==void 0&&{logger:C}}),C?.info({importRunId:u,pairs:N.pairs.length},`imports: pairSink invoked`)}catch(e){C?.warn({importRunId:u,pairs:N.pairs.length,err:e},`imports: pairSink threw — replacement ingestion incomplete, catalogue import unaffected`)}C?.info({rowsRead:N.rowsRead,rowsSucceeded:N.rowsSucceeded,rowsFailed:N.rowsFailed,rowsSkipped:N.rowsSkipped,batches:N.batchesCompleted},`import_run completed`)})}async function y(e,t,n){try{await e.update(t,{status:`failed`,finishedAt:new Date,errorSummary:{fatal:n}}),l(t,n)}catch{}}function b(e){let{importRuns:t,logger:n}=e;return{onJobDead:async e=>{e.jobType===`imports:run`&&await u(async()=>{let r=n?.child({event:`imports.run.dead`,jobId:e.jobId,type:`imports:run`}),i=t.getTable(),a=(await t.findMany({where:m(i.queueJobId,e.jobId),limit:1}))[0];if(!a){r?.warn(`queue.job.dead with no matching import_run — skipping`);return}if(a.status!==`running`&&a.status!==`pending`){r?.info({importRunId:a.id,status:a.status},`queue.job.dead arrived for a row already in a terminal state — leaving as-is`);return}a.status===`pending`&&r?.warn({importRunId:a.id,attempts:e.attempts},`queue.job.dead arrived for an import_run still in pending — worker crashed BEFORE claim/lookup`),await y(t,a.id,e.error),r?.info({importRunId:a.id,attempts:e.attempts,priorStatus:a.status},`queue.job.dead → import_run marked failed`)})}}}export{b as createImportsRunDeadListener,v as createRunImportHandler,_ as importsRunJob,g as importsRunJobPayloadSchema};
|
|
2
2
|
//# sourceMappingURL=worker.mjs.map
|