navu-catalog 4.14.2

package/src/site-catalogs/atlascopco/atlascopco-harvester.ts

@@ -0,0 +1,368 @@
+import type { CatalogHarvestProduct } from "@hivepoint/navu-apis";
+import type { NavuLogger } from "@navu-co/navu-core";
+import { CatalogProgressHeartbeat, CatalogProgressLogger } from "../../catalog-manager/catalog-progress-logger.js";
+import { HarvestStagingStore } from "../../reusable/harvest/harvest-staging-store.js";
+import type { CatalogHarvestStagingFamilyUpsert } from "../../db/types/catalog-harvest-staging-family.js";
+import { SiteCatalogLogging, type SiteCatalogLogContext } from "../site-catalog-logging.js";
+import { AtlascopcoCatalogProfile } from "./atlascopco-catalog-profile.js";
+import { AtlascopcoHarvestOptionsReader } from "./atlascopco-harvest-options.js";
+import { AtlascopcoHarvestValidator } from "./atlascopco-harvest-validator.js";
+import { AtlascopcoListingClassificationHelper } from "./atlascopco-listing-classification-helper.js";
+import type { AtlascopcoCatalogData } from "./atlascopco-catalog-data.js";
+import { AtlascopcoProductBuilder } from "./atlascopco-product-builder.js";
+import { AtlascopcoProductParser } from "./atlascopco-product-parser.js";
+import { AtlascopcoRawPayloadSupport } from "./atlascopco-raw-payload-support.js";
+import { AtlascopcoStagingReparser } from "./atlascopco-staging-reparser.js";
+import { AtlascopcoAlgoliaClient, AtlascopcoStorefrontClient } from "./upstream/atlascopco/atlascopco-storefront-client.js";
+import { AtlascopcoCatalogConstants } from "./upstream/atlascopco/atlascopco-catalog-constants.js";
+import { PdpNotFoundError } from "../../reusable/storefront/pdp-not-found-error.js";
+import type { AtlascopcoAlgoliaHit } from "./upstream/atlascopco/atlascopco-types.js";
+
+export interface AtlascopcoHarvestCollectOptions {
+  logger?: NavuLogger;
+  siteId?: string;
+  catalogName?: string;
+  harvestUrl?: string;
+  harvestConfig?: object;
+  priorHarvestProducts?: readonly CatalogHarvestProduct[];
+  harvestStaging?: HarvestStagingStore;
+}
+
+interface AtlascopcoPdpCollectResult {
+  items: AtlascopcoCatalogData[];
+  algoliaSupplementCount: number;
+  pdpNotFoundProducts: number;
+  pdpNotFoundSpareParts: number;
+  pdpNotFoundMaterialCodes: Set<string>;
+  fetchFailures: number;
+}
+
+export class AtlascopcoCatalogHarvester {
+  static async collect(options?: AtlascopcoHarvestCollectOptions): Promise<CatalogHarvestProduct[]> {
+    const catalogName = options?.catalogName ?? AtlascopcoCatalogProfile.CATALOG_NAME;
+    const harvestUrl = options?.harvestUrl ?? AtlascopcoCatalogProfile.HARVESTER_URL;
+    const clientOptions = AtlascopcoHarvestOptionsReader.read(options?.harvestConfig, catalogName);
+    const logContext: SiteCatalogLogContext = { siteId: options?.siteId, catalogName, harvestUrl };
+    if (clientOptions.reprocessStaging) {
+      if (options?.siteId === undefined || options?.harvestStaging === undefined) {
+        throw new Error("Atlas Copco staging reprocess requires siteId and harvestStaging");
+      }
+      options?.logger?.info("OK ✓ Atlas Copco harvest reprocessing durable staging payloads", SiteCatalogLogging.catalog(logContext));
+      const parsedItems = await AtlascopcoCatalogHarvester.collectFromStaging(options.harvestStaging, logContext, options?.logger);
+      const validation = AtlascopcoHarvestValidator.validate(parsedItems, { validateCounts: false });
+      if (!validation.passed) {
+        throw new Error(`Atlas Copco harvest validation failed: ${validation.issues.join("; ")}`);
+      }
+      const products = AtlascopcoProductBuilder.buildProducts(parsedItems);
+      options?.logger?.info("OK ✓ Atlas Copco harvester built catalog products from staging", SiteCatalogLogging.catalog(logContext, {
+        parsedItemCount: parsedItems.length,
+        productCount: products.length,
+        reprocessStaging: true,
+        productTypeCounts: validation.productTypeCounts,
+      }));
+      return products;
+    }
+    options?.logger?.info("OK ✓ Atlas Copco harvester bootstrapping Algolia credentials", SiteCatalogLogging.catalog(logContext, {
+      pdpEnrichment: clientOptions.pdpEnrichment,
+    }));
+    const storefront = new AtlascopcoStorefrontClient({ locale: clientOptions.locale, requestDelayMs: clientOptions.pdpRequestDelayMs ?? 500 });
+    const credentials = await storefront.bootstrapAlgoliaCredentials();
+    const algolia = new AtlascopcoAlgoliaClient(credentials, { requestDelayMs: clientOptions.requestDelayMs ?? 300 });
+    const listingHits: AtlascopcoAlgoliaHit[] = [];
+    let expectedProductsCount: number | undefined;
+    let expectedSparePartsCount: number | undefined;
+    if (clientOptions.includeProducts) {
+      const productsFilter = `${AtlascopcoCatalogConstants.ALGOLIA_BASE_FILTER} AND ${AtlascopcoCatalogConstants.CATEGORY_FACET_LEVEL0}:"${AtlascopcoCatalogConstants.PRODUCTS_LEVEL0}"`;
+      expectedProductsCount = await algolia.countHits(productsFilter);
+      const hits = await AtlascopcoCatalogHarvester.fetchScopedHits(algolia, productsFilter, clientOptions.maxProducts);
+      listingHits.push(...hits);
+      options?.logger?.info("OK ✓ Atlas Copco enumerated Products tree", SiteCatalogLogging.catalog(logContext, { hitCount: hits.length, expectedCount: expectedProductsCount }));
+    }
+    if (clientOptions.includeSpareParts) {
+      const sparePartsFilter = `${AtlascopcoCatalogConstants.ALGOLIA_BASE_FILTER} AND ${AtlascopcoCatalogConstants.CATEGORY_FACET_LEVEL0}:"${AtlascopcoCatalogConstants.SPARE_PARTS_LEVEL0}"`;
+      const sparePartsResult = clientOptions.maxSpareParts !== undefined
+        ? { hits: await algolia.fetchHitsUpTo(sparePartsFilter, clientOptions.maxSpareParts), expectedCount: await algolia.countHits(sparePartsFilter) }
+        : await AtlascopcoCatalogHarvester.fetchSparePartsHits(algolia, sparePartsFilter, logContext, options?.logger);
+      expectedSparePartsCount = sparePartsResult.expectedCount;
+      listingHits.push(...sparePartsResult.hits);
+      options?.logger?.info("OK ✓ Atlas Copco enumerated Spare Parts tree", SiteCatalogLogging.catalog(logContext, { hitCount: sparePartsResult.hits.length, expectedCount: expectedSparePartsCount }));
+    }
+    let parsedItems: AtlascopcoCatalogData[];
+    let pdpNotFoundProducts = 0;
+    let pdpNotFoundSpareParts = 0;
+    let pdpNotFoundMaterialCodes = new Set<string>();
+    let fetchFailures = 0;
+    if (clientOptions.pdpEnrichment) {
+      if (options?.siteId === undefined || options?.harvestStaging === undefined) {
+        throw new Error("Atlas Copco PDP enrichment requires siteId and harvestStaging");
+      }
+      const pdpResult = await AtlascopcoCatalogHarvester.collectFromPdp(storefront, options.harvestStaging, listingHits, logContext, options?.logger);
+      parsedItems = pdpResult.items;
+      pdpNotFoundProducts = pdpResult.pdpNotFoundProducts;
+      pdpNotFoundSpareParts = pdpResult.pdpNotFoundSpareParts;
+      pdpNotFoundMaterialCodes = pdpResult.pdpNotFoundMaterialCodes;
+      fetchFailures = pdpResult.fetchFailures;
+      if (expectedProductsCount !== undefined) {
+        expectedProductsCount -= pdpNotFoundProducts;
+      }
+      if (expectedSparePartsCount !== undefined) {
+        expectedSparePartsCount -= pdpNotFoundSpareParts;
+      }
+      options?.logger?.info("OK ✓ Atlas Copco PDP fetch complete", SiteCatalogLogging.catalog(logContext, {
+        parsedItemCount: parsedItems.length,
+        algoliaSupplementCount: pdpResult.algoliaSupplementCount,
+        pdpNotFoundProducts,
+        pdpNotFoundSpareParts,
+        fetchFailures,
+      }));
+    } else {
+      parsedItems = AtlascopcoCatalogHarvester.collectFromAlgolia(listingHits);
+      options?.logger?.info("OK ✓ Atlas Copco Algolia-only harvest parsed listing hits", SiteCatalogLogging.catalog(logContext, {
+        parsedItemCount: parsedItems.length,
+      }));
+    }
+    const listingClassification = AtlascopcoListingClassificationHelper.build(listingHits);
+    if (clientOptions.pdpEnrichment) {
+      parsedItems = AtlascopcoListingClassificationHelper.reconcileListingCategories(parsedItems, listingClassification);
+    }
+    const validation = AtlascopcoHarvestValidator.validate(parsedItems, {
+      validateCounts: clientOptions.validateCounts,
+      expectedProductsCount,
+      expectedSparePartsCount,
+      listingProductsMaterialCodes: clientOptions.pdpEnrichment ? listingClassification.productsMaterialCodes : undefined,
+      listingSparePartsMaterialCodes: clientOptions.pdpEnrichment ? listingClassification.sparePartsMaterialCodes : undefined,
+      excludedListingMaterialCodes: clientOptions.pdpEnrichment ? pdpNotFoundMaterialCodes : undefined,
+    });
+    if (!validation.passed) {
+      for (const issue of validation.issues) {
+        options?.logger?.error(new Error(issue), "ERROR ✕ Atlas Copco harvest validation failed", SiteCatalogLogging.catalog(logContext, { issue }));
+      }
+      throw new Error(`Atlas Copco harvest validation failed: ${validation.issues.join("; ")}`);
+    }
+    const products = AtlascopcoProductBuilder.buildProducts(parsedItems);
+    options?.logger?.info("OK ✓ Atlas Copco harvester built catalog products", SiteCatalogLogging.catalog(logContext, {
+      listingHitCount: listingHits.length,
+      parsedItemCount: parsedItems.length,
+      productCount: products.length,
+      pdpEnrichment: clientOptions.pdpEnrichment,
+      pdpNotFoundProducts: clientOptions.pdpEnrichment ? pdpNotFoundProducts : undefined,
+      pdpNotFoundSpareParts: clientOptions.pdpEnrichment ? pdpNotFoundSpareParts : undefined,
+      fetchFailures: clientOptions.pdpEnrichment ? fetchFailures : undefined,
+      productTypeCounts: validation.productTypeCounts,
+    }));
+    return products;
+  }
+
+  private static collectFromAlgolia(listingHits: AtlascopcoAlgoliaHit[]): AtlascopcoCatalogData[] {
+    const fetchedAt = new Date().toISOString();
+    const byMaterialCode = new Map<string, AtlascopcoCatalogData>();
+    for (const hit of listingHits) {
+      const item = AtlascopcoProductParser.parseAlgoliaHit(hit, fetchedAt);
+      if (item !== undefined && !byMaterialCode.has(item.materialCode)) {
+        byMaterialCode.set(item.materialCode, item);
+      }
+    }
+    return [...byMaterialCode.values()].sort((left, right) => left.materialCode.localeCompare(right.materialCode));
+  }
+
+  private static async collectFromStaging(staging: HarvestStagingStore, logContext: SiteCatalogLogContext, logger?: NavuLogger): Promise<AtlascopcoCatalogData[]> {
+    const families = await staging.listFamilies();
+    const familiesWithRaw = families.filter((family) => AtlascopcoRawPayloadSupport.hasRawPayload(family.rawPayload));
+    if (familiesWithRaw.length === 0) {
+      throw new Error("Atlas Copco staging reprocess found no durable raw payloads; run a PDP harvest first");
+    }
+    if (familiesWithRaw.length < families.length) {
+      logger?.warn("WARN ! Atlas Copco staging reprocess skipped families without raw payloads", SiteCatalogLogging.catalog(logContext, {
+        stagedFamilyCount: families.length,
+        rawPayloadFamilyCount: familiesWithRaw.length,
+      }));
+    }
+    return AtlascopcoStagingReparser.reparseAll(familiesWithRaw);
+  }
+
+  private static async collectFromPdp(storefront: AtlascopcoStorefrontClient, staging: HarvestStagingStore, listingHits: AtlascopcoAlgoliaHit[], logContext: SiteCatalogLogContext, logger?: NavuLogger): Promise<AtlascopcoPdpCollectResult> {
+    const materialCodes = AtlascopcoCatalogHarvester.collectMaterialCodes(listingHits);
+    const listingClassification = AtlascopcoListingClassificationHelper.build(listingHits);
+    const listingHitByMaterialCode = AtlascopcoCatalogHarvester.buildListingHitByMaterialCode(listingHits);
+    const stagedFamilies = await staging.listFamilies();
+    const reparsableMaterialCodes = AtlascopcoStagingReparser.buildReparsableMaterialCodeSet(stagedFamilies);
+    const stagedFamilyKeys = new Set(stagedFamilies.map((family) => family.familyKey));
+    const stagedFamilyCount = stagedFamilies.length;
+    if (reparsableMaterialCodes.size > 0) {
+      logger?.info("OK ✓ Atlas Copco PDP harvest resuming from Mongo staging", SiteCatalogLogging.catalog(logContext, {
+        reparsableMaterialCodeCount: reparsableMaterialCodes.size,
+        stagedFamilyCount,
+      }));
+    }
+    let fetchFailures = 0;
+    let pdpNotFoundProducts = 0;
+    let pdpNotFoundSpareParts = 0;
+    const pdpNotFoundMaterialCodes = new Set<string>();
+    let fetchedFamilyCount = 0;
+    const heartbeat = new CatalogProgressHeartbeat();
+    for (let index = 0; index < materialCodes.length; index += 1) {
+      const scannedCount = index + 1;
+      const entry = materialCodes[index]!;
+      const isSparePartListing = listingClassification.sparePartsMaterialCodes.has(entry.materialCode);
+      if (reparsableMaterialCodes.has(entry.materialCode) && (!isSparePartListing || stagedFamilyKeys.has(`sku:${entry.materialCode}`))) {
+        AtlascopcoCatalogHarvester.logPdpCollectProgress(logger, logContext, heartbeat, scannedCount, materialCodes.length, reparsableMaterialCodes.size, stagedFamilyCount + fetchedFamilyCount, pdpNotFoundProducts, pdpNotFoundSpareParts, fetchFailures);
+        continue;
+      }
+      try {
+        const stagedFamily = await AtlascopcoCatalogHarvester.fetchAndStageFamily(storefront, staging, entry.materialCode, entry.slug, isSparePartListing);
+        fetchedFamilyCount += 1;
+        stagedFamilyKeys.add(stagedFamily.familyKey);
+        for (const materialCode of stagedFamily.materialCodes) {
+          reparsableMaterialCodes.add(materialCode);
+        }
+      } catch (err) {
+        if (err instanceof PdpNotFoundError) {
+          const listingHit = listingHitByMaterialCode.get(entry.materialCode);
+          const categoryPath = listingHit !== undefined ? AtlascopcoProductParser.resolveCategoryPathFromAlgolia(listingHit) : [];
+          pdpNotFoundMaterialCodes.add(entry.materialCode);
+          if (categoryPath[0] === AtlascopcoCatalogConstants.SPARE_PARTS_LEVEL0) {
+            pdpNotFoundSpareParts += 1;
+          } else if (categoryPath[0] === AtlascopcoCatalogConstants.PRODUCTS_LEVEL0) {
+            pdpNotFoundProducts += 1;
+          }
+          logger?.info("OK ○ Atlas Copco PDP not found; excluding from catalog", SiteCatalogLogging.catalog(logContext, {
+            materialCode: entry.materialCode,
+            url: err.url,
+          }));
+        } else {
+          fetchFailures += 1;
+          logger?.warn("WARN ! Atlas Copco skipped material code", SiteCatalogLogging.catalog(logContext, {
+            materialCode: entry.materialCode,
+            error: err instanceof Error ? err.message : String(err),
+          }));
+        }
+      }
+      AtlascopcoCatalogHarvester.logPdpCollectProgress(logger, logContext, heartbeat, scannedCount, materialCodes.length, reparsableMaterialCodes.size, stagedFamilyCount + fetchedFamilyCount, pdpNotFoundProducts, pdpNotFoundSpareParts, fetchFailures);
+    }
+    const families = await staging.listFamilies();
+    const reparsedItems = AtlascopcoStagingReparser.reparseAll(families);
+    const fetchedAt = new Date().toISOString();
+    const items = AtlascopcoListingClassificationHelper.supplementMissingListingItems(reparsedItems, listingHits, listingClassification, pdpNotFoundMaterialCodes, fetchedAt);
+    const algoliaSupplementCount = items.length - reparsedItems.length;
+    if (algoliaSupplementCount > 0) {
+      logger?.info("OK ✓ Atlas Copco supplemented listing-only material codes from Algolia", SiteCatalogLogging.catalog(logContext, { algoliaSupplementCount, reparsedItemCount: reparsedItems.length }));
+    }
+    const result: AtlascopcoPdpCollectResult = { items, algoliaSupplementCount, pdpNotFoundProducts, pdpNotFoundSpareParts, pdpNotFoundMaterialCodes, fetchFailures };
+    return result;
+  }
+
+  private static logPdpCollectProgress(logger: NavuLogger | undefined, logContext: SiteCatalogLogContext, heartbeat: CatalogProgressHeartbeat, scannedCount: number, totalCount: number, reparsableMaterialCodeCount: number, stagedFamilyCount: number, pdpNotFoundProducts: number, pdpNotFoundSpareParts: number, fetchFailures: number): void {
+    const countIntervalLog = CatalogProgressLogger.shouldLogProgress(scannedCount, totalCount, CatalogProgressLogger.BULK_HARVEST_INTERVAL);
+    const signature = CatalogProgressHeartbeat.buildSignature({ scannedCount, reparsableMaterialCodeCount, stagedFamilyCount, pdpNotFoundProducts, pdpNotFoundSpareParts, fetchFailures });
+    if (!countIntervalLog && !heartbeat.shouldLog(signature)) {
+      return;
+    }
+    logger?.info("OK ✓ Atlas Copco PDP fetch progress", SiteCatalogLogging.catalog(logContext, {
+      reparsableMaterialCodeCount,
+      stagedFamilyCount,
+      pdpNotFoundProducts,
+      pdpNotFoundSpareParts,
+      fetchFailures,
+      ...CatalogProgressLogger.progressFields(scannedCount, totalCount),
+    }));
+  }
+
+  private static async fetchSparePartsHits(algolia: AtlascopcoAlgoliaClient, sparePartsFilter: string, logContext: SiteCatalogLogContext, logger?: NavuLogger): Promise<{ hits: AtlascopcoAlgoliaHit[]; expectedCount: number }> {
+    const expectedCount = await algolia.countHits(sparePartsFilter);
+    logger?.info("OK ✓ Atlas Copco Spare Parts enumeration starting", SiteCatalogLogging.catalog(logContext, { expectedCount }));
+    const level1Facets = await algolia.queryFacets(sparePartsFilter, AtlascopcoCatalogConstants.CATEGORY_FACET_LEVEL1);
+    const buckets = Object.entries(level1Facets).filter(([, count]) => count > 0).sort((left, right) => right[1] - left[1]);
+    const mergedHits: AtlascopcoAlgoliaHit[] = [];
+    for (let index = 0; index < buckets.length; index += 1) {
+      const [bucket, bucketExpectedCount] = buckets[index]!;
+      const bucketIndex = index + 1;
+      logger?.info("OK ✓ Atlas Copco Spare Parts bucket starting", SiteCatalogLogging.catalog(logContext, {
+        bucket,
+        bucketExpectedCount,
+        bucketIndex,
+        bucketTotal: buckets.length,
+      }));
+      const subFilter = AtlascopcoAlgoliaClient.buildFacetFilter(sparePartsFilter, AtlascopcoCatalogConstants.CATEGORY_FACET_LEVEL1, bucket);
+      const bucketHits = await algolia.fetchAllHitsWithAdaptiveSplit(subFilter);
+      mergedHits.push(...bucketHits);
+      const cumulativeHitCount = AtlascopcoAlgoliaClient.deduplicateHits(mergedHits).length;
+      logger?.info("OK ✓ Atlas Copco Spare Parts bucket complete", SiteCatalogLogging.catalog(logContext, {
+        bucket,
+        bucketHitCount: bucketHits.length,
+        bucketExpectedCount,
+        cumulativeHitCount,
+        ...CatalogProgressLogger.progressFields(bucketIndex, buckets.length),
+      }));
+    }
+    let hits = AtlascopcoAlgoliaClient.deduplicateHits(mergedHits);
+    if (hits.length < expectedCount && buckets.length > 0) {
+      const orphanFilter = AtlascopcoAlgoliaClient.buildOrphanFacetFilter(sparePartsFilter, AtlascopcoCatalogConstants.CATEGORY_FACET_LEVEL1, buckets.map(([bucket]) => bucket));
+      const orphanExpectedCount = await algolia.countHits(orphanFilter);
+      logger?.info("OK ✓ Atlas Copco Spare Parts orphan pass starting", SiteCatalogLogging.catalog(logContext, {
+        orphanExpectedCount,
+        cumulativeHitCount: hits.length,
+        expectedCount,
+      }));
+      const orphanHits = await algolia.fetchAllHits(orphanFilter);
+      hits = AtlascopcoAlgoliaClient.deduplicateHits([...hits, ...orphanHits]);
+      logger?.info("OK ✓ Atlas Copco Spare Parts orphan pass complete", SiteCatalogLogging.catalog(logContext, {
+        orphanHitCount: orphanHits.length,
+        orphanExpectedCount,
+        cumulativeHitCount: hits.length,
+        expectedCount,
+      }));
+    }
+    const result: { hits: AtlascopcoAlgoliaHit[]; expectedCount: number } = { hits, expectedCount };
+    return result;
+  }
+
+  private static async fetchScopedHits(algolia: AtlascopcoAlgoliaClient, filters: string, maxHits?: number): Promise<AtlascopcoAlgoliaHit[]> {
+    if (maxHits !== undefined) {
+      return algolia.fetchHitsUpTo(filters, maxHits);
+    }
+    return algolia.fetchAllHits(filters);
+  }
+
+  private static collectMaterialCodes(hits: AtlascopcoAlgoliaHit[]): { materialCode: string; slug?: string }[] {
+    const byCode = new Map<string, { materialCode: string; slug?: string }>();
+    for (const hit of hits) {
+      const mapped = AtlascopcoProductParser.mapAlgoliaHit(hit);
+      if (mapped.materialCode.length === 0 || byCode.has(mapped.materialCode)) {
+        continue;
+      }
+      byCode.set(mapped.materialCode, mapped);
+    }
+    return [...byCode.values()].sort((left, right) => left.materialCode.localeCompare(right.materialCode));
+  }
+
+  private static buildListingHitByMaterialCode(hits: AtlascopcoAlgoliaHit[]): Map<string, AtlascopcoAlgoliaHit> {
+    const byCode = new Map<string, AtlascopcoAlgoliaHit>();
+    for (const hit of hits) {
+      const materialCode = AtlascopcoProductParser.mapAlgoliaHit(hit).materialCode;
+      if (materialCode.length === 0 || byCode.has(materialCode)) {
+        continue;
+      }
+      byCode.set(materialCode, hit);
+    }
+    return byCode;
+  }
+
+  private static async fetchAndStageFamily(storefront: AtlascopcoStorefrontClient, staging: HarvestStagingStore, materialCode: string, slug?: string, forceSkuFamilyKey = false): Promise<{ familyKey: string; materialCodes: string[] }> {
+    const nextDataJson = await storefront.fetchProductNextDataJson(materialCode, slug);
+    const rawPayload = AtlascopcoRawPayloadSupport.extractPagePropsSlice(nextDataJson);
+    const contentHash = HarvestStagingStore.computeContentHash(nextDataJson);
+    const fetchedAt = new Date().toISOString();
+    const pageProps = AtlascopcoStorefrontClient.parsePageProps(nextDataJson);
+    const familyItems = AtlascopcoProductParser.parsePageProps(pageProps, { nextDataJson, materialCode, slug, fetchedAt });
+    const parsedFamilyKey = familyItems[0]?.familyKey;
+    if (parsedFamilyKey === undefined) {
+      throw new Error(`Atlas Copco PDP parse returned no family for material ${materialCode}`);
+    }
+    const familyKey = forceSkuFamilyKey ? `sku:${materialCode}` : parsedFamilyKey;
+    const materialCodes = familyItems.map((item) => item.materialCode);
+    const upsert: CatalogHarvestStagingFamilyUpsert = { familyKey, listingMaterialCode: materialCode, slug, materialCodes, rawPayload, contentHash, fetchedAt };
+    await staging.upsertFamily(upsert);
+    const result: { familyKey: string; materialCodes: string[] } = { familyKey, materialCodes };
+    return result;
+  }
+}

package/src/site-catalogs/atlascopco/atlascopco-listing-classification-helper.ts

@@ -0,0 +1,92 @@
+import { AtlascopcoCatalogConstants } from "./upstream/atlascopco/atlascopco-catalog-constants.js";
+import type { AtlascopcoCatalogData } from "./atlascopco-catalog-data.js";
+import { AtlascopcoProductParser } from "./atlascopco-product-parser.js";
+import type { AtlascopcoAlgoliaHit } from "./upstream/atlascopco/atlascopco-types.js";
+
+export interface AtlascopcoListingClassification {
+  productsMaterialCodes: Set<string>;
+  sparePartsMaterialCodes: Set<string>;
+}
+
+export class AtlascopcoListingClassificationHelper {
+  static build(hits: readonly AtlascopcoAlgoliaHit[]): AtlascopcoListingClassification {
+    const productsMaterialCodes = new Set<string>();
+    const sparePartsMaterialCodes = new Set<string>();
+    for (const hit of hits) {
+      const materialCode = AtlascopcoProductParser.mapAlgoliaHit(hit).materialCode;
+      if (materialCode.length === 0) {
+        continue;
+      }
+      const categoryPath = AtlascopcoProductParser.resolveCategoryPathFromAlgolia(hit);
+      if (categoryPath[0] === AtlascopcoCatalogConstants.SPARE_PARTS_LEVEL0) {
+        sparePartsMaterialCodes.add(materialCode);
+      } else if (categoryPath[0] === AtlascopcoCatalogConstants.PRODUCTS_LEVEL0) {
+        productsMaterialCodes.add(materialCode);
+      }
+    }
+    const result: AtlascopcoListingClassification = { productsMaterialCodes, sparePartsMaterialCodes };
+    return result;
+  }
+
+  static supplementMissingListingItems(items: readonly AtlascopcoCatalogData[], listingHits: readonly AtlascopcoAlgoliaHit[], listing: AtlascopcoListingClassification, excludedListingMaterialCodes: ReadonlySet<string>, fetchedAt: string): AtlascopcoCatalogData[] {
+    const byMaterialCode = new Map(items.map((item) => [item.materialCode, item]));
+    const listingHitByMaterialCode = AtlascopcoListingClassificationHelper.buildListingHitByMaterialCode(listingHits);
+    for (const materialCode of listing.productsMaterialCodes) {
+      if (excludedListingMaterialCodes.has(materialCode) || byMaterialCode.has(materialCode)) {
+        continue;
+      }
+      const hit = listingHitByMaterialCode.get(materialCode);
+      if (hit === undefined) {
+        continue;
+      }
+      const item = AtlascopcoProductParser.parseAlgoliaHit(hit, fetchedAt);
+      if (item !== undefined) {
+        byMaterialCode.set(materialCode, item);
+      }
+    }
+    for (const materialCode of listing.sparePartsMaterialCodes) {
+      if (excludedListingMaterialCodes.has(materialCode) || byMaterialCode.has(materialCode)) {
+        continue;
+      }
+      const hit = listingHitByMaterialCode.get(materialCode);
+      if (hit === undefined) {
+        continue;
+      }
+      const item = AtlascopcoProductParser.parseAlgoliaHit(hit, fetchedAt);
+      if (item !== undefined) {
+        byMaterialCode.set(materialCode, item);
+      }
+    }
+    return [...byMaterialCode.values()].sort((left, right) => left.materialCode.localeCompare(right.materialCode));
+  }
+
+  private static buildListingHitByMaterialCode(hits: readonly AtlascopcoAlgoliaHit[]): Map<string, AtlascopcoAlgoliaHit> {
+    const byCode = new Map<string, AtlascopcoAlgoliaHit>();
+    for (const hit of hits) {
+      const materialCode = AtlascopcoProductParser.mapAlgoliaHit(hit).materialCode;
+      if (materialCode.length === 0 || byCode.has(materialCode)) {
+        continue;
+      }
+      byCode.set(materialCode, hit);
+    }
+    return byCode;
+  }
+
+  static reconcileListingCategories(items: readonly AtlascopcoCatalogData[], listing: AtlascopcoListingClassification): AtlascopcoCatalogData[] {
+    const reconciled: AtlascopcoCatalogData[] = [];
+    for (const item of items) {
+      if (listing.sparePartsMaterialCodes.has(item.materialCode)) {
+        const sparePartsItem: AtlascopcoCatalogData = { ...item, familyKey: `sku:${item.materialCode}`, categoryPath: [AtlascopcoCatalogConstants.SPARE_PARTS_LEVEL0] };
+        reconciled.push(sparePartsItem);
+        continue;
+      }
+      if (listing.productsMaterialCodes.has(item.materialCode) && item.categoryPath[0] !== AtlascopcoCatalogConstants.PRODUCTS_LEVEL0) {
+        const productsItem: AtlascopcoCatalogData = { ...item, categoryPath: [AtlascopcoCatalogConstants.PRODUCTS_LEVEL0] };
+        reconciled.push(productsItem);
+        continue;
+      }
+      reconciled.push(item);
+    }
+    return reconciled;
+  }
+}

package/src/site-catalogs/atlascopco/atlascopco-manifest-property-builder.ts

@@ -0,0 +1,156 @@
+import type { CatalogProperty } from "@hivepoint/navu-apis";
+import type { CatalogPropertyFieldScan, CatalogPropertyFieldScanEntry } from "../../manifest/catalog-property-field-scan-types.js";
+import { CatalogPropertyLabelHelper } from "../../manifest/catalog-property-label-helper.js";
+import { AtlascopcoPimFeatureSupport } from "./atlascopco-pim-feature-support.js";
+
+export class AtlascopcoManifestPropertyBuilder {
+  static readonly PINNED_TOKENS: readonly string[] = [
+    "product_type",
+    "pressure",
+    "motor_power",
+    "fad",
+    "noise",
+    "voltage",
+    "phase",
+    "frequency",
+    "configuration",
+    "integrated_dryer",
+    "regulation",
+    "image_url",
+  ];
+
+  static readonly PINNED_LABELS: Record<string, string> = {
+    product_type: "Product type",
+    pressure: "Working pressure (bar)",
+    motor_power: "Motor power (kW)",
+    fad: "Free air delivery (l/s)",
+    noise: "Noise level (dB)",
+    voltage: "Supply voltage (V)",
+    phase: "Phase",
+    frequency: "Frequency (Hz)",
+    configuration: "Configuration",
+    integrated_dryer: "Integrated dryer",
+    regulation: "Regulation",
+    image_url: "Image URL",
+  };
+
+  static buildVariantPropertiesFromScan(scan: CatalogPropertyFieldScan): CatalogProperty[] {
+    const properties: CatalogProperty[] = [];
+    const usedTokens = new Set<string>();
+    AtlascopcoManifestPropertyBuilder.appendCoreVariantProperties(properties, usedTokens);
+    for (const entry of scan.entries) {
+      if (entry.grain !== "variant" || usedTokens.has(entry.token)) {
+        continue;
+      }
+      AtlascopcoManifestPropertyBuilder.appendScanVariantProperty(properties, usedTokens, entry);
+    }
+    return properties;
+  }
+
+  static buildVariantProperties(): CatalogProperty[] {
+    const properties: CatalogProperty[] = [];
+    const usedTokens = new Set<string>();
+    AtlascopcoManifestPropertyBuilder.appendCoreVariantProperties(properties, usedTokens);
+    return properties;
+  }
+
+  static resolveLabel(token: string): string {
+    if (AtlascopcoPimFeatureSupport.isOpaqueHarvestToken(token)) {
+      return token;
+    }
+    return AtlascopcoManifestPropertyBuilder.PINNED_LABELS[token] ?? CatalogPropertyLabelHelper.inferLabelFromToken(token);
+  }
+
+  private static appendCoreVariantProperties(properties: CatalogProperty[], usedTokens: Set<string>): void {
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "pressure", "variants.pressure_bar", "number", "range");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "motor_power", "variants.motor_power_kw", "number", "range");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "fad", "variants.fad_ls", "number", "range");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "noise", "variants.noise_db", "number", "range");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "voltage", "variants.voltage_v", "number", "range");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "phase", "variants.phase", "number", "range");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "frequency", "variants.frequency_hz", "number", "range");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "configuration", "variants.configuration", "keyword", "terms");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "integrated_dryer", "variants.integrated_dryer", "boolean", "terms");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "regulation", "variants.regulation", "keyword", "terms");
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, "image_url", "variants.image_url", "keyword", "none", false);
+  }
+
+  private static appendScanVariantProperty(properties: CatalogProperty[], usedTokens: Set<string>, entry: CatalogPropertyFieldScanEntry): void {
+    if (AtlascopcoPimFeatureSupport.isOpaqueHarvestToken(entry.token) || AtlascopcoPimFeatureSupport.isTrapToken(entry.token)) {
+      return;
+    }
+    const facet = AtlascopcoManifestPropertyBuilder.facetForToken(entry.token, entry.type);
+    const type = AtlascopcoManifestPropertyBuilder.typeForEntry(entry);
+    AtlascopcoManifestPropertyBuilder.appendVariantProperty(properties, usedTokens, entry.token, entry.fieldPath, type, facet);
+  }
+
+  private static facetForToken(token: string, type: CatalogProperty["type"]): CatalogProperty["facet"] {
+    if (AtlascopcoPimFeatureSupport.isTrapToken(token)) {
+      return "none";
+    }
+    if (AtlascopcoPimFeatureSupport.isTermsToken(token)) {
+      return "terms";
+    }
+    if (AtlascopcoPimFeatureSupport.isRangeToken(token) || type === "number") {
+      return "range";
+    }
+    return "none";
+  }
+
+  private static typeForEntry(entry: CatalogPropertyFieldScanEntry): CatalogProperty["type"] {
+    if (entry.type === "boolean") {
+      return "boolean";
+    }
+    if (entry.type === "number" || AtlascopcoManifestPropertyBuilder.looksNumericToken(entry.token)) {
+      return "number";
+    }
+    return "keyword";
+  }
+
+  private static looksNumericToken(token: string): boolean {
+    if (AtlascopcoPimFeatureSupport.isRangeToken(token) || token === "phase") {
+      return true;
+    }
+    if (token.startsWith("cts_") || token.startsWith("tag_")) {
+      for (const suffix of CatalogPropertyLabelHelper.TOKEN_UNIT_SUFFIXES) {
+        if (token.endsWith(`_${suffix}`)) {
+          return true;
+        }
+      }
+      return false;
+    }
+    for (const suffix of CatalogPropertyLabelHelper.TOKEN_UNIT_SUFFIXES) {
+      if (token.endsWith(`_${suffix}`)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static appendVariantProperty(
+    properties: CatalogProperty[],
+    usedTokens: Set<string>,
+    token: string,
+    fieldPath: string,
+    type: CatalogProperty["type"],
+    facet: CatalogProperty["facet"],
+    filterable = true,
+  ): void {
+    if (usedTokens.has(token)) {
+      return;
+    }
+    usedTokens.add(token);
+    const property: CatalogProperty = {
+      token,
+      field_path: fieldPath,
+      type,
+      grain: "variant",
+      filterable,
+      facet,
+      scope: false,
+      volatile: false,
+      label: AtlascopcoManifestPropertyBuilder.resolveLabel(token),
+    };
+    properties.push(property);
+  }
+}