@factorypure/client-helpers 1.1.7 → 1.1.9

package/dist/index.js

@@ -3,6 +3,173 @@ import { startOfDay, subDays } from 'date-fns';
 import { Index } from 'flexsearch';
 import EnglishPreset from 'flexsearch/lang/en';
 import { z } from 'zod';
+/**
+ * @packageDocumentation
+ * @module @factorypure/client-helpers
+ *
+ * # Filter System v2
+ *
+ * This package provides an extensible, rule-based filtering system for scrape results.
+ *
+ * ## Key Concepts
+ *
+ * ### Filter Severity
+ * - **BLOCK**: Always hides the result, cannot be overridden
+ * - **WARNING**: Shows result with warning indicator, can be overridden by override rules
+ * - **INFO**: Metadata only, doesn't affect visibility
+ *
+ * ### Visibility States
+ * - **HIDDEN**: Result is filtered out
+ * - **VISIBLE_WITH_WARNINGS**: Result is shown but has warning indicators
+ * - **VISIBLE**: Result is shown without issues
+ *
+ * ### Filter Rules
+ * Rules implement the FilterRule interface and are evaluated against results.
+ * Each rule can:
+ * - Have a severity level (BLOCK, WARNING, INFO)
+ * - Have a priority (higher priority rules evaluated first)
+ * - Be enabled/disabled
+ * - Specify which override rules can override it
+ *
+ * ### Override Rules
+ * Special INFO-level rules that can override WARNING-level rules.
+ * Examples: SKU match, calculated SKU match, alternate SKU match
+ *
+ * ## Quick Start
+ *
+ * ### Using the Default Engine
+ * ```typescript
+ * import { filterScrapeResultsV2, createDefaultFilterEngine } from '@factorypure/client-helpers'
+ *
+ * const filteredResults = filterScrapeResultsV2({
+ *   scrapeResults,
+ *   variant,
+ *   variantScrapeOptions,
+ *   vendorScrapeOptions,
+ *   globalScrapeOptions,
+ * })
+ * ```
+ *
+ * ### Custom Configuration
+ * ```typescript
+ * const filteredResults = filterScrapeResultsV2({
+ *   scrapeResults,
+ *   variant,
+ *   variantScrapeOptions,
+ *   vendorScrapeOptions,
+ *   globalScrapeOptions,
+ *   filterConfig: {
+ *     rules: [
+ *       { id: 'high_price_outlier', enabled: false },
+ *       { id: 'competitor_exclusion', severity: FilterSeverity.BLOCK },
+ *       { id: 'refurbished_used', priority: 100 },
+ *     ]
+ *   }
+ * })
+ * ```
+ *
+ * ### Adding Custom Rules
+ * ```typescript
+ * import { createCustomFilterRule, FilterSeverity } from '@factorypure/client-helpers'
+ *
+ * const myCustomRule = createCustomFilterRule({
+ *   id: 'my_custom_rule',
+ *   name: 'My Custom Rule',
+ *   description: 'Filters results based on custom logic',
+ *   severity: FilterSeverity.WARNING,
+ *   priority: 50,
+ *   canBeOverridden: true,
+ *   overridableBy: ['sku_match'],
+ *   evaluate: (context) => {
+ *     if (context.result.title.includes('bad-keyword')) {
+ *       return {
+ *         ruleId: 'my_custom_rule',
+ *         severity: FilterSeverity.WARNING,
+ *         message: 'Contains bad keyword',
+ *         metadata: { keyword: 'bad-keyword' },
+ *         timestamp: new Date().toISOString(),
+ *       }
+ *     }
+ *     return null
+ *   }
+ * })
+ *
+ * const filteredResults = filterScrapeResultsV2({
+ *   scrapeResults,
+ *   variant,
+ *   variantScrapeOptions,
+ *   vendorScrapeOptions,
+ *   globalScrapeOptions,
+ *   customRules: [myCustomRule],
+ * })
+ * ```
+ *
+ * ### Using the Builder Pattern
+ * ```typescript
+ * import { FilterRuleBuilder, FilterSeverity } from '@factorypure/client-helpers'
+ *
+ * const myRule = new FilterRuleBuilder()
+ *   .id('price_above_threshold')
+ *   .name('Price Above Threshold')
+ *   .description('Filters results above a price threshold')
+ *   .severity(FilterSeverity.WARNING)
+ *   .priority(80)
+ *   .canBeOverridden(true)
+ *   .overridableBy(['sku_match'])
+ *   .evaluate((context) => {
+ *     if (context.result.extracted_price > 10000) {
+ *       return {
+ *         ruleId: 'price_above_threshold',
+ *         severity: FilterSeverity.WARNING,
+ *         message: 'Price exceeds $10,000',
+ *         metadata: { price: context.result.extracted_price },
+ *         timestamp: new Date().toISOString(),
+ *       }
+ *     }
+ *     return null
+ *   })
+ *   .build()
+ * ```
+ *
+ * ## Built-in Rules
+ *
+ * ### Block Rules (Cannot be overridden)
+ * - `high_price_outlier`: Filters results >15% more expensive than variant
+ * - `low_price_outlier`: Filters results significantly cheaper than variant
+ * - `date_outlier`: Filters results outside date window
+ * - `duplicate`: Filters duplicate results
+ * - `scam_source_exclusion`: Filters known scam sources
+ * - `manually_ignored`: Filters manually ignored/excluded results
+ * - `out_of_stock`: Filters out of stock items
+ *
+ * ### Warning Rules (Can be overridden)
+ * - `competitor_exclusion`: Filters excluded competitors
+ * - `search_exclusion`: Filters results not matching search criteria
+ * - `skip_sku`: Filters results with SKUs from skip list
+ * - `vendor_exclusion`: Filters results with excluded vendor names
+ * - `brand_mismatch`: Filters results with mismatched brands
+ * - `calculated_sku_mismatch`: Filters results with mismatched calculated SKUs
+ * - `critical_spec_mismatch`: Filters results with mismatched critical specs
+ * - `refurbished_used`: Filters refurbished/used items
+ *
+ * ### Override Rules
+ * - `sku_match`: SKU found in title
+ * - `calculated_sku_match`: Calculated SKU matches variant SKU
+ * - `alt_sku_match`: Alternate SKU matches
+ * - `product_id_match`: Product ID is linked
+ *
+ * ## Migration from Legacy System
+ *
+ * The legacy `filterScrapeResults` function is still available for backward compatibility.
+ * Results include both old fields (`hide_reasons`, `hide_override_reasons`) and new fields
+ * (`filter_results`, `visibility_state`) during the transition period.
+ *
+ * To migrate:
+ * 1. Replace `filterScrapeResults` with `filterScrapeResultsV2`
+ * 2. Update code to use `visibility_state` instead of `ignore_result`
+ * 3. Use `filter_results` for detailed filter information instead of string arrays
+ * 4. Configure rules as needed using `filterConfig` parameter
+ */
 export const regexUnitResultSchema = z.object({
     value: z.string(),
     source: z.string(),
@@ -22,6 +189,40 @@ export const regexUnitResultsSchema = z.object({
     mile: z.array(regexUnitResultSchema),
     ah: z.array(regexUnitResultSchema),
 });
+// ===============================
+// New Filter System - Core Types (Must come before schemas that use them)
+// ===============================
+/**
+ * Severity levels for filter rules
+ * - BLOCK: Always hides the result, cannot be overridden
+ * - WARNING: Shows result with warning indicator, can be overridden
+ * - INFO: Metadata only, doesn't affect visibility
+ */
+export var FilterSeverity;
+(function (FilterSeverity) {
+    FilterSeverity["BLOCK"] = "BLOCK";
+    FilterSeverity["WARNING"] = "WARNING";
+    FilterSeverity["INFO"] = "INFO";
+})(FilterSeverity || (FilterSeverity = {}));
+/**
+ * Visibility states for results after filter evaluation
+ */
+export var VisibilityState;
+(function (VisibilityState) {
+    VisibilityState["HIDDEN"] = "HIDDEN";
+    VisibilityState["VISIBLE_WITH_WARNINGS"] = "VISIBLE_WITH_WARNINGS";
+    VisibilityState["VISIBLE"] = "VISIBLE";
+})(VisibilityState || (VisibilityState = {}));
+/**
+ * Structured result from a filter rule evaluation
+ */
+export const filterResultSchema = z.object({
+    ruleId: z.string(),
+    severity: z.nativeEnum(FilterSeverity),
+    message: z.string(),
+    metadata: z.record(z.string(), z.any()).optional(),
+    timestamp: z.string(),
+});
 export const scrapeResultsSchema = z.object({
     id: z.number(),
     scrape_id: z.number(),
@@ -57,6 +258,9 @@ export const scrapeResultsSchema = z.object({
     ignore_reasons: z.array(z.string()),
     hide_reasons: z.array(z.string()),
     hide_override_reasons: z.array(z.string()),
+    // New filter system fields
+    filter_results: z.array(filterResultSchema).optional(),
+    visibility_state: z.nativeEnum(VisibilityState).optional(),
     regexUnitResults: z.nullable(regexUnitResultsSchema),
     // add these properties as null to make ts play nice with the ImmersiveScrapeResultsType union
     google_product_link: z.null().optional(),
@@ -121,6 +325,9 @@ export const immersiveScrapeResultsSchema = z.object({
     ignore_reasons: z.array(z.string()),
     hide_reasons: z.array(z.string()),
     hide_override_reasons: z.array(z.string()),
+    // New filter system fields
+    filter_results: z.array(filterResultSchema).optional(),
+    visibility_state: z.nativeEnum(VisibilityState).optional(),
     brand: z.string().nullable(),
     company_id: z.number().nullable(),
     regexUnitResults: z.nullable(regexUnitResultsSchema),
@@ -163,6 +370,21 @@ export const globalScrapeOptionsSchema = z.object({
     })),
     scam_sources: z.array(z.string()),
 });
+/**
+ * Overall filter configuration
+ */
+export const filterConfigurationSchema = z.object({
+    rules: z
+        .array(z.object({
+        id: z.string(),
+        enabled: z.boolean().optional(),
+        severity: z.nativeEnum(FilterSeverity).optional(),
+        priority: z.number().optional(),
+        parameters: z.record(z.string(), z.any()).optional(),
+    }))
+        .optional(),
+    globalParameters: z.record(z.string(), z.any()).optional(),
+});
 export const HIDE_REASONS = {
     IGNORED: 'Ignored',
     HIGH_PRICE_OUTLIER: 'Too Expensive',
@@ -187,6 +409,7 @@ export const HIDE_OVERRIDE_REASONS = {
     ALT_SKU_MATCH: 'Alt SKU Match',
     CALCULATED_SKU_MATCH: 'Calculated SKU Match',
     CALCULATED_SKU_PARTIAL_MATCH: 'Calculated SKU Partial Match',
+    CALCULATED_ALT_SKU_MATCH: 'Calc. Alt SKU Match',
 };
 const HIDE_ALWAYS_MAP = {
     [HIDE_REASONS.HIGH_PRICE_OUTLIER]: true,
@@ -208,6 +431,1004 @@ const HIDE_ALWAYS_MAP = {
 export const TOO_CHEAP_MULTIPLIER = 0.75;
 export const TOO_EXPENSIVE_MULTIPLIER = 1.15;
 // const wattages = Array.from({ length: 41 }, (_, i) => (5000 + i * 500).toString())
+// ===============================
+// Filter Rule Registry
+// ===============================
+/**
+ * Registry for filter rules with configuration management
+ */
+export class FilterRuleRegistry {
+    rules = new Map();
+    configurations = new Map();
+    /**
+     * Register a new filter rule
+     */
+    registerRule(rule) {
+        this.rules.set(rule.id, rule);
+    }
+    /**
+     * Get a rule by ID
+     */
+    getRule(id) {
+        return this.rules.get(id);
+    }
+    /**
+     * Get all registered rules
+     */
+    getAllRules() {
+        return Array.from(this.rules.values());
+    }
+    /**
+     * Get all enabled rules sorted by priority
+     */
+    getEnabledRules() {
+        return Array.from(this.rules.values())
+            .filter((rule) => {
+            const config = this.configurations.get(rule.id);
+            return config?.enabled !== false && rule.enabled !== false;
+        })
+            .sort((a, b) => b.priority - a.priority);
+    }
+    /**
+     * Apply configuration to rules
+     */
+    applyConfiguration(config) {
+        config.rules?.forEach((ruleConfig) => {
+            this.configurations.set(ruleConfig.id, ruleConfig);
+        });
+    }
+    /**
+     * Get effective configuration for a rule
+     */
+    getRuleConfig(ruleId) {
+        return this.configurations.get(ruleId);
+    }
+}
+// ===============================
+// Filter Engine
+// ===============================
+/**
+ * Engine for evaluating filter rules and determining visibility
+ */
+export class FilterEngine {
+    registry;
+    constructor(registry) {
+        this.registry = registry;
+    }
+    /**
+     * Evaluate all rules for a single result
+     */
+    evaluateResult(context) {
+        const results = [];
+        const rules = this.registry.getEnabledRules();
+        for (const rule of rules) {
+            try {
+                const result = rule.evaluate(context);
+                if (result) {
+                    results.push(result);
+                }
+            }
+            catch (error) {
+                console.error(`Error evaluating rule ${rule.id}:`, error);
+            }
+        }
+        return results;
+    }
+    /**
+     * Evaluate rules for a batch of results
+     */
+    evaluateBatch(results, variant, variantScrapeOptions, vendorScrapeOptions, globalScrapeOptions) {
+        return results.map((result) => {
+            const context = {
+                result,
+                variant,
+                variantScrapeOptions,
+                vendorScrapeOptions,
+                globalScrapeOptions,
+            };
+            const filterResults = this.evaluateResult(context);
+            const visibilityState = this.calculateVisibility(filterResults);
+            // Populate new fields
+            result.filter_results = filterResults;
+            result.visibility_state = visibilityState;
+            // Maintain backward compatibility with old fields
+            result.hide_reasons = filterResults
+                .filter((fr) => fr.severity === FilterSeverity.BLOCK || fr.severity === FilterSeverity.WARNING)
+                .map((fr) => fr.message);
+            result.hide_override_reasons = filterResults.filter((fr) => fr.metadata?.isOverride).map((fr) => fr.message);
+            result.ignore_result = visibilityState === VisibilityState.HIDDEN;
+            return result;
+        });
+    }
+    /**
+     * Calculate visibility state based on filter results
+     */
+    calculateVisibility(filterResults) {
+        if (filterResults.length === 0) {
+            return VisibilityState.VISIBLE;
+        }
+        // Check for override results
+        const overrideResults = filterResults.filter((fr) => fr.metadata?.isOverride);
+        const blockResults = filterResults.filter((fr) => fr.severity === FilterSeverity.BLOCK);
+        const warningResults = filterResults.filter((fr) => fr.severity === FilterSeverity.WARNING);
+        // Block results that cannot be overridden
+        const nonOverridableBlocks = blockResults.filter((br) => {
+            const rule = this.registry.getRule(br.ruleId);
+            if (!rule || !rule.canBeOverridden)
+                return true;
+            // Check if any override can override this block
+            const canBeOverridden = overrideResults.some((or) => {
+                const overrideRule = this.registry.getRule(or.ruleId);
+                return overrideRule && rule.overridableBy.includes(overrideRule.id);
+            });
+            return !canBeOverridden;
+        });
+        if (nonOverridableBlocks.length > 0) {
+            return VisibilityState.HIDDEN;
+        }
+        // Check for overridable blocks
+        const overridableBlocks = blockResults.filter((br) => {
+            const rule = this.registry.getRule(br.ruleId);
+            if (!rule || !rule.canBeOverridden)
+                return false;
+            const canBeOverridden = overrideResults.some((or) => {
+                const overrideRule = this.registry.getRule(or.ruleId);
+                return overrideRule && rule.overridableBy.includes(overrideRule.id);
+            });
+            return !canBeOverridden; // Still blocked if no override present
+        });
+        if (overridableBlocks.length > 0) {
+            return VisibilityState.HIDDEN;
+        }
+        // If we have warnings, show with warnings
+        if (warningResults.length > 0) {
+            return VisibilityState.VISIBLE_WITH_WARNINGS;
+        }
+        return VisibilityState.VISIBLE;
+    }
+}
+// ===============================
+// Built-in Filter Rules
+// ===============================
+/**
+ * Rule for filtering high price outliers
+ */
+class HighPriceOutlierRule {
+    id = 'high_price_outlier';
+    name = 'High Price Outlier';
+    description = 'Filters results that are significantly more expensive than the variant price';
+    severity = FilterSeverity.BLOCK;
+    priority = 100;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const multiplier = TOO_EXPENSIVE_MULTIPLIER;
+        const isMoreExpensive = context.result.extracted_price > context.variant.price * multiplier;
+        if (isMoreExpensive) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.HIGH_PRICE_OUTLIER,
+                metadata: {
+                    variantPrice: context.variant.price,
+                    resultPrice: context.result.extracted_price,
+                    threshold: multiplier,
+                },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for filtering low price outliers
+ */
+class LowPriceOutlierRule {
+    id = 'low_price_outlier';
+    name = 'Low Price Outlier';
+    description = 'Filters results that are significantly cheaper than the variant price';
+    severity = FilterSeverity.BLOCK;
+    priority = 100;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const multiplier = getUndercutThreshold(context.variant.price, context.globalScrapeOptions.undercut_threshold_ranges || []);
+        const isTooCheap = context.result.extracted_price < context.variant.price * multiplier;
+        if (isTooCheap) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.LOW_PRICE_OUTLIER,
+                metadata: {
+                    variantPrice: context.variant.price,
+                    resultPrice: context.result.extracted_price,
+                    threshold: multiplier,
+                },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for filtering outdated listings
+ */
+class DateOutlierRule {
+    id = 'date_outlier';
+    name = 'Date Outlier';
+    description = 'Filters results outside the specified date window';
+    severity = FilterSeverity.BLOCK;
+    priority = 90;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const itemDate = new Date(context.result.created_at);
+        const dayWindow = Number(context.variantScrapeOptions.day_window) || 3;
+        const variantDate = startOfDay(subDays(new Date(), dayWindow));
+        if (itemDate < variantDate) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.DATE_OUTLIER,
+                metadata: {
+                    itemDate: itemDate.toISOString(),
+                    threshold: variantDate.toISOString(),
+                    dayWindow,
+                },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for filtering competitor exclusions
+ */
+class CompetitorExclusionRule {
+    id = 'competitor_exclusion';
+    name = 'Competitor Exclusion';
+    description = 'Filters results from excluded competitor sources';
+    severity = FilterSeverity.WARNING;
+    priority = 80;
+    enabled = true;
+    canBeOverridden = true;
+    overridableBy = ['sku_match', 'calculated_sku_match', 'alt_sku_match'];
+    evaluate(context) {
+        const lowerSource = context.result.source.toLowerCase();
+        const isExcluded = context.variantScrapeOptions.competitor_exclusions.some((exclusion) => exclusion && lowerSource === exclusion.toLowerCase());
+        if (isExcluded) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.COMPETITOR_EXCLUSION,
+                metadata: { source: context.result.source },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for filtering duplicates
+ */
+class DuplicateRule {
+    id = 'duplicate';
+    name = 'Duplicate';
+    description = 'Marks duplicate results based on source, title, and price';
+    severity = FilterSeverity.BLOCK;
+    priority = 70;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    // Note: This rule requires batch context, so it's handled specially in batch evaluation
+    evaluate(context) {
+        // This is a placeholder - actual duplicate detection happens in batch processing
+        return null;
+    }
+}
+/**
+ * Rule for search exclusions
+ */
+class SearchExclusionRule {
+    id = 'search_exclusion';
+    name = 'Search Exclusion';
+    description = 'Filters results that do not match search criteria';
+    severity = FilterSeverity.WARNING;
+    priority = 60;
+    enabled = true;
+    canBeOverridden = true;
+    overridableBy = ['sku_match', 'calculated_sku_match'];
+    evaluate(context) {
+        // This is handled in batch processing due to FlexSearch requirement
+        return null;
+    }
+}
+/**
+ * Rule for SKU skip filtering
+ */
+class SkipSkuRule {
+    id = 'skip_sku';
+    name = 'Skip SKU';
+    description = 'Filters results containing SKUs from the skip list';
+    severity = FilterSeverity.WARNING;
+    priority = 50;
+    enabled = true;
+    canBeOverridden = true;
+    overridableBy = ['sku_match', 'calculated_sku_match'];
+    evaluate(context) {
+        const nots = [];
+        const formatted = context.vendorScrapeOptions.default_skip_skus
+            .filter((s) => s.toLowerCase() !== context.variant.sku.toLowerCase())
+            .map((s) => `${s}`);
+        nots.push(...formatted);
+        nots.push(...context.vendorScrapeOptions.vendor_skip_skus.filter((s) => s.toLowerCase() !== context.variant.sku.toLowerCase()));
+        const hasExclusion = nots.some((sku) => {
+            const escapedSku = sku.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
+            const skuRegex = new RegExp(`(?: |\/|\&|\=|"|'|\`)${escapedSku}(?: |\/|\&|\=|"|'|\`)`, 'gi');
+            const skuMatches = context.result.title?.match(skuRegex);
+            return skuMatches && skuMatches.length > 0;
+        });
+        if (hasExclusion) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.SKIP_SKU,
+                metadata: { title: context.result.title },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for vendor exclusions
+ */
+class VendorExclusionRule {
+    id = 'vendor_exclusion';
+    name = 'Vendor Exclusion';
+    description = 'Filters results containing excluded vendor names';
+    severity = FilterSeverity.WARNING;
+    priority = 50;
+    enabled = true;
+    canBeOverridden = true;
+    overridableBy = ['sku_match', 'calculated_sku_match'];
+    evaluate(context) {
+        const nots = [];
+        const formatted = context.vendorScrapeOptions.default_skip_vendors
+            .filter((s) => s.toLowerCase() !== context.variant.vendor.toLowerCase())
+            .map((s) => `${s}`);
+        nots.push(...formatted);
+        const hasExclusion = nots.some((vendor) => {
+            const escaped = vendor.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
+            const matchRegex = new RegExp(`(?: |\/|\&|\=|"|'|\`)${escaped}(?: |\/|\&|\=|"|'|\`)`, 'gi');
+            const matches = context.result.title?.match(matchRegex);
+            return matches && matches.length > 0;
+        });
+        if (hasExclusion) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.VENDOR_EXCLUSION,
+                metadata: { title: context.result.title },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for scam source exclusions
+ */
+class ScamSourceExclusionRule {
+    id = 'scam_source_exclusion';
+    name = 'Scam Source Exclusion';
+    description = 'Filters results from known scam sources';
+    severity = FilterSeverity.BLOCK;
+    priority = 95;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const lowerSource = context.result.source.toLowerCase();
+        const scamSources = context.globalScrapeOptions?.scam_sources || [];
+        const isScam = scamSources.some((exclusion) => exclusion && lowerSource === exclusion.toLowerCase());
+        if (isScam) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.SCAM_SOURCE_EXCLUSION,
+                metadata: { source: context.result.source },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for manually ignored/excluded results
+ */
+class ManuallyIgnoredRule {
+    id = 'manually_ignored';
+    name = 'Manually Ignored';
+    description = 'Filters results that have been manually ignored or excluded';
+    severity = FilterSeverity.BLOCK;
+    priority = 110;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const resultIgnoreKeys = context.variantScrapeOptions.result_ignore_keys;
+        if (!resultIgnoreKeys || resultIgnoreKeys.length === 0) {
+            return null;
+        }
+        for (const key of resultIgnoreKeys) {
+            const resultKeyValues = [context.variant.id];
+            const keyValues = [];
+            const keyParts = JSON.parse(key.key_parts);
+            keyParts.forEach((part) => {
+                keyValues.push(key[part]);
+                if (part !== 'variant_id') {
+                    resultKeyValues.push(context.result[part === 'price' ? 'extracted_price' : part]);
+                }
+            });
+            const parsedIgnoreKey = keyValues.join('-');
+            const resultIgnoreKey = resultKeyValues.join('-');
+            if (parsedIgnoreKey === resultIgnoreKey) {
+                const message = key.reason === 'Ignored' ? HIDE_REASONS.MANUALLY_IGNORED : HIDE_REASONS.MANUALLY_EXCLUDED;
+                return {
+                    ruleId: this.id,
+                    severity: this.severity,
+                    message,
+                    metadata: { reason: key.reason, keyParts },
+                    timestamp: new Date().toISOString(),
+                };
+            }
+        }
+        return null;
+    }
+}
+/**
+ * Rule for brand mismatches
+ */
+class BrandMismatchRule {
+    id = 'brand_mismatch';
+    name = 'Brand Mismatch';
+    description = 'Filters results with mismatched brands';
+    severity = FilterSeverity.WARNING;
+    priority = 40;
+    enabled = true;
+    canBeOverridden = true;
+    overridableBy = ['sku_match', 'calculated_sku_match', 'alt_sku_match'];
+    evaluate(context) {
+        if (!context.result.brand) {
+            return null;
+        }
+        const vendor = context.variant.vendor ? context.variant.vendor.toLowerCase() : null;
+        const brandNameAlternates = context.vendorScrapeOptions.brand_name_alternates;
+        const targetBrands = [vendor, ...(brandNameAlternates || [])].filter((b) => b).map((b) => b?.toLowerCase());
+        const calcBrandMatched = context.result.brand && context.result.brand !== ''
+            ? targetBrands.includes(context.result.brand.toLowerCase())
+            : null;
+        if (calcBrandMatched === false) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.CALCULATED_BRAND_MISMATCH,
+                metadata: {
+                    resultBrand: context.result.brand,
+                    expectedBrands: targetBrands,
+                },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for out of stock items
+ */
+class OutOfStockRule {
+    id = 'out_of_stock';
+    name = 'Out of Stock';
+    description = 'Filters results that are out of stock online';
+    severity = FilterSeverity.BLOCK;
+    priority = 85;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const details = context.result.details_and_offers;
+        if (details && details.includes('Out of stock online')) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.OUT_OF_STOCK_ONLINE,
+                metadata: { details },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for calculated SKU mismatches
+ */
+class CalculatedSkuMismatchRule {
+    id = 'calculated_sku_mismatch';
+    name = 'Calculated SKU Mismatch';
+    description = 'Filters results with mismatched calculated SKUs';
+    severity = FilterSeverity.WARNING;
+    priority = 30;
+    enabled = true;
+    canBeOverridden = true;
+    overridableBy = ['sku_match', 'calculated_sku_match'];
+    evaluate(context) {
+        const result = context.result;
+        if (result.calculated_sku && result.calculated_sku?.toLowerCase() !== context.variant.sku.toLowerCase()) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.CALCULATED_SKU_MISMATCH,
+                metadata: {
+                    calculatedSku: result.calculated_sku,
+                    variantSku: context.variant.sku,
+                },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for critical spec mismatches
+ */
+class CriticalSpecMismatchRule {
+    id = 'critical_spec_mismatch';
+    name = 'Critical Spec Mismatch';
+    description = 'Filters results with mismatched critical specifications';
+    severity = FilterSeverity.WARNING;
+    priority = 35;
+    enabled = true;
+    canBeOverridden = true;
+    overridableBy = ['sku_match', 'calculated_sku_match'];
+    evaluate(context) {
+        if (!context.result.regexUnitResults || !context.variant.regexUnitResults) {
+            return null;
+        }
+        const [regexIgnore, regexIgnoreReasons] = getRegexUnitResultsIgnore(context.result.regexUnitResults, context.variant.regexUnitResults);
+        if (regexIgnore) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.CRITICAL_SPEC_MISMATCH,
+                metadata: { reasons: regexIgnoreReasons },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Rule for refurbished/used items
+ */
+class RefurbishedUsedRule {
+    id = 'refurbished_used';
+    name = 'Refurbished/Used';
+    description = 'Filters refurbished or used items';
+    severity = FilterSeverity.WARNING;
+    priority = 25;
+    enabled = true;
+    canBeOverridden = true;
+    overridableBy = ['sku_match', 'calculated_sku_match'];
+    evaluate(context) {
+        const refurbishedKeywords = getRefurbishedKeywords(context.variant);
+        const normalizedTitle = context.result.title?.toLowerCase().replace(/[^a-z0-9-]/g, ' ') ?? '';
+        const details = context.result.details_and_offers;
+        const isRefurbished = refurbishedKeywords.some((keyword) => normalizedTitle.split(/\s+/).includes(keyword)) ||
+            (details && details.includes('Pre-owned'));
+        if (isRefurbished) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_REASONS.REFURBISHED_USED,
+                metadata: { title: context.result.title },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+// ===============================
+// Override Rules
+// ===============================
+/**
+ * Override rule for SKU matches in title
+ */
+class SkuMatchOverrideRule {
+    id = 'sku_match';
+    name = 'SKU Match';
+    description = 'Overrides hide reasons when SKU is found in title';
+    severity = FilterSeverity.INFO;
+    priority = 200;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const escapedSku = context.variant.sku.replace(/\+/g, '\\+');
+        const skuRegex = new RegExp(`(?:\\b|[\\(\\[\\{])${escapedSku}(?:\\b|[\\)\\]\\}])`, 'gi');
+        if (skuRegex.test(context.result.title)) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_OVERRIDE_REASONS.SKU_MATCH,
+                metadata: { isOverride: true, sku: context.variant.sku },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Override rule for calculated SKU matches
+ */
+class CalculatedSkuMatchOverrideRule {
+    id = 'calculated_sku_match';
+    name = 'Calculated SKU Match';
+    description = 'Overrides hide reasons when calculated SKU matches';
+    severity = FilterSeverity.INFO;
+    priority = 200;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const result = context.result;
+        if (context.variant.sku.toLowerCase() === result.calculated_sku?.toLowerCase()) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_OVERRIDE_REASONS.CALCULATED_SKU_MATCH,
+                metadata: { isOverride: true, calculatedSku: result.calculated_sku },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Override rule for alternate SKU matches
+ */
+class AltSkuMatchOverrideRule {
+    id = 'alt_sku_match';
+    name = 'Alt SKU Match';
+    description = 'Overrides hide reasons when alternate SKU matches';
+    severity = FilterSeverity.INFO;
+    priority = 200;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        const skuAlternates = context.variantScrapeOptions.sku_alternates;
+        // Check title match
+        if (skuAlternates.some((alt) => context.result.title?.toLowerCase().includes(alt.toLowerCase()))) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_OVERRIDE_REASONS.ALT_SKU_MATCH,
+                metadata: { isOverride: true },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        // Check calculated SKU match
+        const result = context.result;
+        if (skuAlternates.some((alt) => result.calculated_sku?.toLowerCase() === alt.toLowerCase())) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: HIDE_OVERRIDE_REASONS.CALCULATED_ALT_SKU_MATCH,
+                metadata: { isOverride: true, calculatedSku: result.calculated_sku },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+/**
+ * Override rule for product ID matches
+ */
+class ProductIdMatchOverrideRule {
+    id = 'product_id_match';
+    name = 'Product ID Match';
+    description = 'Overrides hide reasons when product ID is linked';
+    severity = FilterSeverity.INFO;
+    priority = 200;
+    enabled = true;
+    canBeOverridden = false;
+    overridableBy = [];
+    evaluate(context) {
+        if (context.variant.found_product_ids &&
+            context.variant.found_product_ids.includes(context.result.found_product_id)) {
+            return {
+                ruleId: this.id,
+                severity: this.severity,
+                message: 'Product Id Linked',
+                metadata: { isOverride: true, foundProductId: context.result.found_product_id },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        return null;
+    }
+}
+// ===============================
+// Registry Factory
+// ===============================
+/**
+ * Create and populate a registry with all built-in rules
+ */
+export function createDefaultFilterRegistry() {
+    const registry = new FilterRuleRegistry();
+    // Register all filter rules
+    registry.registerRule(new HighPriceOutlierRule());
+    registry.registerRule(new LowPriceOutlierRule());
+    registry.registerRule(new DateOutlierRule());
+    registry.registerRule(new CompetitorExclusionRule());
+    registry.registerRule(new DuplicateRule());
+    registry.registerRule(new SearchExclusionRule());
+    registry.registerRule(new SkipSkuRule());
+    registry.registerRule(new VendorExclusionRule());
+    registry.registerRule(new ScamSourceExclusionRule());
+    registry.registerRule(new ManuallyIgnoredRule());
+    registry.registerRule(new BrandMismatchRule());
+    registry.registerRule(new OutOfStockRule());
+    registry.registerRule(new CalculatedSkuMismatchRule());
+    registry.registerRule(new CriticalSpecMismatchRule());
+    registry.registerRule(new RefurbishedUsedRule());
+    // Register override rules
+    registry.registerRule(new SkuMatchOverrideRule());
+    registry.registerRule(new CalculatedSkuMatchOverrideRule());
+    registry.registerRule(new AltSkuMatchOverrideRule());
+    registry.registerRule(new ProductIdMatchOverrideRule());
+    return registry;
+}
+/**
+ * Create a filter engine with default configuration
+ */
+export function createDefaultFilterEngine(config) {
+    const registry = createDefaultFilterRegistry();
+    if (config) {
+        registry.applyConfiguration(config);
+    }
+    return new FilterEngine(registry);
+}
+// ===============================
+// Extensibility Helpers
+// ===============================
+/**
+ * Builder for creating custom filter rules
+ */
+export class FilterRuleBuilder {
+    rule = {
+        enabled: true,
+        priority: 50,
+        canBeOverridden: false,
+        overridableBy: [],
+    };
+    id(id) {
+        this.rule.id = id;
+        return this;
+    }
+    name(name) {
+        this.rule.name = name;
+        return this;
+    }
+    description(description) {
+        this.rule.description = description;
+        return this;
+    }
+    severity(severity) {
+        this.rule.severity = severity;
+        return this;
+    }
+    priority(priority) {
+        this.rule.priority = priority;
+        return this;
+    }
+    enabled(enabled) {
+        this.rule.enabled = enabled;
+        return this;
+    }
+    canBeOverridden(canBeOverridden) {
+        this.rule.canBeOverridden = canBeOverridden;
+        return this;
+    }
+    overridableBy(ruleIds) {
+        this.rule.overridableBy = ruleIds;
+        return this;
+    }
+    evaluate(evaluateFn) {
+        this.rule.evaluate = evaluateFn;
+        return this;
+    }
+    build() {
+        if (!this.rule.id || !this.rule.name || !this.rule.description || !this.rule.severity || !this.rule.evaluate) {
+            throw new Error('FilterRuleBuilder: Missing required fields');
+        }
+        return this.rule;
+    }
+}
+/**
+ * Simple factory for creating custom rules without implementing the full interface
+ */
+export function createCustomFilterRule(config) {
+    return {
+        id: config.id,
+        name: config.name,
+        description: config.description,
+        severity: config.severity,
+        priority: config.priority ?? 50,
+        enabled: true,
+        canBeOverridden: config.canBeOverridden ?? false,
+        overridableBy: config.overridableBy ?? [],
+        evaluate: config.evaluate,
+    };
+}
+// ===============================
+// Enhanced Batch Filtering (New API)
+// ===============================
+/**
+ * Filter scrape results using the new rule-based engine
+ * This provides more flexibility and better extensibility than the legacy filterScrapeResults
+ */
+export const filterScrapeResultsV2 = ({ scrapeResults, variant, variantScrapeOptions, vendorScrapeOptions, globalScrapeOptions, filterConfig, customRules, }) => {
+    const registry = createDefaultFilterRegistry();
+    // Register custom rules if provided
+    if (customRules) {
+        customRules.forEach((rule) => registry.registerRule(rule));
+    }
+    // Apply configuration
+    if (filterConfig) {
+        registry.applyConfiguration(filterConfig);
+    }
+    const engine = new FilterEngine(registry);
+    // Handle batch-specific rules manually (duplicates and search exclusions)
+    let results = handleDuplicatesV2(scrapeResults);
+    results = handleSearchExclusionsV2(results, variantScrapeOptions, variant, registry);
+    // Evaluate all other rules
+    results = engine.evaluateBatch(results, variant, variantScrapeOptions, vendorScrapeOptions, globalScrapeOptions);
+    return results;
+};
+/**
+ * Handle duplicates with new filter system
+ */
+function handleDuplicatesV2(results) {
+    const filteredUniqueResultsMap = {};
+    results.forEach((item) => {
+        const key = `${item.source}-${item.title}-${item.extracted_price}`;
+        if (!filteredUniqueResultsMap[key]) {
+            filteredUniqueResultsMap[key] = item;
+        }
+        else {
+            const existingItem = filteredUniqueResultsMap[key];
+            if (new Date(item.created_at) > new Date(existingItem.created_at)) {
+                // Mark old item as duplicate
+                const foundResult = results.find((res) => res.id === existingItem.id);
+                if (foundResult) {
+                    if (!foundResult.filter_results) {
+                        foundResult.filter_results = [];
+                    }
+                    foundResult.filter_results.push({
+                        ruleId: 'duplicate',
+                        severity: FilterSeverity.BLOCK,
+                        message: HIDE_REASONS.DUPLICATE,
+                        metadata: { key },
+                        timestamp: new Date().toISOString(),
+                    });
+                }
+                filteredUniqueResultsMap[key] = item;
+            }
+            else {
+                // Mark current item as duplicate
+                if (!item.filter_results) {
+                    item.filter_results = [];
+                }
+                item.filter_results.push({
+                    ruleId: 'duplicate',
+                    severity: FilterSeverity.BLOCK,
+                    message: HIDE_REASONS.DUPLICATE,
+                    metadata: { key },
+                    timestamp: new Date().toISOString(),
+                });
+            }
+        }
+    });
+    return results;
+}
+/**
+ * Handle search exclusions with new filter system
+ */
+function handleSearchExclusionsV2(dataToSearch, options, variant, registry) {
+    if (!options.search_exclusions || options.search_exclusions.length === 0) {
+        return dataToSearch;
+    }
+    function customEncoder(content) {
+        const tokens = [];
+        const str = content.toLowerCase();
+        const cleanedStr = str.replace(/[\/-]/g, ' ');
+        const cleanedStr2 = cleanedStr.replace(/[^a-z0-9,\/\s]/gi, '');
+        const words = cleanedStr2.split(/\s+/);
+        for (let word of words) {
+            tokens.push(word);
+        }
+        return tokens;
+    }
+    const index = new Index({
+        // @ts-ignore
+        charset: EnglishPreset,
+        encode: customEncoder,
+        tokenize: 'strict',
+    });
+    dataToSearch.forEach((item, id) => {
+        index.add(id, item.title);
+    });
+    const searchTerms = options.match_values;
+    let final = null;
+    searchTerms.forEach((term) => {
+        if (final === null) {
+            final = index.search(term, {
+                resolve: false,
+                suggest: true,
+            });
+        }
+        else {
+            final = final.or({
+                index: index,
+                query: term,
+                resolve: false,
+                suggest: true,
+            });
+        }
+    });
+    final = final.and({
+        index: index,
+        query: variant.sku,
+        resolve: false,
+        suggest: true,
+    });
+    const nots = [];
+    nots.push(...options.search_exclusions);
+    nots.forEach((term) => {
+        final = final.not({
+            index: index,
+            query: term,
+            resolve: false,
+        });
+    });
+    const result = final.resolve({ limit: 1000 });
+    for (let idx = 0; idx < dataToSearch.length; idx++) {
+        const element = dataToSearch[idx];
+        if (!result.includes(idx)) {
+            if (!element.filter_results) {
+                element.filter_results = [];
+            }
+            const rule = registry.getRule('search_exclusion');
+            if (rule) {
+                element.filter_results.push({
+                    ruleId: 'search_exclusion',
+                    severity: rule.severity,
+                    message: HIDE_REASONS.SEARCH_EXCLUSION,
+                    metadata: { searchTerms: searchTerms },
+                    timestamp: new Date().toISOString(),
+                });
+            }
+        }
+    }
+    return dataToSearch;
+}
+// ===============================
+// Legacy Filter Functions (Maintained for backward compatibility)
+// ===============================
 export const filterScrapeResults = ({ scrapeResults, variant, variantScrapeOptions, vendorScrapeOptions, globalScrapeOptions, }) => {
     let filteredResults = scrapeResults;
     filteredResults = filterPriceOutliers(filteredResults, variant.price, globalScrapeOptions);
@@ -558,9 +1779,12 @@ export const calculateHideOverrideReasons = (result, variant, sku_alternates) =>
         // TODO ALEX - Does this work? Is it doing anything?
         hide_override_reasons.push('Product Id Linked');
     }
-    if (sku_alternates.some((alt) => result.title.includes(alt))) {
+    if (sku_alternates.some((alt) => result?.title?.toLowerCase().includes(alt.toLowerCase()))) {
         hide_override_reasons.push(HIDE_OVERRIDE_REASONS.ALT_SKU_MATCH);
     }
+    if (sku_alternates.some((alt) => result?.calculated_sku?.toLowerCase() === alt.toLowerCase())) {
+        hide_override_reasons.push(HIDE_OVERRIDE_REASONS.CALCULATED_ALT_SKU_MATCH);
+    }
     return hide_override_reasons;
 };
 function getRegexUnitResultsIgnore(resultRegexResults, variantRegexResults, criticalUnits = ['ton', 'cc', 'hp']) {