@wix/agent-skills 1.15.0 → 1.16.0

package/skills/wix-replatform/resources/rp-source-csv/lib/canonical-record.js

@@ -0,0 +1,136 @@
+'use strict';
+
+// rp-source-csv — reader shape -> CANONICAL record.
+//
+// This closes the seam between the two deterministic halves:
+//
+//   csv rows --[reader]--> reader-shaped record --[HERE]--> canonical record --[wix-build]--> Wix
+//
+// A reader names fields the way its vendor thinks (`bodyHtml`, `vendor`, `weightKg`); `wix-build.js`
+// consumes the vendor-neutral vocabulary from `wix-target-spec.js` (`description`, `brand`,
+// `weight`). Without this module that rename is hand-written per project — which is exactly the
+// improvisation the deterministic pipeline exists to remove.
+//
+// The maps are DATA, per vendor, so adding a vendor is a data edit plus a fixture.
+//
+// Longer term the readers should emit canonical records directly and this module becomes a no-op
+// identity map. It is deliberately a separate step rather than being folded into the reader, so the
+// change can happen per vendor without a flag day.
+
+const { validateCanonicalPath } = require('../../rp-target-wix/lib/wix-target-spec.js');
+
+// `rename` moves a value unchanged. `derive` computes one from the whole record — kept to genuine
+// multi-field logic, because a derive that only reads one field should be a rename.
+const READER_FIELD_MAPS = {
+  shopify: {
+    product: {
+      rename: {
+        title: 'name',
+        bodyHtml: 'description',
+        slug: 'slug',
+        vendor: 'brand',
+        seoTitle: 'seoTitle',
+        seoDescription: 'seoDescription',
+        images: 'images',
+        optionNames: 'optionNames',
+        categoryPath: 'categoryPath',
+        tags: 'tags',
+        sourceId: 'sourceId',
+      },
+      derive: {
+        // Shopify carries BOTH a lifecycle status and a publish flag; a product is visible only
+        // when both agree. Collapsing to either one alone silently publishes drafts.
+        visible: (record) => record.status === 'active' && record.published !== false,
+      },
+    },
+    variant: {
+      rename: {
+        sku: 'sku',
+        price: 'price',
+        compareAtPrice: 'compareAtPrice',
+        // The reader has already converted grams to the site's weight unit; the canonical field
+        // carries a bare number, so the converted value is the one that travels.
+        weightKg: 'weight',
+        barcode: 'barcode',
+        cost: 'cost',
+        choices: 'choices',
+      },
+      derive: {
+        visible: () => true,
+      },
+    },
+    category: {
+      rename: { name: 'name', parentPath: 'parentPath', path: 'path', sourceId: 'sourceId' },
+      derive: { visible: () => true },
+    },
+  },
+};
+
+// woocommerce / magento / bigcommerce readers are not built yet. Registering them explicitly as
+// absent is better than falling back to shopify's map and silently mis-renaming fields.
+const SUPPORTED_VENDORS = Object.keys(READER_FIELD_MAPS);
+
+function mapEntity(record, entityMap) {
+  const out = {};
+  for (const [readerKey, canonicalKey] of Object.entries(entityMap.rename || {})) {
+    const value = record[readerKey];
+    if (value !== undefined && value !== null) out[canonicalKey] = value;
+  }
+  for (const [canonicalKey, fn] of Object.entries(entityMap.derive || {})) {
+    out[canonicalKey] = fn(record);
+  }
+  return out;
+}
+
+function mapFor(vendor, entity) {
+  const vendorMap = READER_FIELD_MAPS[vendor];
+  if (!vendorMap) {
+    throw new Error(
+      `canonical-record: no reader field map for vendor "${vendor}". ` +
+        `Known: ${SUPPORTED_VENDORS.join(', ')}. Add one in READER_FIELD_MAPS rather than reusing another vendor's.`,
+    );
+  }
+  const entityMap = vendorMap[entity];
+  if (!entityMap) throw new Error(`canonical-record: vendor "${vendor}" has no map for entity "${entity}"`);
+  return entityMap;
+}
+
+function toCanonicalProduct(record, { vendor = 'shopify' } = {}) {
+  const product = mapEntity(record, mapFor(vendor, 'product'));
+  const variantMap = mapFor(vendor, 'variant');
+  product.variants = (record.variants || []).map((variant) => mapEntity(variant, variantMap));
+  return product;
+}
+
+function toCanonicalCategory(record, { vendor = 'shopify' } = {}) {
+  return mapEntity(record, mapFor(vendor, 'category'));
+}
+
+// Self-check: every canonical key a map produces must be a real canonical field. Runs at require
+// time so a typo in the map is a load error, not a field that silently never lands in Wix.
+function validateMaps() {
+  const problems = [];
+  for (const [vendor, entities] of Object.entries(READER_FIELD_MAPS)) {
+    for (const [entity, entityMap] of Object.entries(entities)) {
+      const keys = [...Object.values(entityMap.rename || {}), ...Object.keys(entityMap.derive || {})];
+      for (const key of keys) {
+        // Canonical paths are `<entity>.<field>`; the maps hold the bare field name.
+        if (!validateCanonicalPath(`${entity}.${key}`)) problems.push(`${vendor}.${entity} -> ${entity}.${key}`);
+      }
+    }
+  }
+  return problems;
+}
+
+const MAP_PROBLEMS = validateMaps();
+if (MAP_PROBLEMS.length) {
+  throw new Error(`canonical-record: map targets are not canonical fields: ${MAP_PROBLEMS.join(', ')}`);
+}
+
+module.exports = {
+  READER_FIELD_MAPS,
+  SUPPORTED_VENDORS,
+  toCanonicalProduct,
+  toCanonicalCategory,
+  validateMaps,
+};

package/skills/wix-replatform/resources/rp-source-csv/lib/csv-fingerprint.js

@@ -0,0 +1,310 @@
+'use strict';
+
+// Vendor detection from the header row.
+//
+// The header read at discovery time is authoritative; a vendor profile only
+// (a) identifies the vendor and (b) pre-fills mappings the LLM would otherwise
+// propose. A stale profile can therefore make the pre-fill less complete, but
+// can never cause a wrong import — see SKILL.md → Profile drift & resilience.
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { normalizeHeaderName } = require('./csv-parse.js');
+
+const VENDORS_DIR = path.join(__dirname, '..', 'vendors');
+
+// Scoring constants. Deliberately conservative: falling back to `custom` costs
+// the user a less complete mapping pre-fill, while a wrong vendor match costs
+// them a wrong layout.
+const BASE_REQUIRED = 0.5;
+const STRONG_WEIGHT = 0.5;
+const PARTIAL_FACTOR = 0.6;
+const NEGATIVE_PENALTY = 0.35;
+const MATCH_THRESHOLD = 0.7;
+const MARGIN = 0.15;
+const MIN_STRONG_HITS = 2;
+const NEAR_MISS_FLOOR = 0.25;
+const NEAR_MISS_STRONG_RATIO = 0.6;
+
+const CUSTOM_VENDOR = 'custom';
+
+function normalizedSet(columns) {
+  const set = new Set();
+  for (const column of columns || []) {
+    const normalized = normalizeHeaderName(column);
+    if (normalized) {
+      set.add(normalized);
+    }
+  }
+  return set;
+}
+
+function validateProfile(profile) {
+  const errors = [];
+  if (!profile || typeof profile !== 'object') {
+    return { ok: false, errors: ['profile is not an object'] };
+  }
+  if (!profile.vendor || typeof profile.vendor !== 'string') {
+    errors.push('vendor must be a non-empty string');
+  }
+  if (!profile.profileVersion) {
+    errors.push('profileVersion is required (provenance for drift)');
+  }
+  if (!profile.sourceOfTruth) {
+    errors.push('sourceOfTruth is required');
+  }
+  const anchors = profile.anchors || {};
+  if (!Array.isArray(anchors.required) || anchors.required.length === 0) {
+    errors.push('anchors.required must be a non-empty array');
+  }
+  if (!Array.isArray(anchors.strong) || anchors.strong.length < MIN_STRONG_HITS) {
+    errors.push(`anchors.strong must hold at least ${MIN_STRONG_HITS} columns`);
+  }
+  if (!profile.layout || typeof profile.layout.pattern !== 'string') {
+    errors.push('layout.pattern is required');
+  }
+  if (profile.columnMap && !Array.isArray(profile.columnMap)) {
+    errors.push('columnMap must be an array when present');
+  }
+  for (const entry of profile.columnMap || []) {
+    if (!entry.wixTarget || !Array.isArray(entry.aliases) || entry.aliases.length === 0) {
+      errors.push(`columnMap entry ${JSON.stringify(entry.wixTarget)} needs a wixTarget and a non-empty aliases array`);
+    }
+  }
+  for (const derived of profile.derived || []) {
+    if (!derived.entity || !derived.fromColumn) {
+      errors.push('each derived entry needs entity + fromColumn');
+    }
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+function loadVendorProfiles(dir = VENDORS_DIR) {
+  const files = fs.readdirSync(dir)
+    .filter((file) => file.endsWith('.json'))
+    .sort();
+  return files.map((file) => {
+    const profile = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8'));
+    const validation = validateProfile(profile);
+    if (!validation.ok) {
+      throw new Error(`Invalid vendor profile ${file}: ${validation.errors.join('; ')}`);
+    }
+    return profile;
+  });
+}
+
+function aliasIndex(profile) {
+  // normalized column name -> wixTarget, built from columnMap only.
+  const index = new Map();
+  for (const entry of profile.columnMap || []) {
+    for (const alias of entry.aliases) {
+      const normalized = normalizeHeaderName(alias);
+      if (normalized && !index.has(normalized)) {
+        index.set(normalized, { wixTarget: entry.wixTarget, alias });
+      }
+    }
+  }
+  return index;
+}
+
+// Anchors match by normalized EXACT equality and never through columnMap
+// aliases. Aliases exist to be permissive for mapping; permissiveness is poison
+// for identification — Shopify's variant.sku aliases include a bare "SKU",
+// which every WooCommerce and Magento export also has.
+function scoreProfile(headerColumns, profile) {
+  const header = normalizedSet(headerColumns);
+  const anchors = profile.anchors || {};
+  const required = anchors.required || [];
+  const strong = anchors.strong || [];
+  const negative = anchors.negative || [];
+
+  const matchedRequired = required.filter((column) => header.has(normalizeHeaderName(column)));
+  const missingRequired = required.filter((column) => !header.has(normalizeHeaderName(column)));
+  const matchedStrong = strong.filter((column) => header.has(normalizeHeaderName(column)));
+  const missingStrong = strong.filter((column) => !header.has(normalizeHeaderName(column)));
+  const negativeHits = negative.filter((column) => header.has(normalizeHeaderName(column)));
+
+  const requiredRatio = required.length === 0 ? 0 : matchedRequired.length / required.length;
+  const strongRatio = strong.length === 0 ? 0 : matchedStrong.length / strong.length;
+
+  const core = BASE_REQUIRED + STRONG_WEIGHT * strongRatio;
+  let score = requiredRatio === 1 ? core : PARTIAL_FACTOR * requiredRatio * core;
+  if (negativeHits.length > 0) {
+    score *= NEGATIVE_PENALTY;
+  }
+
+  // Evidence only — deliberately NOT part of the score. Coupling identification
+  // to columnMap completeness would mean adding a mapping alias could change
+  // which vendor a file is detected as.
+  const aliases = aliasIndex(profile);
+  const anchorNames = normalizedSet([...required, ...strong]);
+  const knownColumns = (headerColumns || []).filter((column) => {
+    const normalized = normalizeHeaderName(column);
+    return anchorNames.has(normalized) || aliases.has(normalized);
+  });
+  const vocabularyCoverage = headerColumns && headerColumns.length > 0
+    ? knownColumns.length / headerColumns.length
+    : 0;
+
+  return {
+    vendor: profile.vendor,
+    score: Number(score.toFixed(4)),
+    requiredRatio,
+    strongRatio,
+    matchedRequired,
+    missingRequired,
+    matchedStrong,
+    missingStrong,
+    negativeHits,
+    vocabularyCoverage: Number(vocabularyCoverage.toFixed(4)),
+  };
+}
+
+function evidenceLine(winner, runnerUp) {
+  const parts = [
+    `required ${winner.matchedRequired.length}/${winner.matchedRequired.length + winner.missingRequired.length}`,
+    `strong ${winner.matchedStrong.length}/${winner.matchedStrong.length + winner.missingStrong.length}`,
+    `score ${winner.score}`,
+  ];
+  if (winner.negativeHits.length > 0) {
+    parts.push(`negative anchors present: ${winner.negativeHits.join(', ')}`);
+  }
+  parts.push(runnerUp ? `runner-up ${runnerUp.vendor} ${runnerUp.score}` : 'no runner-up');
+  return parts.join('; ');
+}
+
+function detectVendor(headerColumns, profiles, { statedVendor = null } = {}) {
+  const scores = profiles
+    .map((profile) => scoreProfile(headerColumns, profile))
+    .sort((a, b) => b.score - a.score || a.vendor.localeCompare(b.vendor));
+
+  const winner = scores[0] || null;
+  const runnerUp = scores[1] || null;
+  const profileByVendor = new Map(profiles.map((profile) => [profile.vendor, profile]));
+
+  let reason = 'match';
+  let matched = Boolean(winner);
+  if (!winner || winner.score < MATCH_THRESHOLD) {
+    matched = false;
+    reason = 'below-threshold';
+  } else if (winner.matchedStrong.length < MIN_STRONG_HITS) {
+    matched = false;
+    reason = 'insufficient-strong-anchors';
+  } else if (runnerUp && winner.score - runnerUp.score < MARGIN) {
+    matched = false;
+    reason = 'ambiguous';
+  }
+
+  // A drifted vendor export should become a user question, not silence.
+  let nearMiss = null;
+  if (!matched) {
+    const candidate = scores.find((entry) => entry.score >= NEAR_MISS_FLOOR
+      && entry.strongRatio >= NEAR_MISS_STRONG_RATIO
+      && entry.requiredRatio > 0
+      && entry.negativeHits.length === 0);
+    if (candidate) {
+      nearMiss = {
+        vendor: candidate.vendor,
+        score: candidate.score,
+        missingRequired: candidate.missingRequired,
+        missingStrong: candidate.missingStrong,
+      };
+    }
+  }
+
+  const detectedVendor = matched ? winner.vendor : CUSTOM_VENDOR;
+
+  // A user-stated vendor wins, but detection still runs so a disagreement is
+  // visible rather than silently overridden.
+  if (statedVendor) {
+    const stated = String(statedVendor).toLowerCase();
+    if (stated !== CUSTOM_VENDOR && !profileByVendor.has(stated)) {
+      throw new Error(`Unknown vendor "${statedVendor}". Known vendors: ${[...profileByVendor.keys()].join(', ')}, ${CUSTOM_VENDOR}.`);
+    }
+    const statedScore = scores.find((entry) => entry.vendor === stated) || null;
+    return {
+      vendor: stated,
+      profile: profileByVendor.get(stated) || null,
+      confidence: statedScore ? statedScore.score : 1,
+      source: 'user',
+      reason: 'user-stated',
+      evidence: `vendor stated by the user${statedScore ? `; fingerprint would score it ${statedScore.score}` : ''}`,
+      scores,
+      runnerUp,
+      nearMiss,
+      conflict: detectedVendor !== stated ? { stated, detected: detectedVendor } : null,
+    };
+  }
+
+  return {
+    vendor: detectedVendor,
+    profile: matched ? profileByVendor.get(winner.vendor) : null,
+    confidence: winner ? winner.score : 0,
+    source: matched ? 'fingerprint' : 'fallback',
+    reason,
+    evidence: winner ? evidenceLine(winner, runnerUp) : 'no vendor profiles available',
+    scores,
+    runnerUp,
+    nearMiss,
+    conflict: null,
+  };
+}
+
+// The two drift lists are scoped differently on purpose: `unmappedColumns` is
+// about mapping coverage (anchors + columnMap), `missingExpectedColumns` is
+// about format drift (anchors only — including every columnMap alias would
+// make the list enormous and useless).
+function diffProfile(headerColumns, profile) {
+  if (!profile) {
+    return { unmappedColumns: [...(headerColumns || [])], missingExpectedColumns: [] };
+  }
+  const aliases = aliasIndex(profile);
+  const anchors = profile.anchors || {};
+  const anchorNames = normalizedSet([...(anchors.required || []), ...(anchors.strong || [])]);
+  const header = normalizedSet(headerColumns);
+
+  const unmappedColumns = (headerColumns || []).filter((column) => {
+    const normalized = normalizeHeaderName(column);
+    return normalized && !anchorNames.has(normalized) && !aliases.has(normalized);
+  });
+  const missingExpectedColumns = [...(anchors.required || []), ...(anchors.strong || [])]
+    .filter((column) => !header.has(normalizeHeaderName(column)));
+
+  return { unmappedColumns, missingExpectedColumns };
+}
+
+// Advisory pre-fill for rp-mapper. Every column this does not cover still flows
+// through discovery → mapper → user unchanged.
+function prefillColumnMap(headerColumns, profile) {
+  if (!profile) {
+    return [];
+  }
+  const aliases = aliasIndex(profile);
+  const hints = [];
+  for (const column of headerColumns || []) {
+    const hit = aliases.get(normalizeHeaderName(column));
+    if (hit) {
+      hints.push({ column, wixTarget: hit.wixTarget, matchedAlias: hit.alias });
+    }
+  }
+  return hints;
+}
+
+module.exports = {
+  VENDORS_DIR,
+  BASE_REQUIRED,
+  STRONG_WEIGHT,
+  PARTIAL_FACTOR,
+  NEGATIVE_PENALTY,
+  MATCH_THRESHOLD,
+  MARGIN,
+  MIN_STRONG_HITS,
+  NEAR_MISS_FLOOR,
+  CUSTOM_VENDOR,
+  loadVendorProfiles,
+  validateProfile,
+  scoreProfile,
+  detectVendor,
+  diffProfile,
+  prefillColumnMap,
+};