@natlibfi/melinda-record-matching 5.0.3 → 5.0.5-alpha.2

package/dist/match-detection/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/match-detection/index.js"],
-  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport * as features from './features/index.js';\n\nexport {features};\n\nexport default ({strategy, threshold = 0.8999}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {\n  const minProbabilityQuantifier = 0.5;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n\n  debugData(`Strategy: ${JSON.stringify(strategy)}, Threshold: ${JSON.stringify(threshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n  debugData(`Records: A: ${recordA}\\nB: ${recordB}`);\n  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);\n  // We could add here labels for records if we didn't get external labels\n\n  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});\n\n  debug(`We got an array of records: ${Array.isArray(recordB)}`);\n  const recordsB = Array.isArray(recordB) ? recordB : [recordB];\n  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];\n\n  const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));\n\n  // if we got array of records, we return an array of result\n  // if we got a singular record, we return a singular result\n  return Array.isArray(recordB) ? detectionResults : detectionResults[0];\n\n  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {\n    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';\n    const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';\n\n\n    debug(`Actual detection for record ${index + 1} ${labelB}`);\n    const featuresB = extractFeatures({record, recordExternal, localSettings});\n\n    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);\n    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);\n\n    const featurePairs = generateFeaturePairs(featuresA, featuresB);\n    const similarityVector = generateSimilarityVector(featurePairs);\n\n    if (similarityVector.some(v => v >= minProbabilityQuantifier)) {\n      const probability = calculateProbability(similarityVector);\n      debug(`probability: ${probability} (Threshold: ${threshold})`);\n      return returnResult({match: probability >= threshold, probability});\n    }\n\n    debugData(`No feature yielded minimum probability amount of points (${minProbabilityQuantifier})`);\n    return returnResult({match: false, probability: 0.0});\n  }\n\n  function extractFeatures({record, recordExternal}) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);\n  }\n\n  function returnResult(result) {\n    if (returnStrategy) {\n      debug(`Returning detection strategy with the result`);\n      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), threshold};\n      debugData(`${JSON.stringify(resultWithStrategy)}`);\n      return resultWithStrategy;\n    }\n    return result;\n  }\n\n  function formatStrategy(strategy) {\n    const strategyNames = strategy.map(element => element.name);\n    return strategyNames || [];\n  }\n\n  function calculateProbability(similarityVector) {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function generateSimilarityVector(featurePairs) {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs(featuresA, featuresB) {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n\n};\n"],
-  "mappings": "AACA,OAAO,uBAAuB;AAC9B,YAAY,cAAc;AAE1B,SAAQ;AAER,eAAe,CAAC,EAAC,UAAU,YAAY,OAAM,GAAG,iBAAiB,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAC,SAAS,SAAS,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,EAAC,MAAM;AACjK,QAAM,2BAA2B;AAEjC,QAAM,QAAQ,kBAAkB,mDAAmD;AACnF,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,gBAAgB,KAAK,UAAU,SAAS,CAAC,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAC7I,YAAU,eAAe,OAAO;AAAA,KAAQ,OAAO,EAAE;AACjD,QAAM,iBAAiB,KAAK,UAAU,eAAe,CAAC,QAAQ,KAAK,UAAU,eAAe,CAAC,EAAE;AAG/F,QAAM,YAAY,gBAAgB,EAAC,QAAQ,SAAS,gBAAgB,iBAAiB,cAAa,CAAC;AAEnG,QAAM,+BAA+B,MAAM,QAAQ,OAAO,CAAC,EAAE;AAC7D,QAAM,WAAW,MAAM,QAAQ,OAAO,IAAI,UAAU,CAAC,OAAO;AAC5D,QAAM,mBAAmB,MAAM,QAAQ,OAAO,IAAI,kBAAkB,CAAC,eAAe;AAEpF,QAAM,mBAAmB,SAAS,IAAI,CAAC,QAAQ,UAAU,gBAAgB,EAAC,WAAW,iBAAiB,QAAQ,gBAAgB,iBAAiB,KAAK,GAAG,MAAK,CAAC,CAAC;AAI9J,SAAO,MAAM,QAAQ,OAAO,IAAI,mBAAmB,iBAAiB,CAAC;AAErE,WAAS,gBAAgB,EAAC,WAAAA,YAAW,QAAQ,gBAAgB,OAAO,eAAAC,eAAa,GAAG;AAClF,UAAM,SAAS,mBAAmB,gBAAgB,QAAQ,gBAAgB,QAAQ;AAClF,UAAM,SAAS,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAG/E,UAAM,+BAA+B,QAAQ,CAAC,IAAI,MAAM,EAAE;AAC1D,UAAM,YAAY,gBAAgB,EAAC,QAAQ,gBAAgB,eAAAA,eAAa,CAAC;AAEzE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAUD,UAAS,CAAC,EAAE;AACjE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,eAAe,qBAAqBA,YAAW,SAAS;AAC9D,UAAM,mBAAmB,yBAAyB,YAAY;AAE9D,QAAI,iBAAiB,KAAK,OAAK,KAAK,wBAAwB,GAAG;AAC7D,YAAM,cAAc,qBAAqB,gBAAgB;AACzD,YAAM,gBAAgB,WAAW,gBAAgB,SAAS,GAAG;AAC7D,aAAO,aAAa,EAAC,OAAO,eAAe,WAAW,YAAW,CAAC;AAAA,IACpE;AAEA,cAAU,4DAA4D,wBAAwB,GAAG;AACjG,WAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,EACtD;AAEA,WAAS,gBAAgB,EAAC,QAAQ,eAAc,GAAG;AACjD,WAAO,SAAS,OAAO,CAAC,KAAK,EAAC,MAAM,QAAO,MAAM,IAAI,OAAO,EAAC,MAAM,OAAO,QAAQ,EAAC,QAAQ,eAAc,CAAC,EAAC,CAAC,GAAG,CAAC,CAAC;AAAA,EACnH;AAEA,WAAS,aAAa,QAAQ;AAC5B,QAAI,gBAAgB;AAClB,YAAM,8CAA8C;AACpD,YAAM,qBAAqB,EAAC,OAAO,OAAO,OAAO,aAAa,OAAO,aAAa,UAAU,eAAe,QAAQ,GAAG,UAAS;AAC/H,gBAAU,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AACjD,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAEA,WAAS,eAAeE,WAAU;AAChC,UAAM,gBAAgBA,UAAS,IAAI,aAAW,QAAQ,IAAI;AAC1D,WAAO,iBAAiB,CAAC;AAAA,EAC3B;AAEA,WAAS,qBAAqB,kBAAkB;AAC9C,UAAM,cAAc,iBAAiB,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAG;AACpE,WAAO,cAAc,IAAM,IAAM;AAAA,EACnC;AAEA,WAAS,yBAAyB,cAAc;AAC9C,UAAM,WAAW,aAAa,IAAI,CAAC,EAAC,MAAM,GAAG,EAAC,MAAM;AAClD,YAAM,EAAC,QAAO,IAAI,SAAS,KAAK,CAAC,EAAC,MAAM,YAAW,MAAM,SAAS,WAAW;AAC7E,YAAM,SAAS,QAAQ,GAAG,CAAC;AAC3B,aAAO,EAAC,MAAM,OAAM;AAAA,IACtB,CAAC;AAED,cAAU,WAAW,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC/C,WAAO,SAAS,IAAI,CAAC,EAAC,OAAM,MAAM,MAAM;AAAA,EAC1C;AAEA,WAAS,qBAAqBF,YAAW,WAAW;AAClD,UAAM,QAAQ,cAAc;AAC5B,UAAM,kBAAkB,YAAY;AAEpC,UAAM,2FAA2F,KAAK,UAAU,eAAe,CAAC,EAAE;AAClI,WAAO;AAEP,aAAS,gBAAgB;AACvB,aAAOA,WACJ,OAAO,CAAC,KAAK,EAAC,MAAM,MAAK,GAAG,UAAU,IAAI,OAAO;AAAA,QAChD;AAAA,QACA,GAAG;AAAA,QACH,GAAG,UAAU,KAAK,EAAE;AAAA,MACtB,CAAC,GAAG,CAAC,CAAC,EACL,OAAO,CAAC,EAAC,GAAG,EAAC,MAAM;AAClB,YAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,iBAAO;AAAA,QACT;AAEA,eAAO;AAAA,MACT,CAAC;AAAA,IACL;AAEA,aAAS,cAAc;AACrB,aAAOA,WACJ,IAAI,CAAC,EAAC,KAAI,MAAM,IAAI,EACpB,OAAO,OAAK,MAAM,KAAK,CAAC,EAAC,KAAI,MAAM,SAAS,CAAC,MAAM,KAAK;AAAA,IAC7D;AAAA,EACF;AAEF;",
+  "sourcesContent": ["\nimport createDebugLogger from 'debug';\nimport * as features from './features/index.js';\n\nexport {features};\n\nexport default ({strategy, threshold = 0.8999}, returnStrategy = false, localSettings = {}) => ({recordA, recordB, recordAExternal = {}, recordBExternal = {}}) => {\n  const MIN_PROPABILITY_QUALIFIER = 0.4; // Title perfect match: 0.5\n  const FORCE_FAIL = -1.0;\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection');\n  const debugData = debug.extend('data');\n\n  debugData(`Strategy: ${JSON.stringify(strategy)}, Threshold: ${JSON.stringify(threshold)}, ReturnStrategy: ${JSON.stringify(returnStrategy)}`);\n  debugData(`Records: A: ${recordA}\\nB: ${recordB}`);\n  debug(`Externals: A: ${JSON.stringify(recordAExternal)}, B: ${JSON.stringify(recordBExternal)}`);\n  // We could add here labels for records if we didn't get external labels\n\n  const featuresA = extractFeatures({record: recordA, recordExternal: recordAExternal, localSettings});\n\n  debug(`We got an array of records: ${Array.isArray(recordB)}`);\n  const recordsB = Array.isArray(recordB) ? recordB : [recordB];\n  const recordsBExternal = Array.isArray(recordB) ? recordBExternal : [recordBExternal];\n\n  const detectionResults = recordsB.map((record, index) => actualDetection({featuresA, recordAExternal, record, recordExternal: recordsBExternal[index], index}));\n\n  // if we got array of records, we return an array of result\n  // if we got a singular record, we return a singular result\n  return Array.isArray(recordB) ? detectionResults : detectionResults[0];\n\n  function actualDetection({featuresA, record, recordExternal, index, localSettings}) {\n    const labelA = recordAExternal && recordAExternal.label ? recordAExternal.label : 'a';\n    const labelB = recordExternal && recordExternal.label ? recordExternal.label : 'b';\n\n\n    debug(`Actual detection for record ${index + 1} ${labelB}`);\n    const featuresB = extractFeatures({record, recordExternal, localSettings});\n\n    debugData(`Features (a: ${labelA}): ${JSON.stringify(featuresA)}`);\n    debugData(`Features (b: ${labelB}): ${JSON.stringify(featuresB)}`);\n\n    const featurePairs = generateFeaturePairs(featuresA, featuresB);\n    const similarityVector = generateSimilarityVector(featurePairs);\n\n    if (similarityVector.some(v => v >= MIN_PROPABILITY_QUALIFIER)) {\n      const probability = calculateProbability(similarityVector);\n      debug(`probability: ${probability} (Threshold: ${threshold})`);\n      if (similarityVector.some(v => v <= FORCE_FAIL)) {\n        debug(`Some feature resulted in utter failure (${FORCE_FAIL} or less)`);\n        return returnResult({match: false, probability: 0.0});\n      }\n\n      return returnResult({match: probability >= threshold, probability});\n    }\n\n    debugData(`No feature yielded minimum probability amount of points (${MIN_PROPABILITY_QUALIFIER})`);\n    return returnResult({match: false, probability: 0.0});\n  }\n\n  function extractFeatures({record, recordExternal}) {\n    return strategy.reduce((acc, {name, extract}) => acc.concat({name, value: extract({record, recordExternal})}), []);\n  }\n\n  function returnResult(result) {\n    if (returnStrategy) {\n      debug(`Returning detection strategy with the result`);\n      const resultWithStrategy = {match: result.match, probability: result.probability, strategy: formatStrategy(strategy), threshold};\n      debugData(`${JSON.stringify(resultWithStrategy)}`);\n      return resultWithStrategy;\n    }\n    return result;\n  }\n\n  function formatStrategy(strategy) {\n    const strategyNames = strategy.map(element => element.name);\n    return strategyNames || [];\n  }\n\n  function calculateProbability(similarityVector) {\n    const probability = similarityVector.reduce((acc, v) => acc + v, 0.0);\n    return probability > 1.0 ? 1.0 : probability;\n  }\n\n  function generateSimilarityVector(featurePairs) {\n    const compared = featurePairs.map(({name, a, b}) => {\n      const {compare} = strategy.find(({name: featureName}) => name === featureName);\n      const points = compare(a, b);\n      return {name, points};\n    });\n\n    debugData(`Points: ${JSON.stringify(compared)}`);\n    return compared.map(({points}) => points);\n  }\n\n  function generateFeaturePairs(featuresA, featuresB) {\n    const pairs = generatePairs();\n    const missingFeatures = findMissing();\n\n    debug(`Not comparing the following features because one, or both records are missing features: ${JSON.stringify(missingFeatures)}`);\n    return pairs;\n\n    function generatePairs() {\n      return featuresA\n        .reduce((acc, {name, value}, index) => acc.concat({\n          name,\n          a: value,\n          b: featuresB[index].value\n        }), [])\n        .filter(({a, b}) => {\n          if (a.length === 0 || b.length === 0) {\n            return false;\n          }\n\n          return true;\n        });\n    }\n\n    function findMissing() {\n      return featuresA\n        .map(({name}) => name)\n        .filter(v => pairs.some(({name}) => name === v) === false);\n    }\n  }\n\n};\n"],
+  "mappings": "AACA,OAAO,uBAAuB;AAC9B,YAAY,cAAc;AAE1B,SAAQ;AAER,eAAe,CAAC,EAAC,UAAU,YAAY,OAAM,GAAG,iBAAiB,OAAO,gBAAgB,CAAC,MAAM,CAAC,EAAC,SAAS,SAAS,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,EAAC,MAAM;AACjK,QAAM,4BAA4B;AAClC,QAAM,aAAa;AAEnB,QAAM,QAAQ,kBAAkB,mDAAmD;AACnF,QAAM,YAAY,MAAM,OAAO,MAAM;AAErC,YAAU,aAAa,KAAK,UAAU,QAAQ,CAAC,gBAAgB,KAAK,UAAU,SAAS,CAAC,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAC7I,YAAU,eAAe,OAAO;AAAA,KAAQ,OAAO,EAAE;AACjD,QAAM,iBAAiB,KAAK,UAAU,eAAe,CAAC,QAAQ,KAAK,UAAU,eAAe,CAAC,EAAE;AAG/F,QAAM,YAAY,gBAAgB,EAAC,QAAQ,SAAS,gBAAgB,iBAAiB,cAAa,CAAC;AAEnG,QAAM,+BAA+B,MAAM,QAAQ,OAAO,CAAC,EAAE;AAC7D,QAAM,WAAW,MAAM,QAAQ,OAAO,IAAI,UAAU,CAAC,OAAO;AAC5D,QAAM,mBAAmB,MAAM,QAAQ,OAAO,IAAI,kBAAkB,CAAC,eAAe;AAEpF,QAAM,mBAAmB,SAAS,IAAI,CAAC,QAAQ,UAAU,gBAAgB,EAAC,WAAW,iBAAiB,QAAQ,gBAAgB,iBAAiB,KAAK,GAAG,MAAK,CAAC,CAAC;AAI9J,SAAO,MAAM,QAAQ,OAAO,IAAI,mBAAmB,iBAAiB,CAAC;AAErE,WAAS,gBAAgB,EAAC,WAAAA,YAAW,QAAQ,gBAAgB,OAAO,eAAAC,eAAa,GAAG;AAClF,UAAM,SAAS,mBAAmB,gBAAgB,QAAQ,gBAAgB,QAAQ;AAClF,UAAM,SAAS,kBAAkB,eAAe,QAAQ,eAAe,QAAQ;AAG/E,UAAM,+BAA+B,QAAQ,CAAC,IAAI,MAAM,EAAE;AAC1D,UAAM,YAAY,gBAAgB,EAAC,QAAQ,gBAAgB,eAAAA,eAAa,CAAC;AAEzE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAUD,UAAS,CAAC,EAAE;AACjE,cAAU,gBAAgB,MAAM,MAAM,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,eAAe,qBAAqBA,YAAW,SAAS;AAC9D,UAAM,mBAAmB,yBAAyB,YAAY;AAE9D,QAAI,iBAAiB,KAAK,OAAK,KAAK,yBAAyB,GAAG;AAC9D,YAAM,cAAc,qBAAqB,gBAAgB;AACzD,YAAM,gBAAgB,WAAW,gBAAgB,SAAS,GAAG;AAC7D,UAAI,iBAAiB,KAAK,OAAK,KAAK,UAAU,GAAG;AAC/C,cAAM,2CAA2C,UAAU,WAAW;AACtE,eAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,MACtD;AAEA,aAAO,aAAa,EAAC,OAAO,eAAe,WAAW,YAAW,CAAC;AAAA,IACpE;AAEA,cAAU,4DAA4D,yBAAyB,GAAG;AAClG,WAAO,aAAa,EAAC,OAAO,OAAO,aAAa,EAAG,CAAC;AAAA,EACtD;AAEA,WAAS,gBAAgB,EAAC,QAAQ,eAAc,GAAG;AACjD,WAAO,SAAS,OAAO,CAAC,KAAK,EAAC,MAAM,QAAO,MAAM,IAAI,OAAO,EAAC,MAAM,OAAO,QAAQ,EAAC,QAAQ,eAAc,CAAC,EAAC,CAAC,GAAG,CAAC,CAAC;AAAA,EACnH;AAEA,WAAS,aAAa,QAAQ;AAC5B,QAAI,gBAAgB;AAClB,YAAM,8CAA8C;AACpD,YAAM,qBAAqB,EAAC,OAAO,OAAO,OAAO,aAAa,OAAO,aAAa,UAAU,eAAe,QAAQ,GAAG,UAAS;AAC/H,gBAAU,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AACjD,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AAEA,WAAS,eAAeE,WAAU;AAChC,UAAM,gBAAgBA,UAAS,IAAI,aAAW,QAAQ,IAAI;AAC1D,WAAO,iBAAiB,CAAC;AAAA,EAC3B;AAEA,WAAS,qBAAqB,kBAAkB;AAC9C,UAAM,cAAc,iBAAiB,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAG;AACpE,WAAO,cAAc,IAAM,IAAM;AAAA,EACnC;AAEA,WAAS,yBAAyB,cAAc;AAC9C,UAAM,WAAW,aAAa,IAAI,CAAC,EAAC,MAAM,GAAG,EAAC,MAAM;AAClD,YAAM,EAAC,QAAO,IAAI,SAAS,KAAK,CAAC,EAAC,MAAM,YAAW,MAAM,SAAS,WAAW;AAC7E,YAAM,SAAS,QAAQ,GAAG,CAAC;AAC3B,aAAO,EAAC,MAAM,OAAM;AAAA,IACtB,CAAC;AAED,cAAU,WAAW,KAAK,UAAU,QAAQ,CAAC,EAAE;AAC/C,WAAO,SAAS,IAAI,CAAC,EAAC,OAAM,MAAM,MAAM;AAAA,EAC1C;AAEA,WAAS,qBAAqBF,YAAW,WAAW;AAClD,UAAM,QAAQ,cAAc;AAC5B,UAAM,kBAAkB,YAAY;AAEpC,UAAM,2FAA2F,KAAK,UAAU,eAAe,CAAC,EAAE;AAClI,WAAO;AAEP,aAAS,gBAAgB;AACvB,aAAOA,WACJ,OAAO,CAAC,KAAK,EAAC,MAAM,MAAK,GAAG,UAAU,IAAI,OAAO;AAAA,QAChD;AAAA,QACA,GAAG;AAAA,QACH,GAAG,UAAU,KAAK,EAAE;AAAA,MACtB,CAAC,GAAG,CAAC,CAAC,EACL,OAAO,CAAC,EAAC,GAAG,EAAC,MAAM;AAClB,YAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,iBAAO;AAAA,QACT;AAEA,eAAO;AAAA,MACT,CAAC;AAAA,IACL;AAEA,aAAS,cAAc;AACrB,aAAOA,WACJ,IAAI,CAAC,EAAC,KAAI,MAAM,IAAI,EACpB,OAAO,OAAK,MAAM,KAAK,CAAC,EAAC,KAAI,MAAM,SAAS,CAAC,MAAM,KAAK;AAAA,IAC7D;AAAA,EACF;AAEF;",
   "names": ["featuresA", "localSettings", "strategy"]
 }

package/dist/matching-utils.js

@@ -76,4 +76,18 @@ export function getMatchCounts(aValues, bValues) {
     return aToB < bToA ? aToB : bToA;
   }
 }
+export function stringAfter(string, substring) {
+  const pos = string.indexOf(substring);
+  if (pos === -1) {
+    return string;
+  }
+  return string.substring(pos + substring.length);
+}
+export function stringBefore(string, substring) {
+  const pos = string.indexOf(substring);
+  if (pos === -1) {
+    return string;
+  }
+  return string.substring(0, pos);
+}
 //# sourceMappingURL=matching-utils.js.map

package/dist/matching-utils.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/matching-utils.js"],
-  "sourcesContent": ["import createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n  const debugData = debug.extend('data');\n\n  const f035s = record.getFields('035');\n\n  if (f035s.length < 1) {\n    debug(`No f035s found.`);\n    return [];\n  }\n\n  const allF035MelindaIds = [].concat(...f035s.map(field => toMelindaIds(field)));\n  const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n  return f035MelindaIds;\n}\n\nexport function toMelindaIds({subfields}, subfieldsToParse = ['a', 'z']) {\n  const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n  return subfields\n    .filter(sub => subfieldsToParse.includes(sub.code))\n    .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n    .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n  // Valid SID-fields have just one $c and one $b\n  debugData(`Validating SID field ${JSON.stringify(field)}`);\n  const countC = countSubfields(field, 'c');\n  const countB = countSubfields(field, 'b');\n  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n  return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n  // debug(`Counting subfields ${subfieldCode}`);\n  return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n  return field.subfields\n    .filter(({code}) => code === subfieldCode)\n    .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n    .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n  if (typeof value === 'string' || typeof value === 'number') {\n    return true;\n  }\n  return false;\n}\n\nexport function extractSubfieldsFromField(field, subfieldCodes) {\n  if (field === undefined || field.subfields === undefined) {\n    return [];\n  }\n  const resultSubfields = field.subfields\n    .filter(({code}) => subfieldCodes.includes(code))\n    .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))\n    .filter(value => value);\n  return resultSubfields;\n}\n\nexport function uniqueSubfields(subfields) {\n  return subfields.reduce((arr, e) => {\n    if (!arr.find(item => item.code === e.code && item.value === e.value)) {\n      const newArr = arr.concat(e);\n      return newArr;\n    }\n    return arr;\n  }, []);\n}\n\nexport function getMatchCounts(aValues, bValues) {\n\n  const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n  return {\n    maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n    // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n    possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n    matchingValues\n  };\n\n  function getMatchingValuesAmount(aValues, bValues) {\n    if (bValues.length > aValues.length) {\n      return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n    }\n    if (aValues.length > bValues.length) {\n      return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n    }\n\n    // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n    // there would be duplicate values\n    const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n    const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n    return aToB < bToA ? aToB : bToA;\n  }\n}\n"],
-  "mappings": "AAAA,OAAO,uBAAuB;AAE9B,MAAM,QAAQ,kBAAkB,yCAAyC;AACzE,MAAM,YAAY,MAAM,OAAO,MAAM;AAE9B,gBAAS,kBAAkB,QAAQ;AAExC,QAAMA,SAAQ,kBAAkB,8CAA8C;AAC9E,QAAMC,aAAYD,OAAM,OAAO,MAAM;AAErC,QAAM,QAAQ,OAAO,UAAU,KAAK;AAEpC,MAAI,MAAM,SAAS,GAAG;AACpB,IAAAA,OAAM,iBAAiB;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,CAAC,EAAE,OAAO,GAAG,MAAM,IAAI,WAAS,aAAa,KAAK,CAAC,CAAC;AAC9E,QAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,iBAAiB,CAAC;AAErD,EAAAC,WAAU,WAAW,MAAM,MAAM,MAAM,KAAK,UAAU,KAAK,CAAC,EAAE;AAC9D,EAAAA,WAAU,QAAQ,kBAAkB,MAAM,MAAM,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACnF,EAAAA,WAAU,eAAe,eAAe,MAAM,MAAM,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpF,SAAO;AACT;AAEO,gBAAS,aAAa,EAAC,UAAS,GAAG,mBAAmB,CAAC,KAAK,GAAG,GAAG;AACvE,QAAM,kBAAkB;AAExB,SAAO,UACJ,OAAO,SAAO,iBAAiB,SAAS,IAAI,IAAI,CAAC,EACjD,OAAO,SAAO,mBAAmB,IAAI,KAAK,KAAK,gBAAgB,KAAK,OAAO,IAAI,KAAK,CAAC,CAAC,EACtF,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,EAAE,QAAQ,iBAAiB,OAAO,IAAI,EAAE;AACtG;AAEO,gBAAS,+BAA+B,OAAO;AAEpD,YAAU,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AACzD,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,MAAM,eAAe,MAAM,qBAAqB,WAAW,KAAK,WAAW,CAAC,EAAE;AAE7F,SAAO,WAAW,KAAK,WAAW;AACpC;AAEA,SAAS,eAAe,OAAO,cAAc;AAE3C,SAAO,MAAM,UAAU,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EAAE;AACnE;AAEO,gBAAS,kBAAkB,OAAO,cAAc;AACrD,YAAU,oBAAoB,YAAY,SAAS,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E,SAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EACxC,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,OAAO,WAAS,KAAK;AAC1B;AAEO,gBAAS,mBAAmB,OAAO;AACxC,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAAU;AAC1D,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEO,gBAAS,0BAA0B,OAAO,eAAe;AAC9D,MAAI,UAAU,UAAa,MAAM,cAAc,QAAW;AACxD,WAAO,CAAC;AAAA,EACV;AACA,QAAM,kBAAkB,MAAM,UAC3B,OAAO,CAAC,EAAC,KAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,EAAC,MAAM,MAAK,OAAO,EAAC,MAAM,OAAO,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,GAAE,EAAE,EACtF,OAAO,WAAS,KAAK;AACxB,SAAO;AACT;AAEO,gBAAS,gBAAgB,WAAW;AACzC,SAAO,UAAU,OAAO,CAAC,KAAK,MAAM;AAClC,QAAI,CAAC,IAAI,KAAK,UAAQ,KAAK,SAAS,EAAE,QAAQ,KAAK,UAAU,EAAE,KAAK,GAAG;AACrE,YAAM,SAAS,IAAI,OAAO,CAAC;AAC3B,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,GAAG,CAAC,CAAC;AACP;AAEO,gBAAS,eAAe,SAAS,SAAS;AAE/C,QAAM,iBAAiB,wBAAwB,SAAS,OAAO;AAE/D,SAAO;AAAA,IACL,WAAW,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA;AAAA,IAEtE,qBAAqB,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA,IAChF;AAAA,EACF;AAEA,WAAS,wBAAwBC,UAASC,UAAS;AACjD,QAAIA,SAAQ,SAASD,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AACA,QAAID,SAAQ,SAASC,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AAIA,UAAM,OAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AACjF,UAAM,OAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAEjF,WAAO,OAAO,OAAO,OAAO;AAAA,EAC9B;AACF;",
+  "sourcesContent": ["import createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\nexport function getMelindaIdsF035(record) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n  const debugData = debug.extend('data');\n\n  const f035s = record.getFields('035');\n\n  if (f035s.length < 1) {\n    debug(`No f035s found.`);\n    return [];\n  }\n\n  const allF035MelindaIds = [].concat(...f035s.map(field => toMelindaIds(field)));\n  const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n  return f035MelindaIds;\n}\n\nexport function toMelindaIds({subfields}, subfieldsToParse = ['a', 'z']) {\n  const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n  return subfields\n    .filter(sub => subfieldsToParse.includes(sub.code))\n    .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n    .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n  // Valid SID-fields have just one $c and one $b\n  debugData(`Validating SID field ${JSON.stringify(field)}`);\n  const countC = countSubfields(field, 'c');\n  const countB = countSubfields(field, 'b');\n  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n  return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n  // debug(`Counting subfields ${subfieldCode}`);\n  return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n  return field.subfields\n    .filter(({code}) => code === subfieldCode)\n    .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n    .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n  if (typeof value === 'string' || typeof value === 'number') {\n    return true;\n  }\n  return false;\n}\n\nexport function extractSubfieldsFromField(field, subfieldCodes) {\n  if (field === undefined || field.subfields === undefined) {\n    return [];\n  }\n  const resultSubfields = field.subfields\n    .filter(({code}) => subfieldCodes.includes(code))\n    .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value) : ''}))\n    .filter(value => value);\n  return resultSubfields;\n}\n\nexport function uniqueSubfields(subfields) {\n  return subfields.reduce((arr, e) => {\n    if (!arr.find(item => item.code === e.code && item.value === e.value)) {\n      const newArr = arr.concat(e);\n      return newArr;\n    }\n    return arr;\n  }, []);\n}\n\nexport function getMatchCounts(aValues, bValues) {\n\n  const matchingValues = getMatchingValuesAmount(aValues, bValues);\n\n  return {\n    maxValues: aValues.length > bValues.length ? aValues.length : bValues.length,\n    // possibleMatchingValues: amount of identifiers in set of less identifiers (we cannot more values than these)\n    possibleMatchValues: aValues.length > bValues.length ? bValues.length : aValues.length,\n    matchingValues\n  };\n\n  function getMatchingValuesAmount(aValues, bValues) {\n    if (bValues.length > aValues.length) {\n      return aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n    }\n    if (aValues.length > bValues.length) {\n      return bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n    }\n\n    // If we have same amount of values, we'll check matches both ways, to avoid mixups in cases\n    // there would be duplicate values\n    const aToB = aValues.filter(aValue => bValues.some(bValue => aValue === bValue)).length;\n    const bToA = bValues.filter(bValue => aValues.some(aValue => bValue === aValue)).length;\n\n    return aToB < bToA ? aToB : bToA;\n  }\n}\n\n\nexport function stringAfter(string, substring) {\n  const pos = string.indexOf(substring);\n  if (pos === -1) {\n    return string;\n  }\n  return string.substring(pos+substring.length);\n}\n\nexport function stringBefore(string, substring) {\n  const pos = string.indexOf(substring);\n  if (pos === -1) {\n    return string;\n  }\n  return string.substring(0, pos);\n}\n"],
+  "mappings": "AAAA,OAAO,uBAAuB;AAE9B,MAAM,QAAQ,kBAAkB,yCAAyC;AACzE,MAAM,YAAY,MAAM,OAAO,MAAM;AAE9B,gBAAS,kBAAkB,QAAQ;AAExC,QAAMA,SAAQ,kBAAkB,8CAA8C;AAC9E,QAAMC,aAAYD,OAAM,OAAO,MAAM;AAErC,QAAM,QAAQ,OAAO,UAAU,KAAK;AAEpC,MAAI,MAAM,SAAS,GAAG;AACpB,IAAAA,OAAM,iBAAiB;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,CAAC,EAAE,OAAO,GAAG,MAAM,IAAI,WAAS,aAAa,KAAK,CAAC,CAAC;AAC9E,QAAM,iBAAiB,CAAC,GAAG,IAAI,IAAI,iBAAiB,CAAC;AAErD,EAAAC,WAAU,WAAW,MAAM,MAAM,MAAM,KAAK,UAAU,KAAK,CAAC,EAAE;AAC9D,EAAAA,WAAU,QAAQ,kBAAkB,MAAM,MAAM,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACnF,EAAAA,WAAU,eAAe,eAAe,MAAM,MAAM,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpF,SAAO;AACT;AAEO,gBAAS,aAAa,EAAC,UAAS,GAAG,mBAAmB,CAAC,KAAK,GAAG,GAAG;AACvE,QAAM,kBAAkB;AAExB,SAAO,UACJ,OAAO,SAAO,iBAAiB,SAAS,IAAI,IAAI,CAAC,EACjD,OAAO,SAAO,mBAAmB,IAAI,KAAK,KAAK,gBAAgB,KAAK,OAAO,IAAI,KAAK,CAAC,CAAC,EACtF,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,EAAE,QAAQ,iBAAiB,OAAO,IAAI,EAAE;AACtG;AAEO,gBAAS,+BAA+B,OAAO;AAEpD,YAAU,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AACzD,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,eAAe,OAAO,GAAG;AACxC,QAAM,SAAS,MAAM,eAAe,MAAM,qBAAqB,WAAW,KAAK,WAAW,CAAC,EAAE;AAE7F,SAAO,WAAW,KAAK,WAAW;AACpC;AAEA,SAAS,eAAe,OAAO,cAAc;AAE3C,SAAO,MAAM,UAAU,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EAAE;AACnE;AAEO,gBAAS,kBAAkB,OAAO,cAAc;AACrD,YAAU,oBAAoB,YAAY,SAAS,KAAK,UAAU,KAAK,CAAC,EAAE;AAC1E,SAAO,MAAM,UACV,OAAO,CAAC,EAAC,KAAI,MAAM,SAAS,YAAY,EACxC,IAAI,CAAC,EAAC,MAAK,MAAM,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,EAAE,EAC/D,OAAO,WAAS,KAAK;AAC1B;AAEO,gBAAS,mBAAmB,OAAO;AACxC,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAAU;AAC1D,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEO,gBAAS,0BAA0B,OAAO,eAAe;AAC9D,MAAI,UAAU,UAAa,MAAM,cAAc,QAAW;AACxD,WAAO,CAAC;AAAA,EACV;AACA,QAAM,kBAAkB,MAAM,UAC3B,OAAO,CAAC,EAAC,KAAI,MAAM,cAAc,SAAS,IAAI,CAAC,EAC/C,IAAI,CAAC,EAAC,MAAM,MAAK,OAAO,EAAC,MAAM,OAAO,mBAAmB,KAAK,IAAI,OAAO,KAAK,IAAI,GAAE,EAAE,EACtF,OAAO,WAAS,KAAK;AACxB,SAAO;AACT;AAEO,gBAAS,gBAAgB,WAAW;AACzC,SAAO,UAAU,OAAO,CAAC,KAAK,MAAM;AAClC,QAAI,CAAC,IAAI,KAAK,UAAQ,KAAK,SAAS,EAAE,QAAQ,KAAK,UAAU,EAAE,KAAK,GAAG;AACrE,YAAM,SAAS,IAAI,OAAO,CAAC;AAC3B,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT,GAAG,CAAC,CAAC;AACP;AAEO,gBAAS,eAAe,SAAS,SAAS;AAE/C,QAAM,iBAAiB,wBAAwB,SAAS,OAAO;AAE/D,SAAO;AAAA,IACL,WAAW,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA;AAAA,IAEtE,qBAAqB,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ;AAAA,IAChF;AAAA,EACF;AAEA,WAAS,wBAAwBC,UAASC,UAAS;AACjD,QAAIA,SAAQ,SAASD,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AACA,QAAID,SAAQ,SAASC,SAAQ,QAAQ;AACnC,aAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAAA,IAC7E;AAIA,UAAM,OAAOA,SAAQ,OAAO,YAAUC,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AACjF,UAAM,OAAOA,SAAQ,OAAO,YAAUD,SAAQ,KAAK,YAAU,WAAW,MAAM,CAAC,EAAE;AAEjF,WAAO,OAAO,OAAO,OAAO;AAAA,EAC9B;AACF;AAGO,gBAAS,YAAY,QAAQ,WAAW;AAC7C,QAAM,MAAM,OAAO,QAAQ,SAAS;AACpC,MAAI,QAAQ,IAAI;AACd,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,MAAI,UAAU,MAAM;AAC9C;AAEO,gBAAS,aAAa,QAAQ,WAAW;AAC9C,QAAM,MAAM,OAAO,QAAQ,SAAS;AACpC,MAAI,QAAQ,IAAI;AACd,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,GAAG,GAAG;AAChC;",
   "names": ["debug", "debugData", "aValues", "bValues"]
 }

package/package.json

@@ -13,7 +13,7 @@
 		"url": "https://github.com/NatLibFi/melinda-record-matching-js"
 	},
 	"license": "MIT",
-	"version": "5.0.3",
+	"version": "5.0.5-alpha.2",
 	"type": "module",
 	"main": "./dist/index.js",
 	"bin": "./dist/cli.js",
@@ -32,33 +32,32 @@
 		"lint:dev": "eslint --fix ./src",
 		"test:base": "node --test --test-force-exit --experimental-test-coverage --test-reporter=spec './src/**/*.test.js'",
 		"test": "npm run lint && npm run test:base",
+		"test:only": "npm run lint && npm run test:base:only",
+		"test:base:only": "cross-env DEBUG=@natlibfi/*matching* NODE_ENV=test node --test --test-only --test-force-exit --experimental-test-coverage --test-reporter=spec './src/**/*.test.js'",
 		"watch:test": "cross-env DEBUG=@natlibfi/* NODE_ENV=test node --watch --test --experimental-test-coverage --test-reporter=spec './src/**/*.test.js'",
 		"dev": "npm run watch:test",
 		"dev:debug": "cross-env LOG_LEVEL=debug DEBUG=@natlibfi/* NODE_ENV=test npm run watch:test"
 	},
 	"dependencies": {
 		"@natlibfi/marc-record": "^10.0.1",
-		"@natlibfi/marc-record-serializers": "^11.0.1",
-		"@natlibfi/marc-record-validators-melinda": "^12.0.7",
+		"@natlibfi/marc-record-serializers": "^11.0.2",
+		"@natlibfi/marc-record-validators-melinda": "^12.0.15",
 		"@natlibfi/melinda-commons": "^14.0.2",
 		"@natlibfi/sru-client": "^7.0.1",
 		"debug": "^4.4.3",
 		"isbn3": "^2.0.6",
 		"moment": "^2.30.1",
 		"natural": "^8.1.1",
-		"uuid": "^13.0.0",
+		"uuid": "^14.0.0",
 		"yargs": "^18.0.0"
 	},
 	"devDependencies": {
-		"@natlibfi/fixugen": "^3.0.0",
-		"@natlibfi/fixugen-http-client": "^4.0.2",
-		"@natlibfi/fixura": "^4.0.0",
+		"@natlibfi/fixugen": "^3.0.1",
+		"@natlibfi/fixugen-http-client": "^4.0.3",
+		"@natlibfi/fixura": "^4.0.1",
 		"cross-env": "^10.1.0",
-		"esbuild": "^0.27.3",
-		"eslint": "^10.0.2"
-	},
-	"overrides": {
-		"nanoid": "^3.3.8"
+		"esbuild": "^0.28.0",
+		"eslint": "^10.2.1"
 	},
 	"nvolkComment": "cld3-asm 4.0.0 uses emscripten-wasm-loader ^3.0.3 which uses problematic, non-secure nanoid version 2.X.X."
 }

package/src/candidate-search/query-list/bib.js

@@ -1,9 +1,14 @@
 import createDebugLogger from 'debug';
 import {toQueries} from '../candidate-search-utils.js';
 import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues, testStringOrNumber} from '../../matching-utils.js';
+import {extractHostMelindaIdsFromField, extractPublicationYearFrom773, getHostMelindaFields} from './component.js';
+import {fieldToString} from '@natlibfi/marc-record-validators-melinda';
+import {getTitle} from '../../match-detection/features/bib/title.js';
 
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query');
 
+const IS_OK_ALONE_THRESHOLD = 5;
+
 export function bibSourceIds(record) {
 
   /* Melinda's SRU-index melinda.sourceid includes source IDs from SID fields in Melinda records
@@ -126,31 +131,27 @@ export function bibHostComponents(record) {
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibHostComponents');
   const debugData = debug.extend('data');
   debug(`Creating queries for hostIds`);
+  const f773s = getHostMelindaFields(record);
+  if (f773s.length !== 1) { // This needs some thinking...
+    return [];
+  }
 
-  const id = getHostId();
-  debugData(`Found id: ${JSON.stringify(id)}`);
+  const [id] = extractHostMelindaIdsFromField(f773s[0]);
+  debug(`Found id: ${JSON.stringify(id)}`);
 
-  return testStringOrNumber(id) ? [`melinda.partsofhost=${id}`] : [];
+  // NB! record.isCR() does not work if LDR/07=a, so we get year from 773$g!
+  const year = extractPublicationYearFrom773(f773s[0]);
+  debug(`${fieldToString(f773s[0])} => ${year ? year : 'N/A'}`);
+  if (year) {
+    return [`dc.date=${year} AND melinda.partsofhost=${id}`];
+  }
 
-  function getHostId() {
-    const [field] = record.get(/^773$/u);
+  return [`melinda.partsofhost=${id}`];
 
-    if (field) {
-      const {value} = field.subfields.find(({code}) => code === 'w') || {};
+}
 
-      if (testStringOrNumber(value) && (/^\(FI-MELINDA\)/u).test(String(value))) {
-        return String(value).replace(/^\(FI-MELINDA\)/u, '');
-      }
 
-      if (testStringOrNumber(value) && (/^\(FIN01\)/u).test(String(value))) {
-        return String(value).replace(/^\(FIN01\)/u, '');
-      }
 
-      return false;
-    }
-    return false;
-  }
-}
 
 // SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to (probably) to *headings* index TIT
 // - Aleph cannot currently handle headings searches starting with a boolean - in these cases use word search
@@ -184,43 +185,78 @@ export function bibTitleAuthorYearAlternates(record) {
   return {queryList: Array.from(origQueryList).reverse(), queryListType: 'alternates'};
 }
 
+function getTitleForQuery(record) {
+  const title = getTitle(record, ['a']);
+  if (title) {
+    const nWords = title.split(' ');
+    // If lone $a is deemed too short, fetch $b as well:
+    // (NV: I've seen pairs with 245$a-only vs 245$a$b, so I'd like to use $a only if it is long enough)
+
+    if (nWords < 3 || title.length < 12) {
+      const [f245] = record.get('245');
+      // If punctuation is ' =' I think that f245$a is good enough despite shortness. Trying to balance between two bad situations...
+      // "Suo (= short title) siellä, vetelä (= missing $b name in another language) täällä..."
+      if (f245 && f245.subfields.find(sf => sf.code === 'a' && sf.value.match(/ =$/u)) && title.length >= IS_OK_ALONE_THRESHOLD) {
+        return title;
+      }
+      return getTitle(record, ['a', 'b']); // Try to get a longer title
+    }
+  }
+  return title;
+}
+
+function dcTitle(record, onlyTitleLength, alternates = false) {
+  const title = getTitleForQuery(record);
+  if (!testStringOrNumber(title)) {
+    return [];
+  }
+
+  const formatted = getFormattedTitle(title);
+
+  // use word search for titles starting with a boolean
+  const useWordSearch = checkUseWordSearch(formatted);
+  // Prevent too many matches / SRU crashing by having a minimum length
+  // Note that currently this fails matching if there are no matches from previous matchers
+  if (formatted.length >= onlyTitleLength && !alternates) {
+    return [`dc.title="${useWordSearch ? '' : '^'}${formatted}*"`, formatted, true];
+  }
+
+  const queryIsOkAlone = !useWordSearch && formatted.length >= IS_OK_ALONE_THRESHOLD;
+
+  // use word search without ending * also in combination searches to avoid SRU-server crashes [MRA-189]
+  return [`dc.title="${useWordSearch || !queryIsOkAlone ? '' : '^'}${formatted}${queryIsOkAlone ? '*' : ''}"`, formatted, queryIsOkAlone];
+
+  function getFormattedTitle(title) {
+    const formatted = String(title)
+      .replace(/[\-+ !"(){}\[\]<>;:.?/@*%=^_`~]/gu, ' ')
+      .replace(/[^\w\s\p{Alphabetic}]/gu, '') // Apparently matches to/works with non-aplhabetic scripts such as Chinese as well
+      .replace(/  +/gu, ' ') // Clean up concurrent spaces from eg. subfield changes
+      .trim()
+      .replace(/^(.{30}\S*) .*$/, "$1")
+      .trim();
+
+    return formatted;
+  }
+}
+
 export function bibTitleAuthorPublisher({record, onlyTitleLength, addYear = false, alternates = false, alternateQueries = []}) {
   debug(`bibTitleAuthorPublisher, onlyTitleLength: ${onlyTitleLength}, addYear: ${addYear}, alternates: ${alternates}`);
-  const title = getTitle();
-  if (testStringOrNumber(title)) {
-    const formatted = getFormatted(title);
+  const [query, formatted, queryIsOkAlone] = dcTitle(record, onlyTitleLength, alternates);
+  if (query === undefined) {
+    return [];
+  }
 
-    // use word search for titles starting with a boolean
-    const useWordSearch = checkUseWordSearch(formatted);
-    // Prevent too many matches / SRU crashing by having a minimum length
-    // Note that currently this fails matching if there are no matches from previous matchers
-    if (formatted.length >= onlyTitleLength && !alternates) {
-      return [`dc.title="${useWordSearch ? '' : '^'}${formatted}*"`];
-    }
-    const queryIsOkAlone = formatted.length >= 5;
-
-    // use word search without ending * also in combination searches to avoid SRU-server crashes [MRA-189]
-    const query = `dc.title="${useWordSearch || !queryIsOkAlone ? '' : '^'}${formatted}${queryIsOkAlone ? '*' : ''}"`;
-    debug(`query: ${query}`);
-    const newAlternateQueries = alternates ? [...alternateQueries, query] : alternateQueries;
-
-    return addAuthorsToSearch({query, queryIsOkAlone, addYear, alternates, alternateQueries: newAlternateQueries});
-
-    function getFormatted(title) {
-      const formatted = String(title)
-        .replace(/[\-+ !"(){}\[\]<>;:.?/@*%=^_`~]/gu, ' ')
-        .replace(/[^\w\s\p{Alphabetic}]/gu, '')
-        // Clean up concurrent spaces from fe. subfield changes
-        .replace(/  +/gu, ' ')
-        .trim()
-        .replace(/^(.{30}\S*) .*$/, "$1")
-        .trim();
+  debug(`query: ${query}`);
 
-      return formatted;
-    }
+  // Prevent too many matches / SRU crashing by having a minimum length
+  // Note that currently this fails matching if there are no matches from previous matchers
+  if (formatted.length >= onlyTitleLength && !alternates) {
+    return [query];
   }
 
-  return [];
+  const newAlternateQueries = alternates ? [...alternateQueries, query] : alternateQueries;
+
+  return addAuthorsToSearch({query, queryIsOkAlone, addYear, alternates, alternateQueries: newAlternateQueries});
 
   function addAuthorsToSearch({query, queryIsOkAlone = false, addYear = false, alternates = false, alternateQueries = []}) {
     debug('addAuthorsToSearch');
@@ -254,8 +290,8 @@ export function bibTitleAuthorPublisher({record, onlyTitleLength, addYear = fals
   function addYearToSearch({query, queryIsOkAlone = false, alternates = false, alternateQueries = []}) {
     const [yearQuery] = bibYear(record);
     if (yearQuery !== undefined) {
-      const newAlternateQueries = alternates ? [...alternateQueries, `${yearQuery} AND ${query}`] : alternateQueries;
-      return alternates ? newAlternateQueries : [`${yearQuery} AND ${query}`];
+      const yearAndOtherQueries = `${yearQuery} AND ${query}`;
+      return alternates ? [...alternateQueries, yearAndOtherQueries] : [yearAndOtherQueries];
     }
     if (queryIsOkAlone) {
       return alternates ? alternateQueries : [`${query}`];
@@ -263,26 +299,6 @@ export function bibTitleAuthorPublisher({record, onlyTitleLength, addYear = fals
     return [];
   }
 
-  function getTitle() {
-    const [field] = record.get(/^245$/u);
-
-    if (field) {
-      const titleString = field.subfields
-        //.filter(({code}) => ['a', 'b', 'n', 'p'].includes(code))
-        .filter(({code}) => ['a', 'b'].includes(code))
-        //.filter(({code}) => ['a'].includes(code))
-        .map(({value}) => testStringOrNumber(value) ? String(value) : '')
-        .filter(value => value)
-        // In Melinda's index subfield separators are indexed as ' '
-        .join(' ');
-
-      if (/^[1-9]$/u.test(field.ind2)) { // Skip non-filing characters
-        return titleString.slice(parseInt(field.ind2));
-      }
-      return titleString;
-    }
-    return false;
-  }
 }
 
 export function bibAuthors(record) {
@@ -349,7 +365,10 @@ export function bibPublishers(record) {
       .slice(0, 30)
       .trim();
 
-    debugData(`Publisher string: ${formatted}`);
+    debugData(`Publisher string: '${formatted}'`);
+    if (formatted.length === 0) {
+      return [];
+    }
     // use non-wildcard word search from dc.publisher
     return [`dc.publisher="${formatted}"`];
   }
@@ -358,7 +377,7 @@ export function bibPublishers(record) {
 
   function getPublisher(record) {
     //debugData(record);
-    const [field] = record.get(/^(?:260)|(?:264)$/u);
+    const [field] = record.get(/^26[04]$/u).filter(f => f.subfields.some(sf => sf.code === 'b')); // Filter removes copyright-only fields
     //debugData(field);
 
     if (field) {
@@ -367,7 +386,8 @@ export function bibPublishers(record) {
         .map(({value}) => testStringOrNumber(value) ? String(value) : '')
         .filter(value => value)
         // In Melinda's index subfield separators are indexed as ' '
-        .join(' ');
+        .join(' ')
+        .trim();
       return publisherString;
     }
     return false;
@@ -387,14 +407,17 @@ export function bibYear(record) {
 
   function getYear(record) {
     const [f008] = record.get(/^008$/u);
-    if (f008 === undefined) {
-      debug('f008 missing');
+    if (!f008 || !f008.value || f008.value.length < 11) {
+      debug('f008 missing/crappy');
       return false;
     }
 
     debugData(`f008: ${JSON.stringify(f008)}`);
     const {value} = f008;
-    return testStringOrNumber(value) ? String(value).slice(7, 11) : undefined;
+    if (value === '||||' || value === '    ') { // Meaningless values
+      return false;
+    }
+    return value.slice(7, 11);
   }
 }
 

package/src/candidate-search/query-list/component.js

@@ -1,40 +1,117 @@
 import createDebugLogger from 'debug';
 import {promisify} from 'util';
 import {toQueries} from '../candidate-search-utils.js';
-import {getSubfieldValues, testStringOrNumber, toMelindaIds} from '../../matching-utils.js';
+import {getSubfieldValues, stringAfter, stringBefore, testStringOrNumber, toMelindaIds} from '../../matching-utils.js';
+import {fieldToString, uniqArray} from '@natlibfi/marc-record-validators-melinda/dist/utils.js';
 
 const setTimeoutPromise = promisify(setTimeout); // eslint-disable-line
 
-export function hostIdMelinda(record) {
-  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:hostIdMelinda');
-  const debugData = debug.extend('data'); // eslint-disable-line
-  debug(`Creating query for the Melinda Id host`);
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:component');
+const debugData = debug.extend('data'); // eslint-disable-line
+
 
-  const ids = hostId(record);
-  if (ids.length > 0) {
-    return toQueries(ids, 'melinda.partsofhost');
+
+export function parse773g(field) {
+  const g = field?.subfields?.find(sf => sf.code === 'g'); // NB! returns the first $g
+
+  if (!g) {
+    return {};
   }
 
-  debug(`No valid Melinda Id host found`);
-  return [];
+  const value = normalizeValue(g.value);
 
-  function hostId(record) {
-    // Multi 773 handling
-    const f773s = record.get(/^773$/u)
-      .filter(f773 => f773.subfields.some(sub => sub.code === 'w' && (/\(FI-MELINDA\).*/ui).test(sub.value)));
+  return {
+    // NB! We are not currently interested in volume (vuosikerta)
+    year: gToYear(value) || null,
+    number: gToNumber(value) || null,
+    pages: gToPages(value) || null // Might also be eg. Raita 5. However, returns only the number part
+  }
 
-    if (f773s.length === 0) {
-      return false;
+  function normalizeValue(value) {
+    return value.replace(/(?:\. -|\.)$/u, '');
+  }
+
+  function gToNumber(value) {
+    // Allow up to 3-digit numbers
+    const extract1 = value.replace(/^.*: ([1-9][0-9]?[0-9]), (?:p\.|page|pp\.|s\.|Seite|sida|sidor|sivu).*$/ui, '$1');
+    if (extract1 !== value) {
+      return extract1;
     }
+    return undefined;
+  }
 
-    // Multi $w handling
-    // $w (prefix)<id> handling
-    // $w <id> & $w (prefix)<id> Match
-    const melindaIds = f773s.map(f773 => toMelindaIds(f773, ['w'])).flat()
-      .filter(value => testStringOrNumber(value)) // drop invalid values
-      .filter((value, index, array) => array.indexOf(value) === index); // unique values;
-    return melindaIds;
+  function gToPages(value) {
+    if (value.match(/^[0-9]+(?:-[0-9]+)?(?:, [0-9]+(?:-[0-9]+)?)*$/u)) {
+      return value;
+    }
+    const numberPartOnly = value.replace(/^.*(?:p\.|raidat|raita|s\.|Seite|sivut?|pages?) /ui, '');
+    if (numberPartOnly !== value) {
+      return gToPages(numberPartOnly);
+    }
+    return undefined;
+  }
+
+
+  function gToYear(value) {
+    debug(`EXTRACT YEAR FROM '${value}`);
+    // extract year from within parentheses: $g vsk (yyyy) AND $g (YYYY)
+    if (value.match(/^[1-9][0-9]?[0-9]? ?\((?:20[012][0-9]|19[0-9][0-9])\)/u) || value.match(/^\((?:20[012][0-9]|19[0-9][0-9])\)/u) ) {
+      return stringBefore(stringAfter(value, '('), ')');
+    }
+    // If volume is missing, the year often seems to come without them parentheses:
+    if (value.match(/^(?:20[012][0-9]|19[0-9][0-9]) :/u)) {
+      return stringBefore(value, ' ');
+    }
+    // Seen in SB $g (vsk)year joulukuu
+    if (value.match(/^\([1-9][0-9]?\) ?(?:20[012][0-9]|19[0-9][0-9])[^0-9]/)) {
+      return value.replace(/^\([0-9]+\) ?/u, '').replace(/[^0-9].*$/u, '');
+    }
+    // Some magazines use DD.MM.YYYY
+    if (value.match(/^[^0-9]*(?:[1-9]|[12][0-9]|30|31)\.(?:1[012]|[1-9])\. ?(?:20[012][0-9]|19[0-9][0-9])[^0-9]/u)) {
+      return value.replace(/^[^0-9]*[0-9]+\.[0-9]+\. ?/u, '').replace(/[^0-9].*$/u, '');
+    }
+
+    return undefined;
+  }
+}
+
+export function extractPublicationYearFrom773(field) {
+  const data = parse773g(field);
+  debug(`Extracted year ${data.year} from f773$g ${fieldToString(field)}`);
+  return data.year;
+}
+
+function isMelindaSubfieldW(subfield) {
+  if (subfield.code !== 'w') {
+    return false;
+  }
+  return (/^\((?:FI-MELINDA|FIN01)\)0[0-9]{8}$/ui).test(subfield.value);
+}
+
+function removeMelindaPrefixFromValue(value) {
+  return value.replace(/^(?:\(FI-MELINDA\)|\(FIN01\))/, '');
+}
+
+export function getHostMelindaFields(record) {
+  return record.get(/^773$/u).filter(f => f.subfields.some(sf => isMelindaSubfieldW(sf)));
+}
+
+export function extractHostMelindaIdsFromField(field) {
+  return field.subfields.filter(sf => isMelindaSubfieldW(sf)).map(sf => removeMelindaPrefixFromValue(sf.value));
+}
+
+export function getHostMelindaIds(record) {
+  return uniqArray(getHostMelindaFields(record).map(f => extractHostMelindaIdsFromField(f)).flat());
+}
+
+export function hostIdMelinda(record) { // old function with replaced code
+  debug(`Creating query for the Melinda Id host`);
+  const ids = getHostMelindaIds(record);
+  if (ids.length === 0) {
+    debug(`No valid Melinda Id host found`);
+    return [];
   }
+  return toQueries(getHostMelindaIds(record), 'melinda.partsofhost');
 }
 
 export async function hostIdOtherSource(record, client) {

package/src/cli.js

@@ -76,7 +76,8 @@ async function cli() {
     if (['IDS'].includes(searchType)) {
       return [
         matchDetection.features.bib.melindaId(),
-        matchDetection.features.bib.allSourceIds()
+        matchDetection.features.bib.allSourceIds(),
+        matchDetection.features.bib.f773() // Host data
       ];
     }
 
@@ -101,7 +102,8 @@ async function cli() {
         //       normal publicationTime and a reprintPublication time
         matchDetection.features.bib.publicationTimeAllowConsYearsMulti(),
         matchDetection.features.bib.language(),
-        matchDetection.features.bib.bibliographicLevel()
+        matchDetection.features.bib.bibliographicLevel(),
+        matchDetection.features.bib.f773() // Host data
       ];
     }
 
@@ -113,7 +115,8 @@ async function cli() {
         matchDetection.features.bib.title(),
         matchDetection.features.bib.language(),
         matchDetection.features.bib.authors(),
-        matchDetection.features.bib.bibliographicLevel()
+        matchDetection.features.bib.bibliographicLevel(),
+        matchDetection.features.bib.f773() // Host data
       ];
     }
 
@@ -128,7 +131,8 @@ async function cli() {
         matchDetection.features.bib.recordType(),
         matchDetection.features.bib.publicationTime(),
         matchDetection.features.bib.language(),
-        matchDetection.features.bib.bibliographicLevel()
+        matchDetection.features.bib.bibliographicLevel(),
+        matchDetection.features.bib.f773() // Host data
       ];
     }