@natlibfi/melinda-record-matching 2.2.1-alpha.1 → 2.2.1-alpha.2

package/dist/match-detection/index.spec.js

@@ -0,0 +1,99 @@
+"use strict";
+
+var _fixugen = _interopRequireDefault(require("@natlibfi/fixugen"));
+
+var _fixura = require("@natlibfi/fixura");
+
+var _chai = require("chai");
+
+var _marcRecord = require("@natlibfi/marc-record");
+
+var features = _interopRequireWildcard(require("./features"));
+
+var _ = _interopRequireDefault(require("."));
+
+var _util = require("util");
+
+var _debug = _interopRequireDefault(require("debug"));
+
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:match-detection:test');
+const debugData = debug.extend('data');
+describe('match-detection', () => {
+  (0, _fixugen.default)({
+    path: [__dirname, '..', '..', 'test-fixtures', 'match-detection', 'index'],
+    useMetadataFile: true,
+    recurse: false,
+    fixura: {
+      reader: _fixura.READERS.JSON
+    },
+    callback: ({
+      getFixture,
+      options,
+      expectedResults,
+      array,
+      enabled = true
+    }) => {
+      if (!enabled) {
+        debug(`*** DISABLED TEST! ***`);
+        return;
+      }
+
+      const detect = (0, _.default)(formatOptions());
+      const recordA = new _marcRecord.MarcRecord(getFixture('recordA.json'), {
+        subfieldValues: false
+      });
+      debugData((0, _util.inspect)(recordA));
+      debug(`Our recordB is an array of records: ${array}`);
+      const recordB = array ? getFixture('recordB.json').map(recordJson => new _marcRecord.MarcRecord(recordJson, {
+        subfieldValues: false
+      })) : new _marcRecord.MarcRecord(getFixture('recordB.json'), {
+        subfieldValues: false
+      });
+      debugData((0, _util.inspect)(recordB));
+      const results = detect(recordA, recordB);
+      debugData(`${JSON.stringify(results)}`);
+      (0, _chai.expect)(results).to.eql(expectedResults);
+
+      function formatOptions() {
+        const contextFeatures = features[options.strategy.type];
+        return { ...options,
+          strategy: options.strategy.features.map(v => contextFeatures[v]())
+        };
+      }
+    }
+  });
+});
+//# sourceMappingURL=index.spec.js.map

package/dist/match-detection/index.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.spec.js","names":["debug","createDebugLogger","debugData","extend","describe","generateTests","path","__dirname","useMetadataFile","recurse","fixura","reader","READERS","JSON","callback","getFixture","options","expectedResults","array","enabled","detect","createDetectionInterface","formatOptions","recordA","MarcRecord","subfieldValues","inspect","recordB","map","recordJson","results","stringify","expect","to","eql","contextFeatures","features","strategy","type","v"],"sources":["../../src/match-detection/index.spec.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport generateTests from '@natlibfi/fixugen';\nimport {READERS} from '@natlibfi/fixura';\nimport {expect} from 'chai';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport * as features from './features';\nimport createDetectionInterface from '.';\nimport {inspect} from 'util';\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:test');\nconst debugData = debug.extend('data');\n\ndescribe('match-detection', () => {\n  generateTests({\n    path: [__dirname, '..', '..', 'test-fixtures', 'match-detection', 'index'],\n    useMetadataFile: true,\n    recurse: false,\n    fixura: {\n      reader: READERS.JSON\n    },\n    callback: ({getFixture, options, expectedResults, array, enabled = true}) => {\n\n      if (!enabled) {\n        debug(`*** DISABLED TEST! ***`);\n        return;\n      }\n\n      const detect = createDetectionInterface(formatOptions());\n      const recordA = new MarcRecord(getFixture('recordA.json'), {subfieldValues: false});\n      debugData(inspect(recordA));\n\n      debug(`Our recordB is an array of records: ${array}`);\n      const recordB = array\n        ? getFixture('recordB.json').map(recordJson => new MarcRecord(recordJson, {subfieldValues: false}))\n        : new MarcRecord(getFixture('recordB.json'), {subfieldValues: false});\n      debugData(inspect(recordB));\n\n      const results = detect(recordA, recordB);\n      debugData(`${JSON.stringify(results)}`);\n\n      expect(results).to.eql(expectedResults);\n\n      function formatOptions() {\n        const contextFeatures = features[options.strategy.type];\n\n        return {\n          ...options,\n          strategy: options.strategy.features.map(v => contextFeatures[v]())\n        };\n      }\n    }\n  });\n});\n"],"mappings":";;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;;;;;AAnCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAWA,MAAMA,KAAK,GAAG,IAAAC,cAAA,EAAkB,wDAAlB,CAAd;AACA,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAN,CAAa,MAAb,CAAlB;AAEAC,QAAQ,CAAC,iBAAD,EAAoB,MAAM;EAChC,IAAAC,gBAAA,EAAc;IACZC,IAAI,EAAE,CAACC,SAAD,EAAY,IAAZ,EAAkB,IAAlB,EAAwB,eAAxB,EAAyC,iBAAzC,EAA4D,OAA5D,CADM;IAEZC,eAAe,EAAE,IAFL;IAGZC,OAAO,EAAE,KAHG;IAIZC,MAAM,EAAE;MACNC,MAAM,EAAEC,eAAA,CAAQC;IADV,CAJI;IAOZC,QAAQ,EAAE,CAAC;MAACC,UAAD;MAAaC,OAAb;MAAsBC,eAAtB;MAAuCC,KAAvC;MAA8CC,OAAO,GAAG;IAAxD,CAAD,KAAmE;MAE3E,IAAI,CAACA,OAAL,EAAc;QACZnB,KAAK,CAAE,wBAAF,CAAL;QACA;MACD;;MAED,MAAMoB,MAAM,GAAG,IAAAC,SAAA,EAAyBC,aAAa,EAAtC,CAAf;MACA,MAAMC,OAAO,GAAG,IAAIC,sBAAJ,CAAeT,UAAU,CAAC,cAAD,CAAzB,EAA2C;QAACU,cAAc,EAAE;MAAjB,CAA3C,CAAhB;MACAvB,SAAS,CAAC,IAAAwB,aAAA,EAAQH,OAAR,CAAD,CAAT;MAEAvB,KAAK,CAAE,uCAAsCkB,KAAM,EAA9C,CAAL;MACA,MAAMS,OAAO,GAAGT,KAAK,GACjBH,UAAU,CAAC,cAAD,CAAV,CAA2Ba,GAA3B,CAA+BC,UAAU,IAAI,IAAIL,sBAAJ,CAAeK,UAAf,EAA2B;QAACJ,cAAc,EAAE;MAAjB,CAA3B,CAA7C,CADiB,GAEjB,IAAID,sBAAJ,CAAeT,UAAU,CAAC,cAAD,CAAzB,EAA2C;QAACU,cAAc,EAAE;MAAjB,CAA3C,CAFJ;MAGAvB,SAAS,CAAC,IAAAwB,aAAA,EAAQC,OAAR,CAAD,CAAT;MAEA,MAAMG,OAAO,GAAGV,MAAM,CAACG,OAAD,EAAUI,OAAV,CAAtB;MACAzB,SAAS,CAAE,GAAEW,IAAI,CAACkB,SAAL,CAAeD,OAAf,CAAwB,EAA5B,CAAT;MAEA,IAAAE,YAAA,EAAOF,OAAP,EAAgBG,EAAhB,CAAmBC,GAAnB,CAAuBjB,eAAvB;;MAEA,SAASK,aAAT,GAAyB;QACvB,MAAMa,eAAe,GAAGC,QAAQ,CAACpB,OAAO,CAACqB,QAAR,CAAiBC,IAAlB,CAAhC;QAEA,OAAO,EACL,GAAGtB,OADE;UAELqB,QAAQ,EAAErB,OAAO,CAACqB,QAAR,CAAiBD,QAAjB,CAA0BR,GAA1B,CAA8BW,CAAC,IAAIJ,eAAe,CAACI,CAAD,CAAf,EAAnC;QAFL,CAAP;MAID;IACF;EArCW,CAAd;AAuCD,CAxCO,CAAR"}

package/dist/matching-utils.js

@@ -0,0 +1,104 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getMelindaIdsF035 = getMelindaIdsF035;
+exports.getSubfieldValues = getSubfieldValues;
+exports.testStringOrNumber = testStringOrNumber;
+exports.validateSidFieldSubfieldCounts = validateSidFieldSubfieldCounts;
+
+var _debug = _interopRequireDefault(require("debug"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:utils');
+const debugData = debug.extend('data'); // eslint-disable-next-line max-statements
+
+function getMelindaIdsF035(record) {
+  const debug = (0, _debug.default)('@natlibfi/melinda-record-matching:melinda-id');
+  const debugData = debug.extend('data');
+  const f035s = record.getFields('035');
+
+  if (f035s.length < 1) {
+    debug(`No f035s found.`);
+    return [];
+  }
+
+  const allF035MelindaIds = [].concat(...f035s.map(toMelindaIds));
+  const f035MelindaIds = [...new Set(allF035MelindaIds)];
+  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);
+  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);
+  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);
+  return f035MelindaIds;
+
+  function toMelindaIds({
+    subfields
+  }) {
+    const melindaIdRegExp = /^(?<prefix>\(FI-MELINDA\)|FCC)(?<id>\d{9})$/u;
+    return subfields.filter(sub => ['a', 'z'].includes(sub.code)).filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value))).map(({
+      value
+    }) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');
+  }
+}
+
+function validateSidFieldSubfieldCounts(field) {
+  // Valid SID-fields have just one $c and one $b
+  debugData(`Validating SID field ${JSON.stringify(field)}`);
+  const countC = countSubfields(field, 'c');
+  const countB = countSubfields(field, 'b');
+  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);
+  return countC === 1 && countB === 1;
+}
+
+function countSubfields(field, subfieldCode) {
+  // debug(`Counting subfields ${subfieldCode}`);
+  return field.subfields.filter(({
+    code
+  }) => code === subfieldCode).length;
+}
+
+function getSubfieldValues(field, subfieldCode) {
+  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);
+  return field.subfields.filter(({
+    code
+  }) => code === subfieldCode).map(({
+    value
+  }) => testStringOrNumber(value) ? String(value) : '').filter(value => value);
+}
+
+function testStringOrNumber(value) {
+  if (typeof value === 'string' || typeof value === 'number') {
+    return true;
+  }
+
+  return false;
+}
+//# sourceMappingURL=matching-utils.js.map

package/dist/matching-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matching-utils.js","names":["debug","createDebugLogger","debugData","extend","getMelindaIdsF035","record","f035s","getFields","length","allF035MelindaIds","concat","map","toMelindaIds","f035MelindaIds","Set","JSON","stringify","subfields","melindaIdRegExp","filter","sub","includes","code","testStringOrNumber","value","test","String","replace","validateSidFieldSubfieldCounts","field","countC","countSubfields","countB","subfieldCode","getSubfieldValues"],"sources":["../src/matching-utils.js"],"sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Melinda record matching modules for Javascript\n*\n* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-record-matching-js\n*\n* melinda-record-matching-js program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Lesser General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-record-matching-js is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Lesser General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport createDebugLogger from 'debug';\n\nconst debug = createDebugLogger('@natlibfi/melinda-record-matching:utils');\nconst debugData = debug.extend('data');\n\n// eslint-disable-next-line max-statements\nexport function getMelindaIdsF035(record) {\n\n  const debug = createDebugLogger('@natlibfi/melinda-record-matching:melinda-id');\n  const debugData = debug.extend('data');\n\n  const f035s = record.getFields('035');\n\n  if (f035s.length < 1) {\n    debug(`No f035s found.`);\n    return [];\n  }\n\n  const allF035MelindaIds = [].concat(...f035s.map(toMelindaIds));\n  const f035MelindaIds = [...new Set(allF035MelindaIds)];\n\n  debugData(`Fields (${f035s.length}): ${JSON.stringify(f035s)}`);\n  debugData(`Ids (${allF035MelindaIds.length}): ${JSON.stringify(allF035MelindaIds)}`);\n  debugData(`Unique ids (${f035MelindaIds.length}): ${JSON.stringify(f035MelindaIds)}`);\n\n  return f035MelindaIds;\n\n  function toMelindaIds({subfields}) {\n    const melindaIdRegExp = /^(?<prefix>\\(FI-MELINDA\\)|FCC)(?<id>\\d{9})$/u;\n\n    return subfields\n      .filter(sub => ['a', 'z'].includes(sub.code))\n      .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))\n      .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');\n\n  }\n}\n\nexport function validateSidFieldSubfieldCounts(field) {\n  // Valid SID-fields have just one $c and one $b\n  debugData(`Validating SID field ${JSON.stringify(field)}`);\n  const countC = countSubfields(field, 'c');\n  const countB = countSubfields(field, 'b');\n  debug(`Found ${countC} sf $cs and ${countB} sf $bs. IsValid: ${countC === 1 && countB === 1}`);\n\n  return countC === 1 && countB === 1;\n}\n\nfunction countSubfields(field, subfieldCode) {\n  // debug(`Counting subfields ${subfieldCode}`);\n  return field.subfields.filter(({code}) => code === subfieldCode).length;\n}\n\nexport function getSubfieldValues(field, subfieldCode) {\n  debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);\n  return field.subfields\n    .filter(({code}) => code === subfieldCode)\n    .map(({value}) => testStringOrNumber(value) ? String(value) : '')\n    .filter(value => value);\n}\n\nexport function testStringOrNumber(value) {\n  if (typeof value === 'string' || typeof value === 'number') {\n    return true;\n  }\n  return false;\n}\n"],"mappings":";;;;;;;;;;AA4BA;;;;AA5BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAIA,MAAMA,KAAK,GAAG,IAAAC,cAAA,EAAkB,yCAAlB,CAAd;AACA,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAN,CAAa,MAAb,CAAlB,C,CAEA;;AACO,SAASC,iBAAT,CAA2BC,MAA3B,EAAmC;EAExC,MAAML,KAAK,GAAG,IAAAC,cAAA,EAAkB,8CAAlB,CAAd;EACA,MAAMC,SAAS,GAAGF,KAAK,CAACG,MAAN,CAAa,MAAb,CAAlB;EAEA,MAAMG,KAAK,GAAGD,MAAM,CAACE,SAAP,CAAiB,KAAjB,CAAd;;EAEA,IAAID,KAAK,CAACE,MAAN,GAAe,CAAnB,EAAsB;IACpBR,KAAK,CAAE,iBAAF,CAAL;IACA,OAAO,EAAP;EACD;;EAED,MAAMS,iBAAiB,GAAG,GAAGC,MAAH,CAAU,GAAGJ,KAAK,CAACK,GAAN,CAAUC,YAAV,CAAb,CAA1B;EACA,MAAMC,cAAc,GAAG,CAAC,GAAG,IAAIC,GAAJ,CAAQL,iBAAR,CAAJ,CAAvB;EAEAP,SAAS,CAAE,WAAUI,KAAK,CAACE,MAAO,MAAKO,IAAI,CAACC,SAAL,CAAeV,KAAf,CAAsB,EAApD,CAAT;EACAJ,SAAS,CAAE,QAAOO,iBAAiB,CAACD,MAAO,MAAKO,IAAI,CAACC,SAAL,CAAeP,iBAAf,CAAkC,EAAzE,CAAT;EACAP,SAAS,CAAE,eAAcW,cAAc,CAACL,MAAO,MAAKO,IAAI,CAACC,SAAL,CAAeH,cAAf,CAA+B,EAA1E,CAAT;EAEA,OAAOA,cAAP;;EAEA,SAASD,YAAT,CAAsB;IAACK;EAAD,CAAtB,EAAmC;IACjC,MAAMC,eAAe,GAAG,8CAAxB;IAEA,OAAOD,SAAS,CACbE,MADI,CACGC,GAAG,IAAI,CAAC,GAAD,EAAM,GAAN,EAAWC,QAAX,CAAoBD,GAAG,CAACE,IAAxB,CADV,EAEJH,MAFI,CAEGC,GAAG,IAAIG,kBAAkB,CAACH,GAAG,CAACI,KAAL,CAAlB,IAAiCN,eAAe,CAACO,IAAhB,CAAqBC,MAAM,CAACN,GAAG,CAACI,KAAL,CAA3B,CAF3C,EAGJb,GAHI,CAGA,CAAC;MAACa;IAAD,CAAD,KAAaD,kBAAkB,CAACC,KAAD,CAAlB,GAA4BE,MAAM,CAACF,KAAD,CAAN,CAAcG,OAAd,CAAsBT,eAAtB,EAAuC,OAAvC,CAA5B,GAA8E,EAH3F,CAAP;EAKD;AACF;;AAEM,SAASU,8BAAT,CAAwCC,KAAxC,EAA+C;EACpD;EACA3B,SAAS,CAAE,wBAAuBa,IAAI,CAACC,SAAL,CAAea,KAAf,CAAsB,EAA/C,CAAT;EACA,MAAMC,MAAM,GAAGC,cAAc,CAACF,KAAD,EAAQ,GAAR,CAA7B;EACA,MAAMG,MAAM,GAAGD,cAAc,CAACF,KAAD,EAAQ,GAAR,CAA7B;EACA7B,KAAK,CAAE,SAAQ8B,MAAO,eAAcE,MAAO,qBAAoBF,MAAM,KAAK,CAAX,IAAgBE,MAAM,KAAK,CAAE,EAAvF,CAAL;EAEA,OAAOF,MAAM,KAAK,CAAX,IAAgBE,MAAM,KAAK,CAAlC;AACD;;AAED,SAASD,cAAT,CAAwBF,KAAxB,EAA+BI,YAA/B,EAA6C;EAC3C;EACA,OAAOJ,KAAK,CAACZ,SAAN,CAAgBE,MAAhB,CAAuB,CAAC;IAACG;EAAD,CAAD,KAAYA,IAAI,KAAKW,YAA5C,EAA0DzB,MAAjE;AACD;;AAEM,SAAS0B,iBAAT,CAA2BL,KAA3B,EAAkCI,YAAlC,EAAgD;EACrD/B,SAAS,CAAE,oBAAmB+B,YAAa,SAAQlB,IAAI,CAACC,SAAL,CAAea,KAAf,CAAsB,EAAhE,CAAT;EACA,OAAOA,KAAK,CAACZ,SAAN,CACJE,MADI,CACG,CAAC;IAACG;EAAD,CAAD,KAAYA,IAAI,KAAKW,YADxB,EAEJtB,GAFI,CAEA,CAAC;IAACa;EAAD,CAAD,KAAaD,kBAAkB,CAACC,KAAD,CAAlB,GAA4BE,MAAM,CAACF,KAAD,CAAlC,GAA4C,EAFzD,EAGJL,MAHI,CAGGK,KAAK,IAAIA,KAHZ,CAAP;AAID;;AAEM,SAASD,kBAAT,CAA4BC,KAA5B,EAAmC;EACxC,IAAI,OAAOA,KAAP,KAAiB,QAAjB,IAA6B,OAAOA,KAAP,KAAiB,QAAlD,EAA4D;IAC1D,OAAO,IAAP;EACD;;EACD,OAAO,KAAP;AACD"}

package/package.json

@@ -14,7 +14,7 @@
 		"url": "git@github.com:natlibfi/melinda-record-matching-js.git"
 	},
 	"license": "LGPL-3.0+",
-	"version": "2.2.1-alpha.1",
+	"version": "2.2.1-alpha.2",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=14"
@@ -23,6 +23,7 @@
 		"access": "public"
 	},
 	"scripts": {
+		"prepare": "npm run build",
 		"start": "node dist/index.js",
 		"cli": "node dist/cli.js",
 		"lint": "eslint src",

package/src/candidate-search/query-list/bib.js

@@ -5,7 +5,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -28,7 +28,7 @@
 */
 import createDebugLogger from 'debug';
 import {toQueries} from '../candidate-search-utils';
-import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues} from '../../matching-utils';
+import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues, testStringOrNumber} from '../../matching-utils';
 
 
 export function bibSourceIds(record) {
@@ -108,7 +108,7 @@ export function bibSourceIds(record) {
 
         function removeSourcePrefix(subfieldValue) {
           const sourcePrefixRegex = (/^(?<sourcePrefix>\([A-Za-z0-9-]+\))(?<id>.+)$/u);
-          const normalizedValue = subfieldValue ? subfieldValue.replace(sourcePrefixRegex, '$<id>') : '';
+          const normalizedValue = testStringOrNumber(subfieldValue) ? String(subfieldValue).replace(sourcePrefixRegex, '$<id>') : '';
           debugData(`Normalized ${subfieldValue} to ${normalizedValue}`);
           return normalizedValue;
         }
@@ -116,7 +116,7 @@ export function bibSourceIds(record) {
         function normalizeSidSubfieldValue(subfieldValue) {
           debugData(`Normalizing ${subfieldValue}`);
           const normalizeAwayRegex = (/[- ]/u);
-          return subfieldValue ? subfieldValue.replace(normalizeAwayRegex, '') : '';
+          return testStringOrNumber(subfieldValue) ? String(subfieldValue).replace(normalizeAwayRegex, '') : '';
         }
 
       }
@@ -151,7 +151,7 @@ export function bibMelindaIds(record) {
 // bibHostComponents should search all 773 $ws for possible host id, but what should it do in case of multiple host ids?
 export function bibHostComponents(record) {
   const id = getHostId();
-  return id ? [`melinda.partsofhost=${id}`] : [];
+  return testStringOrNumber(id) ? [`melinda.partsofhost=${id}`] : [];
 
   function getHostId() {
     const [field] = record.get(/^773$/u);
@@ -159,12 +159,12 @@ export function bibHostComponents(record) {
     if (field) {
       const {value} = field.subfields.find(({code}) => code === 'w') || {};
 
-      if (value && (/^\(FI-MELINDA\)/u).test(value)) {
-        return value.replace(/^\(FI-MELINDA\)/u, '');
+      if (testStringOrNumber(value) && (/^\(FI-MELINDA\)/u).test(String(value))) {
+        return String(value).replace(/^\(FI-MELINDA\)/u, '');
       }
 
-      if (value && (/^\(FIN01\)/u).test(value)) {
-        return value.replace(/^\(FIN01\)/u, '');
+      if (testStringOrNumber(value) && (/^\(FIN01\)/u).test(String(value))) {
+        return String(value).replace(/^\(FIN01\)/u, '');
       }
 
       return false;
@@ -185,8 +185,8 @@ export function bibTitle(record) {
   const title = getTitle();
   const booleanStartWords = ['and', 'or', 'nor', 'not'];
 
-  if (title) {
-    const formatted = title
+  if (testStringOrNumber(title)) {
+    const formatted = String(title)
       .replace(/[^\w\s\p{Alphabetic}]/gu, '')
       // Clean up concurrent spaces from fe. subfield changes
       .replace(/ +/gu, ' ')
@@ -208,7 +208,8 @@ export function bibTitle(record) {
     if (field) {
       return field.subfields
         .filter(({code}) => ['a', 'b'].includes(code))
-        .map(({value}) => value)
+        .map(({value}) => testStringOrNumber(value) ? String(value) : '')
+        .filter(value => value)
         // In Melinda's index subfield separators are indexed as ' '
         .join(' ');
     }
@@ -244,18 +245,18 @@ export function bibStandardIdentifiers(record) {
 
     if (tag === '022') {
       return subfields
-        .filter(sub => ['a', 'z', 'y'].includes(sub.code) && issnIsbnReqExp.test(sub.value) && sub.value !== undefined)
-        .map(({value}) => value);
+        .filter(sub => ['a', 'z', 'y'].includes(sub.code) && testStringOrNumber(sub.value) && issnIsbnReqExp.test(String(sub.value)))
+        .map(({value}) => String(value));
     }
 
     if (tag === '020') {
       return subfields
-        .filter(sub => ['a', 'z'].includes(sub.code) && issnIsbnReqExp.test(sub.value) && sub.value !== undefined)
-        .map(({value}) => value);
+        .filter(sub => ['a', 'z'].includes(sub.code) && testStringOrNumber(sub.value) && issnIsbnReqExp.test(String(sub.value)))
+        .map(({value}) => String(value));
     }
 
     return subfields
-      .filter(sub => ['a', 'z'].includes(sub.code) && otherIdReqExp.test(sub.value) && sub.value !== undefined)
-      .map(({value}) => value);
+      .filter(sub => ['a', 'z'].includes(sub.code) && testStringOrNumber(sub.value) && otherIdReqExp.test(String(sub.value)))
+      .map(({value}) => String(value));
   }
 }

package/src/candidate-search/query-list/index.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *

package/src/match-detection/features/bib/authors.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -27,6 +27,9 @@
 */
 
 import {LevenshteinDistance as leven} from 'natural';
+import {testStringOrNumber} from '../../../matching-utils';
+
+// We should extract also organisational authors (110/710)
 
 export default ({nameTreshold = 10} = {}) => ({
   name: 'Authors',
@@ -40,7 +43,7 @@ export default ({nameTreshold = 10} = {}) => ({
 
       function toObj({code, value}) {
         if (code === 'a') {
-          return {name: value ? value.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase() : ''};
+          return {name: testStringOrNumber(value) ? String(value).replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase() : ''};
         }
 
         return {id: value};

package/src/match-detection/features/bib/language.js

@@ -5,7 +5,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -86,7 +86,7 @@ export default () => ({
       return 0;
     }
 
-    debugData(`There area languages to compare`);
+    debugData(`There are languages to compare`);
 
     if (a[0] === b[0]) {
       return 0.1;

package/src/match-detection/features/bib/publication-time.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -26,11 +26,16 @@
 *
 */
 
+import {testStringOrNumber} from '../../../matching-utils';
+
+// We should also get copyright time and copyright/publication times from 26x
+// We could also enable matching of records which have consequent publication years
+
 export default () => ({
   name: 'Publication time',
   extract: record => {
-    const value = record.get(/^008$/u)?.[0]?.value || '';
-    return value ? [value.slice(7, 11)] : [];
+    const value = record.get(/^008$/u)?.[0]?.value || undefined;
+    return testStringOrNumber(value) ? [String(value).slice(7, 11)] : [];
   },
   compare: (a, b) => a[0] === b[0] ? 0.1 : -1.0
 });

package/src/match-detection/features/bib/record-type.js

@@ -26,6 +26,8 @@
 *
 */
 
+// we could handle the case of books/notes
+
 export default () => ({
   name: 'Record type',
   extract: r => r.leader[6] ? [r.leader[6]] : [],

package/src/match-detection/features/bib/standard-identifier-factory.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -27,6 +27,7 @@
 */
 
 import createDebugLogger from 'debug';
+import {testStringOrNumber} from '../../../matching-utils';
 
 const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:standard-identifiers');
 const debugData = debug.extend('data');
@@ -40,8 +41,8 @@ export default ({pattern, subfieldCodes}) => {
     if (field) {
       return field.subfields
         .filter(({code}) => subfieldCodes.includes(code))
-      //  .map(({code, value}) => ({code, value: value.replace(/-/ug, '')}));
-        .map(({code, value}) => ({code, value: value ? value.replace(/-/ug, '') : ''}));
+        .map(({code, value}) => ({code, value: testStringOrNumber(value) ? String(value).replace(/-/ug, '') : ''}))
+        .filter(value => value);
     }
 
     return [];

package/src/match-detection/features/bib/title.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -28,14 +28,15 @@
 
 import createDebugLogger from 'debug';
 import {LevenshteinDistance as leven} from 'natural';
+import {testStringOrNumber} from '../../../matching-utils';
 
 export default ({treshold = 10} = {}) => ({
   name: 'Title',
   extract: record => {
     const title = getTitle();
 
-    if (title) {
-      return [title.replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
+    if (testStringOrNumber(title)) {
+      return [String(title).replace(/[^\p{Letter}\p{Number}]/gu, '').toLowerCase()];
     }
 
     return [];
@@ -46,9 +47,10 @@ export default ({treshold = 10} = {}) => ({
       if (field) {
         return field.subfields
           .filter(({code}) => ['a', 'b'].includes(code))
-          .map(({value}) => value)
+          .map(({value}) => testStringOrNumber(value) ? String(value) : '')
           .join('');
       }
+      return false;
     }
   },
   compare: (a, b) => {

package/src/matching-utils.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -58,8 +58,8 @@ export function getMelindaIdsF035(record) {
 
     return subfields
       .filter(sub => ['a', 'z'].includes(sub.code))
-      .filter(sub => melindaIdRegExp.test(sub.value))
-      .map(({value}) => value ? value.replace(melindaIdRegExp, '$<id>') : '');
+      .filter(sub => testStringOrNumber(sub.value) && melindaIdRegExp.test(String(sub.value)))
+      .map(({value}) => testStringOrNumber(value) ? String(value).replace(melindaIdRegExp, '$<id>') : '');
 
   }
 }
@@ -81,6 +81,15 @@ function countSubfields(field, subfieldCode) {
 
 export function getSubfieldValues(field, subfieldCode) {
   debugData(`Get subfield(s) $${subfieldCode} from ${JSON.stringify(field)}`);
-  return field.subfields.filter(({code}) => code === subfieldCode).map(({value}) => value);
+  return field.subfields
+    .filter(({code}) => code === subfieldCode)
+    .map(({value}) => testStringOrNumber(value) ? String(value) : '')
+    .filter(value => value);
 }
 
+export function testStringOrNumber(value) {
+  if (typeof value === 'string' || typeof value === 'number') {
+    return true;
+  }
+  return false;
+}