@natlibfi/melinda-record-matching 1.0.9 → 2.0.1-alpha.1

package/src/candidate-search/query-list/bib.js

@@ -27,6 +27,125 @@
 *
 */
 import createDebugLogger from 'debug';
+import {toQueries} from '../candidate-search-utils';
+import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues} from '../../matching-utils';
+
+
+export function bibSourceIds(record) {
+
+  /* Melinda's SRU-index melinda.sourceid includes source IDs from SID fields in Melinda records
+     SID-fields in Melinda have sf $c with local id and sf $b with a code for the local db:
+
+     SID__ $c 123457 $b helka
+     SID__ $c (ANDL100020)1077305 $b sata
+     SID__ $c VER999999 $ FI-KV
+     SID__ $c /10024/508126 $ REPO_THESEUS
+
+    In melinda.sourceid -index case is kept, sourceprefixes in brackets and hyphens are normalized away:
+    Note: slashes are not normalized away, but a SRU-search-string including slashes needs to be quoted
+
+    1234567helka
+    1077305sata
+    VER999999FIKV
+    /10024/508126REPO_THESEUS
+
+    Note: All Melinda records that have a matching records in a local db do NOT have SID for that local records,
+          existence of a SID field depends on how the record has been added to Melinda and how it has been handled
+          afterwards. SIDs are also not reliably maintained. A record might or might not have a SID for a local db
+          after the matching record is removed from the local db.
+
+   */
+
+
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:source-ids');
+  const debugData = debug.extend('data');
+  //const debugInfo = debug.extend('info');
+
+  debug(`Creating queries for sourceid's`);
+
+  const fSids = record.get('SID');
+  debugData(`SID-fields (${fSids.length}): ${JSON.stringify(fSids)}`);
+
+  return fSids.length > 0 ? toSidQueries(fSids) : [];
+
+  function toSidQueries(fSids) {
+    debug(`Creating actual queries for sourceid's`);
+
+    const sidStrings = getSidStrings(fSids);
+
+    if (sidStrings.length < 1) {
+      debug(`No identifiers found, no queries created.`);
+      return [];
+    }
+
+    const sidQueries = toQueries(sidStrings, 'melinda.sourceid');
+
+    return sidQueries;
+
+    function getSidStrings(fSids) {
+      debug(`Getting Sid strings from SID fields`);
+
+      // Map SID fields to valid sidStrings, filter out empty strings
+      const sidStrings = fSids.map(toSidString).filter(nonEmptySid => nonEmptySid);
+      return sidStrings;
+
+      function toSidString(field) {
+        debug(`Getting string from a field`);
+
+        return validateSidFieldSubfieldCounts(field) ? createSidString(field) : '';
+
+        function createSidString(field) {
+          debug(`Creating string from a field`);
+          const [sfC] = getSubfieldValues(field, 'c');
+          const [sfB] = getSubfieldValues(field, 'b');
+
+          const cleanedSfC = removeSourcePrefix(normalizeSidSubfieldValue(sfC));
+          const cleanedSfB = normalizeSidSubfieldValue(sfB);
+
+          debugData(`${JSON.stringify(sfC)} + ${JSON.stringify(sfB)}`);
+          return cleanedSfC.concat(cleanedSfB);
+        }
+
+        function removeSourcePrefix(subfieldValue) {
+          const sourcePrefixRegex = (/^(?<sourcePrefix>\([A-Za-z0-9-]+\))(?<id>.+)$/u);
+          const normalizedValue = subfieldValue.replace(sourcePrefixRegex, '$<id>');
+          debugData(`Normalized ${subfieldValue} to ${normalizedValue}`);
+          return normalizedValue;
+        }
+
+        function normalizeSidSubfieldValue(subfieldValue) {
+          debugData(`Normalizing ${subfieldValue}`);
+          const normalizeAwayRegex = (/[- ]/u);
+          return subfieldValue.replace(normalizeAwayRegex, '');
+        }
+
+      }
+    }
+  }
+}
+
+export function bibMelindaIds(record) {
+  // Melinda's SRU-index melinda.melindaid includes f001 controlnumbers and old Melinda-IDs from f035z's for all non-deleted Melinda-records
+
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibMelindaIds');
+  const debugData = debug.extend('data');
+  debug(`Creating queries for MelindaIds`);
+
+  // Note: Melinda-ID's for search queries are created just from records f035a's and f035z's
+  // Both (FI-MELINDA)- and FCC-prefixed forms are found
+  // f001 controlnumber is not currently included, even if record's f003 is FI-MELINDA
+  const melindaIds = getMelindaIdsF035(record);
+
+  debugData(`Unique identifiers (${melindaIds.length}): ${JSON.stringify(melindaIds)}`);
+
+  if (melindaIds.length < 1) {
+    debug(`No identifiers found, no queries created.`);
+    return [];
+  }
+
+  return toQueries(melindaIds, 'melinda.melindaid');
+}
+
 
 // bibHostComponents returns host id from the first subfield $w of first field f773, see test-fixtures 04 and 05
 // bibHostComponents should search all 773 $ws for possible host id, but what should it do in case of multiple host ids?
@@ -54,12 +173,20 @@ export function bibHostComponents(record) {
   }
 }
 
+// SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to
+// (probably) to *headings* index TIT
+// Headings index TIT drops articles etc. from the start of the title according to the filing indicator
+// Currently filing indicator is not implemented - if the title starts with an article and the Melinda
+// record is correctly catalogued using a filing indicator -> dc.title search won't match
+
 export function bibTitle(record) {
   const title = getTitle();
 
   if (title) {
     const formatted = title
       .replace(/[^\w\s\p{Alphabetic}]/gu, '')
+      // Clean up concurrent spaces from fe. subfield changes
+      .replace(/ +/gu, ' ')
       .trim()
       .slice(0, 30)
       .trim();
@@ -77,7 +204,8 @@ export function bibTitle(record) {
       return field.subfields
         .filter(({code}) => ['a', 'b'].includes(code))
         .map(({value}) => value)
-        .join('');
+        // In Melinda's index subfield separators are indexed as ' '
+        .join(' ');
     }
     return false;
   }
@@ -103,25 +231,7 @@ export function bibStandardIdentifiers(record) {
     return [];
   }
 
-  return toQueries(uniqueIdentifiers);
-
-  function toQueries(identifiers) {
-    // Aleph supports only two queries with or -operator (This is not actually true)
-    const pairs = toPairs(identifiers);
-    const queries = pairs.map(([a, b]) => b ? `dc.identifier=${a} or dc.identifier=${b}` : `dc.identifier=${a}`);
-
-    debugData(`Pairs (${pairs.length}): ${JSON.stringify(pairs)}`);
-    debugData(`Queries (${queries.length}): ${JSON.stringify(queries)}`);
-
-    return queries;
-
-    function toPairs(array) {
-      if (array.length === 0) {
-        return [];
-      }
-      return [array.slice(0, 2)].concat(toPairs(array.slice(2), 2));
-    }
-  }
+  return toQueries(uniqueIdentifiers, 'dc.identifier');
 
   function toIdentifiers({tag, subfields}) {
     const issnIsbnReqExp = (/^[A-Za-z0-9-]+$/u);

package/src/candidate-search/query-list/bib.spec.js

@@ -32,20 +32,22 @@ import {expect} from 'chai';
 import {MarcRecord} from '@natlibfi/marc-record';
 import * as generators from './bib';
 
-generateTests({
-  path: [__dirname, '..', '..', '..', 'test-fixtures', 'candidate-search', 'query-list', 'bib'],
-  useMetadataFile: true,
-  fixura: {
-    reader: READERS.JSON
-  },
-  callback: ({type, inputRecord, expectedQuery, enabled = true}) => {
-    const generate = generators[type];
-    const record = new MarcRecord(inputRecord, {subfieldValues: false});
+describe('candidate-search/query-list/bib/', () => {
+  generateTests({
+    path: [__dirname, '..', '..', '..', 'test-fixtures', 'candidate-search', 'query-list', 'bib'],
+    useMetadataFile: true,
+    fixura: {
+      reader: READERS.JSON
+    },
+    callback: ({type, inputRecord, expectedQuery, enabled = true}) => {
+      const generate = generators[type];
+      const record = new MarcRecord(inputRecord, {subfieldValues: false});
 
-    if (!enabled) {
-      return;
-    }
+      if (!enabled) {
+        return;
+      }
 
-    expect(generate(record)).to.eql(expectedQuery);
-  }
+      expect(generate(record)).to.eql(expectedQuery);
+    }
+  });
 });

package/src/candidate-search/query-list/index.js

@@ -32,7 +32,9 @@ export const searchTypes = {
   bib: {
     standardIdentifiers: 'bibStandardIdentifiers',
     hostComponents: 'bibHostComponents',
-    title: 'bibTitle'
+    title: 'bibTitle',
+    melindaId: 'bibMelindaIds',
+    sourceIds: 'bibSourceIds'
   }
 };
 

package/src/index.spec.js

@@ -32,49 +32,56 @@ import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
 import createMatchInterface, {matchDetection} from '.';
 
-generateTests({
-  callback,
-  path: [__dirname, '..', 'test-fixtures', 'index'],
-  recurse: false,
-  fixura: {
-    reader: READERS.JSON
-  }
-});
+describe('INDEX', () => {
+  generateTests({
+    callback,
+    path: [__dirname, '..', 'test-fixtures', 'index'],
+    recurse: false,
+    fixura: {
+      reader: READERS.JSON
+    }
+  });
 
-async function callback({getFixture, options}) {
-  const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});
-  const expectedMatches = getFixture('expectedMatches.json');
+  async function callback({getFixture, options, enabled = true}) {
 
-  const match = createMatchInterface(formatOptions());
-  const matches = await match(record);
+    if (!enabled) {
+      return;
+    }
 
-  expect(formatResults()).to.eql(expectedMatches);
+    const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});
+    const expectedMatches = getFixture('expectedMatches.json');
 
-  function formatOptions() {
-    const contextFeatures = matchDetection.features[options.detection.strategy.type];
+    const match = createMatchInterface(formatOptions());
+    const matches = await match(record);
 
-    return {
-      ...options,
-      search: {
-        ...options.search,
-        maxRecordsPerRequest: 1
-      },
-      detection: {
-        ...options.detect,
-        strategy: options.detection.strategy.features.map(v => contextFeatures[v]())
-      },
-      maxMatches: 2,
-      maxCandidates: 1
-    };
-  }
+    expect(formatResults()).to.eql(expectedMatches);
 
-  function formatResults() {
-    return matches.map(({candidate, probability}) => ({
-      probability,
-      candidate: {
-        id: candidate.id,
-        record: candidate.record.toObject()
-      }
-    }));
+    function formatOptions() {
+      const contextFeatures = matchDetection.features[options.detection.strategy.type];
+
+      return {
+        ...options,
+        search: {
+          ...options.search,
+          maxRecordsPerRequest: 1
+        },
+        detection: {
+          ...options.detect,
+          strategy: options.detection.strategy.features.map(v => contextFeatures[v]())
+        },
+        maxMatches: 2,
+        maxCandidates: 1
+      };
+    }
+
+    function formatResults() {
+      return matches.map(({candidate, probability}) => ({
+        probability,
+        candidate: {
+          id: candidate.id,
+          record: candidate.record.toObject()
+        }
+      }));
+    }
   }
-}
+});

package/src/match-detection/features/bib/all-source-ids.js

@@ -0,0 +1,122 @@
+/* eslint-disable max-statements */
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createDebugLogger from 'debug';
+import {validateSidFieldSubfieldCounts, getSubfieldValues} from '../../../matching-utils';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:all-source-ids');
+const debugData = debug.extend('data');
+
+/*
+
+SID-fields in Melinda have sf $c with local id and sf $b with a code for the local db:
+
+ SID__ $c 123457 $b helka
+ SID__ $c (ANDL100020)1077305 $b sata
+ SID__ $c VER999999 $ FI-KV
+
+allSourceIds matching feature is formatted as [{sourceDb: "helka", sourceId: "123457"}, {sourceDb: "sata", sourceId: "(ANDL100020)1077305"}, {sourceDb: "FI-KV", sourceId: "VER999999"}]
+- no normalization / prefix deletions etc. in the feature
+
+Note: All Melinda records that have a matching records in a local db do NOT have SID for that local records,
+      existence of a SID field depends on how the record has been added to Melinda and how it has been handled
+      afterwards. SIDs are also not reliably maintained. A record might or might not have a SID for a local db
+      after the matching record is removed from the local db.
+
+Records with mismatching local ids for matching local db are a strong mismatch.
+Records with matching local ids for matching local db are a very good match.
+
+*/
+
+export default () => ({
+  name: 'All source IDs',
+  extract: record => {
+    debug(`Creating match detection features for all local source id's`);
+
+    const fSids = record.get('SID');
+    debugData(`SID-fields (${fSids.length}): ${JSON.stringify(fSids)}`);
+
+    const sidFeatures = getSidFeatures(fSids);
+    debugData(`SID-features (${sidFeatures.length}): ${JSON.stringify(sidFeatures)}`);
+
+    return sidFeatures;
+
+    function getSidFeatures(fSids) {
+      debug(`Getting Sid strings from SID fields`);
+
+      // Map SID fields to sidFeatures, filter out empty strings
+      const sidFeatures = fSids.map(toSidFeature).filter(nonEmptySid => nonEmptySid);
+      return sidFeatures;
+
+      function toSidFeature(field) {
+        debug(`Getting feature from a field`);
+
+        return validateSidFieldSubfieldCounts(field) ? createSidFeature(field) : '';
+
+        function createSidFeature(field) {
+          debug(`Creating feature from a field`);
+          const [sfC] = getSubfieldValues(field, 'c');
+          const [sfB] = getSubfieldValues(field, 'b');
+
+          debugData(`${JSON.stringify(sfC)} + ${JSON.stringify(sfB)}`);
+          debugData(`sourceDb: ${sfB}, sourceId: ${sfC}`);
+          return {'sourceDb': sfB, 'sourceId': sfC};
+        }
+      }
+
+    }
+
+  },
+  compare: (a, b) => {
+
+    debugData(`Comparing ${JSON.stringify(a)} and ${JSON.stringify(b)}`);
+
+    const matches = a.filter(sidA => b.some(sidB => sidA.sourceDb === sidB.sourceDb &&
+            sidA.sourceId === sidB.sourceId));
+    debugData(`Matches (${matches.length}): ${JSON.stringify(matches)}`);
+
+    const mismatches = a.filter(sidA => b.some(sidB => sidA.sourceDb === sidB.sourceDb &&
+      sidA.sourceId !== sidB.sourceId));
+    debugData(`Mismatches (${mismatches.length}): ${JSON.stringify(mismatches)}`);
+
+    // If there's at least one mismatching source ID from matching source db
+    if (mismatches.length > 0) {
+      return -1;
+    }
+
+    // If there's at least one matching source ID from matching source db
+    if (matches.length > 0) {
+      return 1;
+    }
+
+    // If there are no source IDs with matching sources
+    return 0;
+
+  }
+});

package/src/match-detection/features/bib/index.js

@@ -36,3 +36,5 @@ export {default as recordType} from './record-type';
 export {default as publicationTime} from './publication-time';
 export {default as language} from './language';
 export {default as bibliographicLevel} from './bibliographic-level';
+export {default as melindaId} from './melinda-id';
+export {default as allSourceIds} from './all-source-ids';

package/src/match-detection/features/bib/index.spec.js

@@ -32,30 +32,39 @@ import {expect} from 'chai';
 import {MarcRecord} from '@natlibfi/marc-record';
 import * as features from '.';
 
-generateTests({
-  path: [__dirname, '..', '..', '..', '..', 'test-fixtures', 'match-detection', 'features', 'bib'],
-  useMetadataFile: true,
-  fixura: {
-    reader: READERS.JSON
-  },
-  callback: ({feature, options, type, ...expectations}) => {
-    if (type === 'extract') {
-      const {expectedFeatures, inputRecord} = expectations;
-      const record = new MarcRecord(inputRecord, {subfieldValues: false});
-      const {extract} = features[feature](options);
-
-      expect(extract(record)).to.eql(expectedFeatures);
-      return;
-    }
 
-    if (type === 'compare') {
-      const {featuresA, featuresB, expectedPoints} = expectations;
-      const {compare} = features[feature](options);
+describe('match-detection/features/bib/', () => {
+  generateTests({
+    path: [__dirname, '..', '..', '..', '..', 'test-fixtures', 'match-detection', 'features', 'bib'],
+    useMetadataFile: true,
+    fixura: {
+      reader: READERS.JSON
+    },
+    // eslint-disable-next-line max-statements
+    callback: ({enabled = true, feature, options, type, ...expectations}) => {
 
-      expect(compare(featuresA, featuresB)).to.equal(expectedPoints);
-      return;
-    }
+      if (!enabled) {
+        return;
+      }
+
+      if (type === 'extract') {
+        const {expectedFeatures, inputRecord} = expectations;
+        const record = new MarcRecord(inputRecord, {subfieldValues: false});
+        const {extract} = features[feature](options);
+
+        expect(extract(record)).to.eql(expectedFeatures);
+        return;
+      }
 
-    throw new Error(`Invalid type ${type}`);
-  }
+      if (type === 'compare') {
+        const {featuresA, featuresB, expectedPoints} = expectations;
+        const {compare} = features[feature](options);
+
+        expect(compare(featuresA, featuresB)).to.equal(expectedPoints);
+        return;
+      }
+
+      throw new Error(`Invalid type ${type}`);
+    }
+  });
 });

package/src/match-detection/features/bib/melinda-id.js

@@ -0,0 +1,34 @@
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createInterface from './melinda-identifier-factory';
+
+export default () => {
+  const {extract, compare} = createInterface();
+  return {extract, compare, name: 'melinda-id'};
+};

package/src/match-detection/features/bib/melinda-identifier-factory.js

@@ -0,0 +1,94 @@
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createDebugLogger from 'debug';
+import {getMelindaIdsF035} from '../../../matching-utils';
+
+// 003+001 FI-MELINDA <melinda-id>
+// 035 $a (FI-MELINDA)<melinda-id>
+// 035 $z (FI-MELINDA)<melinda-id>
+// 035 $a FCC<melinda-id>
+// 035 $z FCC<melinda-id>
+// melinda-id = 001234567
+
+export default () => {
+
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features/bib/melinda-id');
+  const debugData = debug.extend('data');
+
+  return {extract, compare};
+
+  function extract(record) {
+
+    const isMelindaRecord = record.get('003').some(f003 => f003.value === 'FI-MELINDA');
+    const [f001] = record.get('001').map(field => field.value);
+    const f035MelindaIds = getMelindaIdsF035(record);
+
+    if (
+      isMelindaRecord === undefined &&
+      f001 === undefined &&
+      f035MelindaIds.length < 1) {
+
+      debug(`No Melinda-IDs found`);
+      return {};
+    }
+
+    return {isMelindaRecord, f001, f035MelindaIds};
+
+  }
+
+  // eslint-disable-next-line max-statements
+  function compare(a, b) {
+
+    if (a.isMelindaRecord && b.isMelindaRecord &&
+        a.f001 === b.f001) {
+      debugData(`Melinda record's A f001 ${a.f001} matches Melinda record's B f001 ${a.f001}`);
+      return 1;
+    }
+
+    if (a.isMelindaRecord && typeof b.f035MelindaIds !== 'undefined' &&
+        b.f035MelindaIds.some(id => id === a.f001)) {
+      debugData(`Melinda record's A f001 ${a.f001} matches record B f035 ${JSON.stringify(b.f035MelindaIds)}`);
+      return 1;
+    }
+
+    if (b.isMelindaRecord && typeof a.f035MelindaIds !== 'undefined' &&
+        a.f035MelindaIds.some(id => id === b.f001)) {
+      debugData(`Melinda record's B f001 ${b.f001} matches record A f035 ${JSON.stringify(a.f035MelindaIds)}`);
+      return 1;
+    }
+
+    if (typeof a.f035MelindaIds !== 'undefined' && typeof b.f035MelindaIds !== 'undefined' &&
+         a.f035MelindaIds.some(idA => b.f035MelindaIds.some(idB => idB === idA))) {
+      debugData(`Record A f035 ${JSON.stringify(a.f035MelindaIds)} matches record B f035 ${JSON.stringify(b.f035MelindaIds)}`);
+      return 1;
+    }
+    debug(`No matching Melinda-IDs.`);
+    return 0;
+  }
+};