@natlibfi/melinda-record-matching 1.0.8 → 2.0.0

package/src/candidate-search/index.spec.js

@@ -32,78 +32,84 @@ import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
 import createSearchInterface, {CandidateSearchError} from '.';
 
-generateTests({
-  callback,
-  path: [__dirname, '..', '..', 'test-fixtures', 'candidate-search', 'index'],
-  recurse: false,
-  fixura: {
-    reader: READERS.JSON
-  }
-});
+describe('candidate-search', () => {
+  generateTests({
+    callback,
+    path: [__dirname, '..', '..', 'test-fixtures', 'candidate-search', 'index'],
+    recurse: false,
+    fixura: {
+      reader: READERS.JSON
+    }
+  });
 
-// eslint-disable-next-line max-statements
-async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError, expectedSearchError}) {
-  const url = 'http://foo.bar';
+  // eslint-disable-next-line max-statements
+  async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError, expectedSearchError, enabled = true}) {
+    const url = 'http://foo.bar';
 
-  if (expectedFactoryError) {
-    if (expectedFactoryError.isCandidateSearchError) {
-      expect(() => createSearchInterface({...formatFactoryOptions(), url})).to.throw(CandidateSearchError, new RegExp(expectedFactoryError, 'u'));
+    if (!enabled) {
       return;
     }
 
-    expect(() => createSearchInterface({...formatFactoryOptions(), url})).to.throw(new RegExp(expectedFactoryError, 'u'));
-    return;
-  }
+    if (expectedFactoryError) {
+      if (expectedFactoryError.isCandidateSearchError) {
+        expect(() => createSearchInterface({...formatFactoryOptions(), url})).to.throw(CandidateSearchError, new RegExp(expectedFactoryError, 'u'));
+        return;
+      }
 
-  const search = createSearchInterface({...formatFactoryOptions(), url});
-  await iterate({searchOptions});
+      expect(() => createSearchInterface({...formatFactoryOptions(), url})).to.throw(new RegExp(expectedFactoryError, 'u'));
+      return;
+    }
 
-  function formatFactoryOptions() {
-    return {
-      ...factoryOptions,
-      maxRecordsPerRequest: 1,
-      record: new MarcRecord(factoryOptions.record, {subfieldValues: false})
-    };
-  }
+    const search = createSearchInterface({...formatFactoryOptions(), url});
+    await iterate({searchOptions});
 
-  // eslint-disable-next-line max-statements
-  async function iterate({searchOptions, count = 1}) {
-    const expectedResults = getFixture(`expectedResults${count}.json`);
+    function formatFactoryOptions() {
+      return {
+        ...factoryOptions,
+        maxRecordsPerRequest: 1,
+        record: new MarcRecord(factoryOptions.record, {subfieldValues: false})
+      };
+    }
 
-    if (expectedSearchError) { // eslint-disable-line functional/no-conditional-statement
-      try {
-        await search(searchOptions);
-        throw new Error('Expected an error');
-      } catch (err) {
-        expect(err).to.be.an('error');
-        expect(err.message).to.match(new RegExp(expectedSearchError, 'u'));
-        return;
+    // eslint-disable-next-line max-statements
+    async function iterate({searchOptions, count = 1}) {
+      const expectedResults = getFixture(`expectedResults${count}.json`);
+
+      if (expectedSearchError) { // eslint-disable-line functional/no-conditional-statement
+        try {
+          await search(searchOptions);
+          throw new Error('Expected an error');
+        } catch (err) {
+          expect(err).to.be.an('error');
+          expect(err.message).to.match(new RegExp(expectedSearchError, 'u'));
+          return;
+        }
       }
-    }
 
-    const results = await search(searchOptions);
+      const results = await search(searchOptions);
 
-    expect(formatResults(results)).to.eql(expectedResults);
+      expect(formatResults(results)).to.eql(expectedResults);
 
-    if (results.records.length > 0) {
-      return iterate({
-        searchOptions: resultsToOptions(results),
-        count: count + 1
-      });
-    }
+      if (results.records.length > 0) {
+        return iterate({
+          searchOptions: resultsToOptions(results),
+          count: count + 1
+        });
+      }
 
-    function formatResults(results) {
-      // console.log(results); //eslint-disable-line
-      return {
-        ...results,
-        records: results.records.map(({record, id}) => ({id, record: record.toObject()}))
-      };
-    }
+      function formatResults(results) {
+        // console.log(results); //eslint-disable-line
+        return {
+          ...results,
+          records: results.records.map(({record, id}) => ({id, record: record.toObject()}))
+        };
+      }
 
-    function resultsToOptions(results) {
-      return Object.entries(results)
-        .filter(([k]) => k === 'records' === false)
-        .reduce((acc, [k, v]) => ({...acc, [k]: v}), {});
+      function resultsToOptions(results) {
+        return Object.entries(results)
+          .filter(([k]) => k === 'records' === false) // If key is 'records' return false
+          .reduce((acc, [k, v]) => ({...acc, [k]: v}), {});
+      }
     }
   }
-}
+});

package/src/candidate-search/query-list/bib.js

@@ -27,6 +27,125 @@
 *
 */
 import createDebugLogger from 'debug';
+import {toQueries} from '../candidate-search-utils';
+import {getMelindaIdsF035, validateSidFieldSubfieldCounts, getSubfieldValues} from '../../matching-utils';
+
+
+export function bibSourceIds(record) {
+
+  /* Melinda's SRU-index melinda.sourceid includes source IDs from SID fields in Melinda records
+     SID-fields in Melinda have sf $c with local id and sf $b with a code for the local db:
+
+     SID__ $c 123457 $b helka
+     SID__ $c (ANDL100020)1077305 $b sata
+     SID__ $c VER999999 $ FI-KV
+     SID__ $c /10024/508126 $ REPO_THESEUS
+
+    In melinda.sourceid -index case is kept, sourceprefixes in brackets and hyphens are normalized away:
+    Note: slashes are not normalized away, but a SRU-search-string including slashes needs to be quoted
+
+    1234567helka
+    1077305sata
+    VER999999FIKV
+    /10024/508126REPO_THESEUS
+
+    Note: All Melinda records that have a matching records in a local db do NOT have SID for that local records,
+          existence of a SID field depends on how the record has been added to Melinda and how it has been handled
+          afterwards. SIDs are also not reliably maintained. A record might or might not have a SID for a local db
+          after the matching record is removed from the local db.
+
+   */
+
+
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:source-ids');
+  const debugData = debug.extend('data');
+  //const debugInfo = debug.extend('info');
+
+  debug(`Creating queries for sourceid's`);
+
+  const fSids = record.get('SID');
+  debugData(`SID-fields (${fSids.length}): ${JSON.stringify(fSids)}`);
+
+  return fSids.length > 0 ? toSidQueries(fSids) : [];
+
+  function toSidQueries(fSids) {
+    debug(`Creating actual queries for sourceid's`);
+
+    const sidStrings = getSidStrings(fSids);
+
+    if (sidStrings.length < 1) {
+      debug(`No identifiers found, no queries created.`);
+      return [];
+    }
+
+    const sidQueries = toQueries(sidStrings, 'melinda.sourceid');
+
+    return sidQueries;
+
+    function getSidStrings(fSids) {
+      debug(`Getting Sid strings from SID fields`);
+
+      // Map SID fields to valid sidStrings, filter out empty strings
+      const sidStrings = fSids.map(toSidString).filter(nonEmptySid => nonEmptySid);
+      return sidStrings;
+
+      function toSidString(field) {
+        debug(`Getting string from a field`);
+
+        return validateSidFieldSubfieldCounts(field) ? createSidString(field) : '';
+
+        function createSidString(field) {
+          debug(`Creating string from a field`);
+          const [sfC] = getSubfieldValues(field, 'c');
+          const [sfB] = getSubfieldValues(field, 'b');
+
+          const cleanedSfC = removeSourcePrefix(normalizeSidSubfieldValue(sfC));
+          const cleanedSfB = normalizeSidSubfieldValue(sfB);
+
+          debugData(`${JSON.stringify(sfC)} + ${JSON.stringify(sfB)}`);
+          return cleanedSfC.concat(cleanedSfB);
+        }
+
+        function removeSourcePrefix(subfieldValue) {
+          const sourcePrefixRegex = (/^(?<sourcePrefix>\([A-Za-z0-9-]+\))(?<id>.+)$/u);
+          const normalizedValue = subfieldValue.replace(sourcePrefixRegex, '$<id>');
+          debugData(`Normalized ${subfieldValue} to ${normalizedValue}`);
+          return normalizedValue;
+        }
+
+        function normalizeSidSubfieldValue(subfieldValue) {
+          debugData(`Normalizing ${subfieldValue}`);
+          const normalizeAwayRegex = (/[- ]/u);
+          return subfieldValue.replace(normalizeAwayRegex, '');
+        }
+
+      }
+    }
+  }
+}
+
+export function bibMelindaIds(record) {
+  // Melinda's SRU-index melinda.melindaid includes f001 controlnumbers and old Melinda-IDs from f035z's for all non-deleted Melinda-records
+
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:query:bibMelindaIds');
+  const debugData = debug.extend('data');
+  debug(`Creating queries for MelindaIds`);
+
+  // Note: Melinda-ID's for search queries are created just from records f035a's and f035z's
+  // Both (FI-MELINDA)- and FCC-prefixed forms are found
+  // f001 controlnumber is not currently included, even if record's f003 is FI-MELINDA
+  const melindaIds = getMelindaIdsF035(record);
+
+  debugData(`Unique identifiers (${melindaIds.length}): ${JSON.stringify(melindaIds)}`);
+
+  if (melindaIds.length < 1) {
+    debug(`No identifiers found, no queries created.`);
+    return [];
+  }
+
+  return toQueries(melindaIds, 'melinda.melindaid');
+}
+
 
 // bibHostComponents returns host id from the first subfield $w of first field f773, see test-fixtures 04 and 05
 // bibHostComponents should search all 773 $ws for possible host id, but what should it do in case of multiple host ids?
@@ -54,12 +173,20 @@ export function bibHostComponents(record) {
   }
 }
 
+// SRU search dc.title with a search phrase starting with ^ maps currently in Melinda to
+// (probably) to *headings* index TIT
+// Headings index TIT drops articles etc. from the start of the title according to the filing indicator
+// Currently filing indicator is not implemented - if the title starts with an article and the Melinda
+// record is correctly catalogued using a filing indicator -> dc.title search won't match
+
 export function bibTitle(record) {
   const title = getTitle();
 
   if (title) {
     const formatted = title
       .replace(/[^\w\s\p{Alphabetic}]/gu, '')
+      // Clean up concurrent spaces from fe. subfield changes
+      .replace(/ +/gu, ' ')
       .trim()
       .slice(0, 30)
       .trim();
@@ -77,7 +204,8 @@ export function bibTitle(record) {
       return field.subfields
         .filter(({code}) => ['a', 'b'].includes(code))
         .map(({value}) => value)
-        .join('');
+        // In Melinda's index subfield separators are indexed as ' '
+        .join(' ');
     }
     return false;
   }
@@ -103,25 +231,7 @@ export function bibStandardIdentifiers(record) {
     return [];
   }
 
-  return toQueries(uniqueIdentifiers);
-
-  function toQueries(identifiers) {
-    // Aleph supports only two queries with or -operator (This is not actually true)
-    const pairs = toPairs(identifiers);
-    const queries = pairs.map(([a, b]) => b ? `dc.identifier=${a} or dc.identifier=${b}` : `dc.identifier=${a}`);
-
-    debugData(`Pairs (${pairs.length}): ${JSON.stringify(pairs)}`);
-    debugData(`Queries (${queries.length}): ${JSON.stringify(queries)}`);
-
-    return queries;
-
-    function toPairs(array) {
-      if (array.length === 0) {
-        return [];
-      }
-      return [array.slice(0, 2)].concat(toPairs(array.slice(2), 2));
-    }
-  }
+  return toQueries(uniqueIdentifiers, 'dc.identifier');
 
   function toIdentifiers({tag, subfields}) {
     const issnIsbnReqExp = (/^[A-Za-z0-9-]+$/u);

package/src/candidate-search/query-list/bib.spec.js

@@ -32,20 +32,22 @@ import {expect} from 'chai';
 import {MarcRecord} from '@natlibfi/marc-record';
 import * as generators from './bib';
 
-generateTests({
-  path: [__dirname, '..', '..', '..', 'test-fixtures', 'candidate-search', 'query-list', 'bib'],
-  useMetadataFile: true,
-  fixura: {
-    reader: READERS.JSON
-  },
-  callback: ({type, inputRecord, expectedQuery, enabled = true}) => {
-    const generate = generators[type];
-    const record = new MarcRecord(inputRecord, {subfieldValues: false});
+describe('candidate-search/query-list/bib/', () => {
+  generateTests({
+    path: [__dirname, '..', '..', '..', 'test-fixtures', 'candidate-search', 'query-list', 'bib'],
+    useMetadataFile: true,
+    fixura: {
+      reader: READERS.JSON
+    },
+    callback: ({type, inputRecord, expectedQuery, enabled = true}) => {
+      const generate = generators[type];
+      const record = new MarcRecord(inputRecord, {subfieldValues: false});
 
-    if (!enabled) {
-      return;
-    }
+      if (!enabled) {
+        return;
+      }
 
-    expect(generate(record)).to.eql(expectedQuery);
-  }
+      expect(generate(record)).to.eql(expectedQuery);
+    }
+  });
 });

package/src/candidate-search/query-list/index.js

@@ -32,7 +32,9 @@ export const searchTypes = {
   bib: {
     standardIdentifiers: 'bibStandardIdentifiers',
     hostComponents: 'bibHostComponents',
-    title: 'bibTitle'
+    title: 'bibTitle',
+    melindaId: 'bibMelindaIds',
+    sourceIds: 'bibSourceIds'
   }
 };
 

package/src/index.spec.js

@@ -32,49 +32,56 @@ import generateTests from '@natlibfi/fixugen-http-client';
 import {MarcRecord} from '@natlibfi/marc-record';
 import createMatchInterface, {matchDetection} from '.';
 
-generateTests({
-  callback,
-  path: [__dirname, '..', 'test-fixtures', 'index'],
-  recurse: false,
-  fixura: {
-    reader: READERS.JSON
-  }
-});
+describe('INDEX', () => {
+  generateTests({
+    callback,
+    path: [__dirname, '..', 'test-fixtures', 'index'],
+    recurse: false,
+    fixura: {
+      reader: READERS.JSON
+    }
+  });
 
-async function callback({getFixture, options}) {
-  const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});
-  const expectedMatches = getFixture('expectedMatches.json');
+  async function callback({getFixture, options, enabled = true}) {
 
-  const match = createMatchInterface(formatOptions());
-  const matches = await match(record);
+    if (!enabled) {
+      return;
+    }
 
-  expect(formatResults()).to.eql(expectedMatches);
+    const record = new MarcRecord(getFixture('inputRecord.json'), {subfieldValues: false});
+    const expectedMatches = getFixture('expectedMatches.json');
 
-  function formatOptions() {
-    const contextFeatures = matchDetection.features[options.detection.strategy.type];
+    const match = createMatchInterface(formatOptions());
+    const matches = await match(record);
 
-    return {
-      ...options,
-      search: {
-        ...options.search,
-        maxRecordsPerRequest: 1
-      },
-      detection: {
-        ...options.detect,
-        strategy: options.detection.strategy.features.map(v => contextFeatures[v]())
-      },
-      maxMatches: 2,
-      maxCandidates: 1
-    };
-  }
+    expect(formatResults()).to.eql(expectedMatches);
 
-  function formatResults() {
-    return matches.map(({candidate, probability}) => ({
-      probability,
-      candidate: {
-        id: candidate.id,
-        record: candidate.record.toObject()
-      }
-    }));
+    function formatOptions() {
+      const contextFeatures = matchDetection.features[options.detection.strategy.type];
+
+      return {
+        ...options,
+        search: {
+          ...options.search,
+          maxRecordsPerRequest: 1
+        },
+        detection: {
+          ...options.detect,
+          strategy: options.detection.strategy.features.map(v => contextFeatures[v]())
+        },
+        maxMatches: 2,
+        maxCandidates: 1
+      };
+    }
+
+    function formatResults() {
+      return matches.map(({candidate, probability}) => ({
+        probability,
+        candidate: {
+          id: candidate.id,
+          record: candidate.record.toObject()
+        }
+      }));
+    }
   }
-}
+});

package/src/match-detection/features/bib/all-source-ids.js

@@ -0,0 +1,122 @@
+/* eslint-disable max-statements */
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createDebugLogger from 'debug';
+import {validateSidFieldSubfieldCounts, getSubfieldValues} from '../../../matching-utils';
+
+const debug = createDebugLogger('@natlibfi/melinda-record-matching:match-detection:features:all-source-ids');
+const debugData = debug.extend('data');
+
+/*
+
+SID-fields in Melinda have sf $c with local id and sf $b with a code for the local db:
+
+ SID__ $c 123457 $b helka
+ SID__ $c (ANDL100020)1077305 $b sata
+ SID__ $c VER999999 $ FI-KV
+
+allSourceIds matching feature is formatted as [{sourceDb: "helka", sourceId: "123457"}, {sourceDb: "sata", sourceId: "(ANDL100020)1077305"}, {sourceDb: "FI-KV", sourceId: "VER999999"}]
+- no normalization / prefix deletions etc. in the feature
+
+Note: All Melinda records that have a matching records in a local db do NOT have SID for that local records,
+      existence of a SID field depends on how the record has been added to Melinda and how it has been handled
+      afterwards. SIDs are also not reliably maintained. A record might or might not have a SID for a local db
+      after the matching record is removed from the local db.
+
+Records with mismatching local ids for matching local db are a strong mismatch.
+Records with matching local ids for matching local db are a very good match.
+
+*/
+
+export default () => ({
+  name: 'All source IDs',
+  extract: record => {
+    debug(`Creating match detection features for all local source id's`);
+
+    const fSids = record.get('SID');
+    debugData(`SID-fields (${fSids.length}): ${JSON.stringify(fSids)}`);
+
+    const sidFeatures = getSidFeatures(fSids);
+    debugData(`SID-features (${sidFeatures.length}): ${JSON.stringify(sidFeatures)}`);
+
+    return sidFeatures;
+
+    function getSidFeatures(fSids) {
+      debug(`Getting Sid strings from SID fields`);
+
+      // Map SID fields to sidFeatures, filter out empty strings
+      const sidFeatures = fSids.map(toSidFeature).filter(nonEmptySid => nonEmptySid);
+      return sidFeatures;
+
+      function toSidFeature(field) {
+        debug(`Getting feature from a field`);
+
+        return validateSidFieldSubfieldCounts(field) ? createSidFeature(field) : '';
+
+        function createSidFeature(field) {
+          debug(`Creating feature from a field`);
+          const [sfC] = getSubfieldValues(field, 'c');
+          const [sfB] = getSubfieldValues(field, 'b');
+
+          debugData(`${JSON.stringify(sfC)} + ${JSON.stringify(sfB)}`);
+          debugData(`sourceDb: ${sfB}, sourceId: ${sfC}`);
+          return {'sourceDb': sfB, 'sourceId': sfC};
+        }
+      }
+
+    }
+
+  },
+  compare: (a, b) => {
+
+    debugData(`Comparing ${JSON.stringify(a)} and ${JSON.stringify(b)}`);
+
+    const matches = a.filter(sidA => b.some(sidB => sidA.sourceDb === sidB.sourceDb &&
+            sidA.sourceId === sidB.sourceId));
+    debugData(`Matches (${matches.length}): ${JSON.stringify(matches)}`);
+
+    const mismatches = a.filter(sidA => b.some(sidB => sidA.sourceDb === sidB.sourceDb &&
+      sidA.sourceId !== sidB.sourceId));
+    debugData(`Mismatches (${mismatches.length}): ${JSON.stringify(mismatches)}`);
+
+    // If there's at least one mismatching source ID from matching source db
+    if (mismatches.length > 0) {
+      return -1;
+    }
+
+    // If there's at least one matching source ID from matching source db
+    if (matches.length > 0) {
+      return 1;
+    }
+
+    // If there are no source IDs with matching sources
+    return 0;
+
+  }
+});

package/src/match-detection/features/bib/index.js

@@ -36,3 +36,5 @@ export {default as recordType} from './record-type';
 export {default as publicationTime} from './publication-time';
 export {default as language} from './language';
 export {default as bibliographicLevel} from './bibliographic-level';
+export {default as melindaId} from './melinda-id';
+export {default as allSourceIds} from './all-source-ids';

package/src/match-detection/features/bib/index.spec.js

@@ -32,30 +32,39 @@ import {expect} from 'chai';
 import {MarcRecord} from '@natlibfi/marc-record';
 import * as features from '.';
 
-generateTests({
-  path: [__dirname, '..', '..', '..', '..', 'test-fixtures', 'match-detection', 'features', 'bib'],
-  useMetadataFile: true,
-  fixura: {
-    reader: READERS.JSON
-  },
-  callback: ({feature, options, type, ...expectations}) => {
-    if (type === 'extract') {
-      const {expectedFeatures, inputRecord} = expectations;
-      const record = new MarcRecord(inputRecord, {subfieldValues: false});
-      const {extract} = features[feature](options);
-
-      expect(extract(record)).to.eql(expectedFeatures);
-      return;
-    }
 
-    if (type === 'compare') {
-      const {featuresA, featuresB, expectedPoints} = expectations;
-      const {compare} = features[feature](options);
+describe('match-detection/features/bib/', () => {
+  generateTests({
+    path: [__dirname, '..', '..', '..', '..', 'test-fixtures', 'match-detection', 'features', 'bib'],
+    useMetadataFile: true,
+    fixura: {
+      reader: READERS.JSON
+    },
+    // eslint-disable-next-line max-statements
+    callback: ({enabled = true, feature, options, type, ...expectations}) => {
 
-      expect(compare(featuresA, featuresB)).to.equal(expectedPoints);
-      return;
-    }
+      if (!enabled) {
+        return;
+      }
+
+      if (type === 'extract') {
+        const {expectedFeatures, inputRecord} = expectations;
+        const record = new MarcRecord(inputRecord, {subfieldValues: false});
+        const {extract} = features[feature](options);
+
+        expect(extract(record)).to.eql(expectedFeatures);
+        return;
+      }
 
-    throw new Error(`Invalid type ${type}`);
-  }
+      if (type === 'compare') {
+        const {featuresA, featuresB, expectedPoints} = expectations;
+        const {compare} = features[feature](options);
+
+        expect(compare(featuresA, featuresB)).to.equal(expectedPoints);
+        return;
+      }
+
+      throw new Error(`Invalid type ${type}`);
+    }
+  });
 });