music-routes-data 3.4.0 → 5.0.0

package/README.md

@@ -6,19 +6,18 @@ Music Routes Data: What musicians played on which tracks?
 Schema
 ======
 
-Data is stored in eight JSON files in the `data` directory.
+Data is stored in JSON files in the `data` directory.
 
-Four of the files contain basic data:
+These files contain basic data:
 
 * `tracks.json` contains the `names` and unique `id` string for a track.
 * `releases.json` contains the `names` and unique `id` string for a release.
 * `artists.json` contains the `names` and unique `id` string for an artist.
 * `individuals.json` contains the `names` and unique `id` string for an individual.
 
-Four of the files describe relationships between the basic data and sometimes include extra data relevant to the relationship:
+These files describe relationships between the basic data and sometimes include extra data relevant to the relationship:
 
 * `artist_track.json` contains `artistId` and `trackId` pairs where a track is attributed to an artist.
-* `individual_artist.json` contains `individualId` and `artistId` pairs where an individual is a member of an artist.
 * `individual_track.json` contains `individualId` and `trackId` pairs where the individual performed on the track. There is also an optional `credits` array that explains the person's role on the track (e.g., `["vocals","guitar"]`).
 * `track_release.json` contains `trackId` and `releaseId` pairs where a track appears on a release.
 

package/bin/mrd

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-var cli = require("../lib/cli");
-var argv = require('minimist')(process.argv.slice(2));
+const cli = require('../lib/cli')
+const argv = require('minimist')(process.argv.slice(2))
 
-cli.argv(argv);
+cli.argv(argv)

package/bin/verify

@@ -1,67 +1,103 @@
 #!/usr/bin/env node
 
-var assert = require("assert");
-var Data = require("../lib/Data");
+const assert = require('assert')
+const path = require('path')
+const Data = require('../lib/Data')
 
-var data = new Data();
+const data = new Data()
 
-var rv;
-
-rv = data.read(__dirname + "/../data");
+const rv = data.read(path.join(__dirname, '/../data'))
 if (rv.status === data.StatusEnum.ERROR) {
-  console.error(rv.message || "Unknown error");
-  process.exit(1);
+  console.error(rv.message || 'Unknown error')
+  process.exit(1)
 }
 
-var dump = {}; // local copy of all collections
+const dump = {} // local copy of all collections
+
+const collections = ['tracks', 'artists', 'releases', 'individuals', 'artist_track', 'individual_track', 'track_release']
+
+collections.forEach(function (collection) {
+  const rv = data.search({ collection })
+  assert.ok(rv.status === data.StatusEnum.OK, 'Could not retrieve contents of collection ' + collection)
+  assert.ok(rv.results.length > 0, 'Collection ' + collection + ' has no entries')
+  dump[collection] = rv.results
+})
+
+const getSet = function (collectionName, options = {}) {
+  const idName = options.idName || '_id'
+  const ignoreDuplicates = options.ignoreDuplicates || false
+  const myArray = dump[collectionName].map((item) => item[idName])
+  const mySet = new Set(myArray)
+  if (!ignoreDuplicates) {
+    assert.strictEqual(mySet.size, myArray.length, `Duplicate IDs found in ${collectionName} collection`)
+  }
+  return mySet
+}
 
-collections = ["tracks", "artists", "releases", "individuals", "artist_track", "individual_artist", "individual_track", "track_release"];
+const compare = function (a, b, nameA, nameB) {
+  const diff = a.symmetricDifference(b)
+  assert.strictEqual(diff.size, 0, `The ${nameA} and ${nameB} collections have different IDs: ${Array.from(diff).join(', ')}`)
+}
 
-collections.forEach(function(collection) {
-  var rv = data.search({collection: collection});
-  assert.ok(rv.status === data.StatusEnum.OK, "Could not retrieve contents of collection " + collection);
-  assert.ok(rv.results.length > 0, "Collection " + collection + " has no entries");
-  dump[collection] = rv.results;
-});
+const trackIds = getSet('tracks')
+const artistIds = getSet('artists')
+const releaseIds = getSet('releases')
+const individualIds = getSet('individuals')
 
-// Check that all instances of basic are included in relation as propName.
-// For example, all instances of 'artists' included in 'at' as 'artistId'
-dump.included = function (basic, relation, propName) {
+const artistTrackIds = {
+  artist: getSet('artist_track', { idName: 'artist_id', ignoreDuplicates: true }),
+  track: getSet('artist_track', { idName: 'track_id', ignoreDuplicates: true })
+}
+compare(artistTrackIds.artist, artistIds, 'artist_track', 'artist_id')
+compare(artistTrackIds.track, trackIds, 'artist_track', 'track_id')
 
-  var related = function (basicValue) {
+const individualTrackIds = {
+  individual: getSet('individual_track', { idName: 'individual_id', ignoreDuplicates: true }),
+  track: getSet('individual_track', { idName: 'track_id', ignoreDuplicates: true })
+}
+compare(individualTrackIds.individual, individualIds, 'individual_track', 'individual_id')
+compare(individualTrackIds.track, trackIds, 'individual_track', 'track_id')
 
-    var _idInRelatedTable = function (relationValue) {
-      return relationValue[propName] === basicValue._id;
-    };
+const trackReleaseIds = {
+  release: getSet('track_release', { idName: 'release_id', ignoreDuplicates: true }),
+  track: getSet('track_release', { idName: 'track_id', ignoreDuplicates: true })
+}
+compare(trackReleaseIds.release, releaseIds, 'track_release', 'release_id')
+compare(trackReleaseIds.track, trackIds, 'track_release', 'track_id')
+
+// Check that all instances of basic are included in relation as propName.
+// For example, check that all instances of 'artists' are included in 'at' as 'artistId'.
+dump.included = function (basic, relation, propName) {
+  const related = function (basicValue) {
+    const _idInRelatedTable = function (relationValue) {
+      return relationValue[propName] === basicValue._id
+    }
 
-    var rv = dump[relation].some(_idInRelatedTable);
+    const rv = dump[relation].some(_idInRelatedTable)
 
-    if (! rv) {
-      console.log("`" + basic + "` contains _id " + basicValue._id + " but it is not found in `" + relation + "` as `" + propName + "`");
+    if (!rv) {
+      console.log('`' + basic + '` contains _id ' + basicValue._id + ' but it is not found in `' + relation + '` as `' + propName + '`')
     }
 
-    return rv;
-  };
+    return rv
+  }
 
-  return dump[basic].every(related);
-};
+  return dump[basic].every(related)
+}
 
-//TODO: Correct/expand data so that all of these are true.
 // All artists should have tracks.
-assert.ok(dump.included("artists", "artist_track", "artist_id"));
+assert.ok(dump.included('artists', 'artist_track', 'artist_id'))
 // All tracks should have artists.
-assert.ok(dump.included("tracks", "artist_track", "track_id"));
-// All artists have individuals. (Not all individuals are members of artists, though.)
-// assert.ok(dump.included("artists", "individual_artist", "artist_id"));
+assert.ok(dump.included('tracks', 'artist_track', 'track_id'))
 // All individuals are associated with at least one track.
-// assert.ok(dump.included("individuals", "individual_track", "individual_id"));
+assert.ok(dump.included('individuals', 'individual_track', 'individual_id'))
+// All tracks are associated with at least one individual.
+assert.ok(dump.included('tracks', 'individual_track', 'track_id'))
 // All tracks are on at least one release.
-assert.ok(dump.included("tracks", "track_release", "track_id"));
+assert.ok(dump.included('tracks', 'track_release', 'track_id'))
 // All releases have at least one track.
-assert.ok(dump.included("releases", "track_release", "release_id"));
+assert.ok(dump.included('releases', 'track_release', 'release_id'))
 
-// TODO: Make sure that all artist IDs that appear in artist_track actually exist in artists, etc.
-// TODO: Make sure that all _ids are unique
 // TODO: Make sure that all required fields are present in every entry (including "credits", yeah? even if empty)
 // TODO: Make sure that there are no duplicate link entries (Elvis on "Love Me Tender" exists twice should be an error)
 // TODO: Check that all fields that should be strings are strings and all fields that should be arrays of strings are arrays of strings

package/data/artist_track.json

@@ -41523,10 +41523,6 @@
     "artist_id": "1517",
     "track_id": "10255"
   },
-  {
-    "artist_id": "591",
-    "track_id": "10256"
-  },
   {
     "artist_id": "483",
     "track_id": "10257"
@@ -41607,10 +41603,6 @@
     "artist_id": "1520",
     "track_id": "10276"
   },
-  {
-    "artist_id": "1521",
-    "track_id": "10277"
-  },
   {
     "artist_id": "1522",
     "track_id": "10278"
@@ -41687,26 +41679,6 @@
     "artist_id": "1531",
     "track_id": "10297"
   },
-  {
-    "artist_id": "1531",
-    "track_id": "10298"
-  },
-  {
-    "artist_id": "1531",
-    "track_id": "10299"
-  },
-  {
-    "artist_id": "1531",
-    "track_id": "10300"
-  },
-  {
-    "artist_id": "1531",
-    "track_id": "10301"
-  },
-  {
-    "artist_id": "1532",
-    "track_id": "10302"
-  },
   {
     "artist_id": "1533",
     "track_id": "10303"
@@ -41771,14 +41743,6 @@
     "artist_id": "1537",
     "track_id": "10318"
   },
-  {
-    "artist_id": "1538",
-    "track_id": "10319"
-  },
-  {
-    "artist_id": "1538",
-    "track_id": "10320"
-  },
   {
     "artist_id": "1539",
     "track_id": "10321"
@@ -41959,10 +41923,6 @@
     "artist_id": "1547",
     "track_id": "10365"
   },
-  {
-    "artist_id": "1422",
-    "track_id": "10366"
-  },
   {
     "artist_id": "1548",
     "track_id": "10367"
@@ -42571,10 +42531,6 @@
     "artist_id": "1234",
     "track_id": "10517"
   },
-  {
-    "artist_id": "1234",
-    "track_id": "10518"
-  },
   {
     "artist_id": "1234",
     "track_id": "10519"
@@ -43271,10 +43227,6 @@
     "artist_id": "479",
     "track_id": "10692"
   },
-  {
-    "artist_id": "479",
-    "track_id": "10693"
-  },
   {
     "artist_id": "479",
     "track_id": "10694"
@@ -45047,34 +44999,6 @@
     "artist_id": "2",
     "track_id": "11135"
   },
-  {
-    "artist_id": "2",
-    "track_id": "11136"
-  },
-  {
-    "artist_id": "2",
-    "track_id": "11137"
-  },
-  {
-    "artist_id": "2",
-    "track_id": "11138"
-  },
-  {
-    "artist_id": "2",
-    "track_id": "11139"
-  },
-  {
-    "artist_id": "2",
-    "track_id": "11140"
-  },
-  {
-    "artist_id": "2",
-    "track_id": "11141"
-  },
-  {
-    "artist_id": "2",
-    "track_id": "11142"
-  },
   {
     "artist_id": "2",
     "track_id": "11143"
@@ -45363,10 +45287,6 @@
     "artist_id": "2",
     "track_id": "11215"
   },
-  {
-    "artist_id": "2",
-    "track_id": "11216"
-  },
   {
     "artist_id": "2",
     "track_id": "11217"
@@ -47975,14 +47895,6 @@
     "artist_id": "97",
     "track_id": "11837"
   },
-  {
-    "artist_id": "97",
-    "track_id": "11838"
-  },
-  {
-    "artist_id": "97",
-    "track_id": "11839"
-  },
   {
     "artist_id": "97",
     "track_id": "11840"
@@ -48727,18 +48639,10 @@
     "artist_id": "212",
     "track_id": "12025"
   },
-  {
-    "artist_id": "212",
-    "track_id": "12026"
-  },
   {
     "artist_id": "212",
     "track_id": "12027"
   },
-  {
-    "artist_id": "212",
-    "track_id": "12028"
-  },
   {
     "artist_id": "212",
     "track_id": "12029"
@@ -48975,10 +48879,6 @@
     "artist_id": "212",
     "track_id": "12087"
   },
-  {
-    "artist_id": "212",
-    "track_id": "12088"
-  },
   {
     "artist_id": "212",
     "track_id": "12089"
@@ -49267,10 +49167,6 @@
     "artist_id": "212",
     "track_id": "12160"
   },
-  {
-    "artist_id": "212",
-    "track_id": "12161"
-  },
   {
     "artist_id": "212",
     "track_id": "12162"
@@ -49287,18 +49183,6 @@
     "artist_id": "212",
     "track_id": "12165"
   },
-  {
-    "artist_id": "212",
-    "track_id": "12166"
-  },
-  {
-    "artist_id": "212",
-    "track_id": "12167"
-  },
-  {
-    "artist_id": "212",
-    "track_id": "12168"
-  },
   {
     "artist_id": "212",
     "track_id": "12169"
@@ -50247,10 +50131,6 @@
     "artist_id": "2",
     "track_id": "12403"
   },
-  {
-    "artist_id": "2",
-    "track_id": "12404"
-  },
   {
     "artist_id": "2",
     "track_id": "12405"

package/data/artists.json

@@ -8955,12 +8955,6 @@
       "Actionslacks"
     ]
   },
-  {
-    "_id": "1521",
-    "names": [
-      "The Admirals"
-    ]
-  },
   {
     "_id": "1522",
     "names": [
@@ -9021,12 +9015,6 @@
       "Robert Craft"
     ]
   },
-  {
-    "_id": "1532",
-    "names": [
-      "Edgar Varèse"
-    ]
-  },
   {
     "_id": "1533",
     "names": [
@@ -9057,12 +9045,6 @@
       "Common"
     ]
   },
-  {
-    "_id": "1538",
-    "names": [
-      "David Fanshawe"
-    ]
-  },
   {
     "_id": "1539",
     "names": [