museria 0.2.32 → 0.2.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "museria",
-  "version": "0.2.32",
+  "version": "0.2.36",
   "description": "Decentralized music storage",
   "main": "./src/index.js",
   "bin": {
@@ -69,6 +69,7 @@
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.0",
     "akili": "^1.2.17",
+    "array-chunk-reader": "^0.1.15",
     "base64url": "^3.0.1",
     "bootstrap": "^4.5.3",
     "chalk": "^3.0.0",
@@ -76,7 +77,7 @@
     "express": "^4.17.1",
     "fs-extra": "^9.0.1",
     "lodash": "^4.17.20",
-    "metastocle": "^0.2.20",
+    "metastocle": "^0.2.23",
     "music-metadata": "^6.4.0",
     "node-fetch": "^2.6.1",
     "node-id3": "^0.2.2",
@@ -85,8 +86,8 @@
     "serve-favicon": "^2.5.0",
     "sharp": "^0.25.2",
     "splaytree": "^3.1.0",
-    "spreadable": "^0.2.16",
-    "storacle": "^0.2.16",
+    "spreadable": "^0.2.19",
+    "storacle": "^0.2.19",
     "transliteration": "^2.2.0"
   },
   "repository": {

package/src/browser/face/controllers/app/app.html

@@ -238,6 +238,13 @@
                 Once you are here, then the network is public or you have access. 
                 You can try to find a song in the storage or add your own there.
               </div>
+            </li> 
+            <li>
+              <div>How many songs are in this node?</div>
+              <div>
+                There are <b class="text-warning">${ this.songsCount }</b> song(s) at the moment. You also can find more info about the node status 
+                <a href="/status?pretty" target="_blank">here</a>.
+              </div>
             </li>          
             <li>
               <div>How to learn more about all this?</div>

package/src/browser/face/controllers/app/app.js

@@ -1,6 +1,7 @@
 import './app.scss';
 import Akili from 'akili';
 import router from 'akili/src/services/router';
+import request from 'akili/src/services/request';
 import client from '../../client';
 
 export default class App extends Akili.Component {
@@ -18,6 +19,9 @@ export default class App extends Akili.Component {
   created() {  
     this.captchaWidth = 240;
     this.findingSongsLimit = 10;
+    this.songsCountInterval = 10 * 1000;
+    this.songsCountIntervalObj = null;
+    this.scope.songsCount = 0;
     this.scope.searchInputValue = this.transition.query.f;
     this.scope.showCaptcha = false;
     this.scope.isUploading = false;
@@ -39,11 +43,16 @@ export default class App extends Akili.Component {
     this.scope.checkUploadSongTitle = this.checkUploadSongTitle.bind(this);    
     this.resetSearchEvent();
     this.resetUploadEvent();
-    this.resetSongUploadInfo();    
+    this.resetSongUploadInfo(); 
   }
 
   async compiled() {
     this.scope.searchInputValue && await this.findSongs();
+    await this.enableSongsCountCounter();
+  }
+
+  removed() {
+    clearInterval(this.songsCountIntervalObj);
   }
 
   resetSearchEvent() {
@@ -67,6 +76,16 @@ export default class App extends Akili.Component {
     };
   }
 
+  async enableSongsCountCounter() {
+    this.songsCountIntervalObj = setInterval(() => this.setSongsCount(), this.songsCountInterval);
+    await this.setSongsCount();
+  }
+
+  async setSongsCount() {
+    const data = (await request.get('/status', { json: true })).data;
+    this.scope.songsCount = data.filesCount;
+  }
+
   setFindingValue(val) {
     this.scope.searchInputValue = val;
     router.reload({}, { f: this.scope.searchInputValue || null }, undefined, { reload: false, saveScrollPosition: true });

package/src/db/transports/loki/index.js

@@ -2,6 +2,7 @@ const DatabaseMuseria = require('../database')();
 const DatabaseLoki = require('spreadable/src/db/transports/loki')(DatabaseMuseria);
 const DatabaseLokiMetastocle = require('metastocle/src/db/transports/loki')(DatabaseLoki);
 const DatabaseLokiStoracle = require('storacle/src/db/transports/loki')(DatabaseLokiMetastocle);
+const ArrayChunkReader = require('array-chunk-reader');
 const utils = require('../../../utils');
 
 module.exports = (Parent) => {
@@ -18,13 +19,13 @@ module.exports = (Parent) => {
         similarity: this.node.options.music.similarity
       }, options);      
       const fullName = this.createCollectionName('music');
-      const collection = await this.node.getCollection('music');      
+      const collection = await this.node.getCollection('music');
       const documents = this.col[fullName].find();
+      const reader = new ArrayChunkReader(documents, { limit: 1000, log: false });
       let max = null;
 
-      for(let i = 0; i < documents.length; i++) {
-        const doc = documents[i];
-        let score = doc[collection.pk] === title? 1: 0;       
+      await reader.start((doc) => {
+        let score = doc[collection.pk] === title? 1: 0;
 
         if(!score) {
           score = utils.getSongSimilarity(doc[collection.pk], title, { 
@@ -35,18 +36,18 @@ module.exports = (Parent) => {
 
         if(score === 1) {
           max = { score, doc };
-          break;
+          return reader.stop();
         }
 
         if(!max || score > max.score) {
           max = { score, doc };
-          continue;
+          return;
         }
 
         if(score == max.score && Math.random() > 0.5) {
           max = { score, doc };
         }
-      }
+      });
 
       if(max && max.score >= options.similarity) {
         return this.prepareDocumentToGet(max.doc);

package/src/node.js

@@ -5,6 +5,7 @@ const sharp = require('sharp');
 const fse = require('fs-extra');
 const qs = require('querystring');
 const SplayTree = require('splaytree');
+const ArrayChunkReader = require('array-chunk-reader');
 const DatabaseLokiMuseria = require('./db/transports/loki')();
 const ServerExpressMuseria = require('./server/transports/express')();
 const MusicCollection = require('./collection/transports/music')();
@@ -78,7 +79,8 @@ module.exports = (Parent) => {
           ]
         },
         task: {
-          cleanUpMusicInterval: '30s'
+          cleanUpMusicInterval: '30s',
+          beautifySongTitlesInterval: '10m',
         }
       }, options);
 
@@ -110,10 +112,28 @@ module.exports = (Parent) => {
       if(this.options.task.cleanUpMusicInterval) {
         await this.task.add('cleanUpMusic', this.options.task.cleanUpMusicInterval, () => this.cleanUpMusic());
       }
+
+      if(this.options.task.beautifySongTitlesInterval) {
+        await this.task.add('beautifySongTitles', this.options.task.beautifySongTitlesInterval, () => this.beautifySongTitles());
+      }     
+    }
+
+    /** 
+     * Beautify the song titles
+     * 
+     * @async
+     */
+    async beautifySongTitles() {
+      const docs = await this.db.getDocuments('music');
+      const reader = new ArrayChunkReader(docs, { size: 1000, log: false });
+      await reader.start(async (doc) => {
+        doc.title = utils.beautifySongTitle(doc.title);
+        await this.db.updateDocument(doc);
+      });
     }
 
     /** 
-     * Clean up the music  
+     * Clean up the music
      * 
      * @async
      */
@@ -124,7 +144,7 @@ module.exports = (Parent) => {
       for(let i = 0; i < docs.length; i++) {
         const doc = docs[i];
         
-        if(          
+        if(
           !doc.fileHash || 
           typeof doc.fileHash != 'string' ||
           (
@@ -516,6 +536,7 @@ module.exports = (Parent) => {
           return [];
         }
         
+        artist = utils.prepareSongFindingString(artist);
         const collection = await this.getCollection('music');     
         const actions = utils.prepareDocumentGettingActions({ 
           offset: 0,

package/src/server/transports/express/index.js

@@ -29,7 +29,7 @@ module.exports = (Parent) => {
       const remove = [
         'addDocument', 'updateDocuments', 
         'deleteDocuments', 'getDocumentsCount',
-        'getDocumentByPk'
+        'getDocumentByPk', 'getDocuments'
       ];
       return super.getClientRoutes().filter(r => !remove.includes(r.name)).concat(routesClient);
     }

package/src/utils.js

@@ -85,14 +85,13 @@ utils.prepareSongFindingString = function (str) {
     return '';
   }
   
-  str = str
+  str = this.normalizeString(str)
     .trim()
     .replace(/[–—]+/g, '-')
     .replace(/[\sᅠ]+/g, ' ');
   return str;
 }
 
-
 /**
  * Split the song title
  * 
@@ -120,7 +119,7 @@ utils.beautifySongTitle = function (title) {
     return '';
   }
 
-  title = emojiStrip(title)    
+  title = emojiStrip(this.normalizeString(title))
     .replace(/[–—]+/g, '-')
     .replace(this.regexSongLinks, '')
     .replace(/[\sᅠ]+/g, ' ')
@@ -568,10 +567,12 @@ utils.isValidSongPriority = function (value) {
  * @param {string} second
  * @param {object} [options]
  * @param {number} [options.min]
+ * @param {boolean} [options.ignoreOrder]
+ * 
  * @returns {number} 
  */
 utils.getStringSimilarity = function(first, second, options = {}) {  
-  const min = options.min || 0;
+  const min = options.min || 0;  
   let short = first;
   let long = second;
 
@@ -579,7 +580,7 @@ utils.getStringSimilarity = function(first, second, options = {}) {
     short = second;
     long = first;
   }
-
+  
   long = long.toLowerCase().split('');
   short = short.toLowerCase().split('');
   const coef = Math.sqrt(short.length * long.length);
@@ -619,4 +620,14 @@ utils.getStringSimilarity = function(first, second, options = {}) {
   return matches / coef;
 }
 
+/**
+ * Normalize the string
+ * 
+ * @param {string} str
+ * @returns {string}
+ */
+utils.normalizeString = function (str) {
+  return str.normalize("NFD").replace(/(?!\^)\p{Diacritic}/gu, "");
+}
+
 module.exports = utils;

package/test/node.js

@@ -366,6 +366,18 @@ describe('Node', () => {
     });
   });
 
+  describe('.beautifySongTitles()', () => {
+    it('should beautify song titles', async () => {
+      const title = 'ARTIST - test';
+      await node.db.addDocument('music', { title });
+      await node.beautifySongTitles();
+      const beauty = utils.beautifySongTitle(title);
+      const docs = await node.db.getDocuments('music');
+      assert.notEqual(title, beauty, 'check the title');
+      assert.equal(docs[docs.length - 1].title, beauty, 'check the doc');
+    });
+  });
+
   describe('.getStorageCleaningUpTree()', () => {
     it('should get right order of priority', async () => {
       for(let i = 0; i < 3; i++) {

package/test/utils.js

@@ -262,4 +262,10 @@ describe('utils', () => {
       assert.containsAllKeys(res, ['bitrate', 'duration', 'sampleRate']);
     });
   });
+
+  describe('.normalizeString()', () => {
+    it('should remove accents', () => {
+      assert.equal(utils.normalizeString('Mylène Farmer'), 'Mylene Farmer');
+    }); 
+  });
 });