@igea/oac_backend 1.0.46 → 1.0.47

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@igea/oac_backend",
-  "version": "1.0.46",
+  "version": "1.0.47",
   "description": "Backend service for the OAC project",
   "main": "src/index.js",
   "bin": {
@@ -29,6 +29,7 @@
     "crypto": "1.0.1",
     "express": "5.1.0",
     "express-rate-limit": "8.1.0",
+    "fast-xml-parser": "^5.3.3",
     "get-port": "7.1.0",
     "knex": "3.1.0",
     "libxmljs2": "0.37.0",

package/src/controllers/fuseki.js

@@ -75,33 +75,61 @@ router.post('/upload/vocabularies', upload.array('files'), (req, res) => {
         deleteFiles(uploadedFiles)
         return res.status(400).json({ message: 'Uploaded XML file is not valid' });
     }
-    VocabParser.insertQuery(xmlFile.path).then(query => {
+    VocabParser.insertQuery(xmlFile.path).then(queries => {
         //console.log("Query to insert vocabularies: ", query);
-        
-        deleteFiles(uploadedFiles)
-        
-        axios.post(fusekiUrlUpdate, query, {
-            headers: {
-                'Content-Type': 'application/sparql-update',
-                'Accept': 'application/sparql-results+json'
+        let results = Array(queries.length, null);
+        let checkCompleted = function(){
+            console.log(results)
+            deleteFiles(uploadedFiles)
+            let failed = results.filter(r => r.status === false);
+            console.log(failed);
+            if(failed.length > 0){
+                let message = `Error inserting vocabularies in ${failed.length} files.`;
+                return res.status(500).json({ 
+                    message,
+                    files: uploadedFiles,
+                    results
+                });
+            }else{
+                res.json({
+                    message: 'File correctly uploaded and vocabularies updated in the triple store',
+                    files: uploadedFiles
+                });
             }
+        }
+        let fusekiCall = function(index){
+            return new Promise((resolve, reject) => {
+                let query = queries[index];
+                try{
+                    axios.post(fusekiUrlUpdate, query, {
+                        headers: {
+                            'Content-Type': 'application/sparql-update',
+                            'Accept': 'application/sparql-results+json'
+                        }
+                    })
+                    .then(() => {
+                        resolve({
+                            index, success: true, message: 'Vocabulary inserted correctly'
+                        });
+                    }).catch(error => {
+                        let msg = (error.response?.status + error.response?.data) || error.message
+                        resolve({
+                            index, success: false, message: `Error from SPARQL end-point: ${msg}`
+                        });
+                    });
+                }catch(e){
+                    resolve({
+                        index, success: false, message: `Error: ${e}`
+                    });
+                }
+            })
+        }
+        setTimeout(async ()=>{
+            for(let index=0; index<queries.length; index++){
+                results[index] = await fusekiCall(index);
+            }
+            checkCompleted();
         })
-        .then(response => {
-            res.json({
-                message: 'File correctly uploaded and vocabularies updated: ' + response.data,
-                files: uploadedFiles
-            });
-        }).catch(error => {
-            let message = (error.response?.status + error.response?.data) || error.message
-            console.log(message);
-            //fs.writeFileSync('/home/nicole/Scaricati/spqr_error.txt', query);   
-            res.status(500).json({ 
-                message: 'Error from SPARQL end-point: ' + message, 
-                files: uploadedFiles,
-                query
-            });
-        });
-        
     }).catch(err => {
         deleteFiles(uploadedFiles)
         console.error('Error transforming XML:', err);

package/src/models/vocabolaries/parser

@@ -5,7 +5,7 @@ const { exec } = require('child_process');
 const stripBom = require('strip-bom').default;
 const config = require('../../config');
 const VocabPrefix = config.fuseki.vocabularies.prefix || 'diagnostica';
-
+const Splitter = require('./splitter');
 
 const transformMode = {
     default: 'default',
@@ -75,46 +75,73 @@ class Parser{
             }catch(e){
                 reject(e)        
             }
-            
-            /*    
-            let xmlData = this._getXmlData(xmlPath);
-            const xml = xmlParser.xmlParse(xmlData);
-            xslt.xsltProcess(xml, this.xslt)
-            .then(result => {
-                var terms = result.split('
')
-                resolve(terms.map(line => line.trim()).filter(line => line.length > 0));
-            }).catch(err => {
-                reject(err)
-            });
-            */
         });
     }
 
     insertQuery(xmlPath, className='crm:E55_Type'){
         return new Promise((resolve, reject) => {
-            this.transform(xmlPath, transformMode.forInsert).then(terms => {
-                let query = `
-                PREFIX crm: <http://www.cidoc-crm.org/cidoc-crm/>
-                PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+            const xmlFolder = path.dirname(xmlPath);
+            const xmlFile = path.basename(xmlPath);
+            const splitter = new Splitter(xmlFolder, xmlFile);
+            const files = splitter.splitFiles();
+            let queries = new Array(files.length);
+            let cleanTempFiles = function(queries){
+                for(let file of files){
+                    try{
+                        fs.unlinkSync(file.path);
+                    }catch(e){
+                        //console.log(`Error deleting temp file ${file.path}: ${e.message}`);
+                    }
+                }
+                resolve(queries);
+            }
+            let checkCompleted = function(index, query, status, message){
+                console.log(`Completed ${index+1}/${files.length}`);
+                files[index].status = status;
+                files[index].message = message;
+                if(status) queries[index]=query;
+                let allDone = files.every(f => f.status !== null);
+                if(allDone){
+                    cleanTempFiles(queries);
+                }
+            }
+            let transformFn = function(instance, index, className, xmlPath){
+                return function(){
+                    instance.transform(xmlPath, transformMode.forInsert).then(terms => {
+                        let query = `
+                        PREFIX crm:     <http://www.cidoc-crm.org/cidoc-crm/> 
+                        PREFIX basecpm: <http://ontome.net/ns/cpm/> 
+                        PREFIX j.0:     <http://www.cidoc-crm.org/extensions/crmsci/> 
+                        PREFIX crmsci:  <http://www.ics.forth.gr/isl/CRMsci> 
+                        PREFIX base:    <http://www.ics.forth.gr/isl/CRMinf/> 
+                        PREFIX rdfs:    <http://www.w3.org/2000/01/rdf-schema#>
 
-                INSERT {
-                    ?term a ${className} ;
-                    rdfs:label ?label ;
-                    crm:P127_has_broader_term ?broader .
+                        INSERT {
+                            ?term a ${className} ;
+                            rdfs:label ?label ;
+                            crm:P127_has_broader_term ?broader .
+                        }
+                        WHERE {
+                            VALUES (?term ?label ?broader) {
+                                ${terms.join(' \n')}
+                            }
+                            FILTER NOT EXISTS {
+                                ?term rdfs:label ?label .
+                            }
+                        }`
+                        checkCompleted(index, query, true, null);
+                    }).catch(err => {
+                        console.log(err);
+                        checkCompleted(index, null, false, `${err}`);
+                    });
                 }
-                WHERE {
-                    VALUES (?term ?label ?broader) {
-                        ${terms.join(' \n')}
-                    }
-                    FILTER NOT EXISTS {
-                        ?term rdfs:label ?label .
-                    }
-                }`
-                resolve(query);
-            }).catch(err => {
-                console.log(err);
-                reject(err);
-            });
+            }
+
+            for(let index=0; index<files.length; index++){
+                let fn = transformFn(this, index, files[index].class, files[index].path);
+                fn();
+            }
+            
         })
     }
 

package/src/models/vocabolaries/splitter.js

@@ -0,0 +1,69 @@
+const fs = require('fs');
+const path = require('path');
+const { XMLParser, XMLBuilder } = require('fast-xml-parser');
+
+class Splitter {
+
+    constructor(xmlFolder, xmlName, skipWrite=false) {
+        this.xmlFolder = xmlFolder;
+        this.xmlName = xmlName;
+        this.skipWrite = skipWrite;
+        const xmlPath = path.join(this.xmlFolder, this.xmlName);
+        this.xml = fs.readFileSync(xmlPath, 'utf8');
+        // Parser
+        this.parser = new XMLParser({
+            ignoreAttributes: false,
+            attributeNamePrefix: '@_'
+        });
+    }
+
+    splitByClass() {
+        const data = this.parser.parse(this.xml);
+        const vocabularies = data.vocabularies.vocabulary;
+
+        // Normalizza a array
+        const vocabArray = Array.isArray(vocabularies)
+            ? vocabularies
+            : [vocabularies];
+
+        // Raggruppa per class
+        const grouped = {};
+        for (const vocab of vocabArray) {
+            const cls = vocab['@_class'] || 'NO_CLASS';
+            if (!grouped[cls]) grouped[cls] = [];
+            grouped[cls].push(vocab);
+        }
+
+        return grouped;
+    }
+
+    splitFiles() {
+        const grouped = this.splitByClass();
+        const builder = new XMLBuilder({
+            ignoreAttributes: false,
+            attributeNamePrefix: '@_',
+            format: true,
+            indentBy: '  '
+        });
+        let index = 0
+        let results = [];
+        for (const cls in grouped) {
+            const outObj = {
+                vocabularies: {
+                    vocabulary: grouped[cls]
+                }
+            };
+            index++;
+            const xmlOut = builder.build(outObj);
+            const xmlOutPath = path.join(this.xmlFolder, `${this.xmlName}_split_${index}.xml`);    
+            if(!this.skipWrite)
+                fs.writeFileSync(xmlOutPath, xmlOut, 'utf8');
+            results.push({ class: cls, path: xmlOutPath, status:null, message:null });
+        }
+        return results.sort((a, b) => a.class.localeCompare(b.class));
+    }
+
+}
+
+
+module.exports = Splitter;

package/test/models/vocabolaries/parser.test.js

@@ -44,7 +44,10 @@ describe('Vocabolaries.Parsers', () => {
   it('should get the insert query for the vocabolaries.xml file', async () => {
     const parser = Parser.GET_INSTANCE();
     var query = await parser.insertQuery(__dirname + '/vocabolaries.xml');
-    expect(query.length).to.be.equal(40619);
+    expect(query.length).to.be.equal(3);
+    expect(query[0].length).to.be.equal(5279);
+    expect(query[1].length).to.be.equal(2665);
+    expect(query[2].length).to.be.equal(35150);
   });
 
 });

package/test/models/vocabolaries/splitter.test.js

@@ -0,0 +1,36 @@
+const chai = require('chai');
+const expect = chai.expect;
+const request = require('supertest');
+const Splitter = require('../../../src/models/vocabolaries/splitter');
+
+
+describe('Vocabolaries.Splitter', () => {
+  
+  beforeEach(() => {
+    
+  });
+
+  it('should create a parser instance', () => {
+    const splitter = new Splitter(__dirname, 'vocabolaries_2.xml');
+    expect(splitter).to.be.an.instanceof(Splitter);
+  });
+
+  it('should get group of classes', () => {
+    const splitter = new Splitter(__dirname, 'vocabolaries_2.xml');
+    const groups = splitter.splitByClass();
+    expect(Object.keys(groups).length).to.be.equal(6);
+  });
+
+  it('should get group of classes', () => {
+    const splitter = new Splitter(__dirname, 'vocabolaries_2.xml', true);
+    const files = splitter.splitFiles();
+    expect(files.length).to.be.equal(6);
+    expect(files[0].class).to.be.equal("base:I2_Belief");
+    expect(files[1].class).to.be.equal("basecpm:CP42_Material_Decay");
+    expect(files[2].class).to.be.equal("basecpm:CP43_Structural_Damage");
+    expect(files[3].class).to.be.equal("crm:E29_Design_or_Procedure");
+    expect(files[4].class).to.be.equal("crm:E55_Type");
+    expect(files[5].class).to.be.equal("crm:E58_Measurement_Unit");
+  });
+  
+});