@igea/oac_backend 1.0.46 → 1.0.48

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@igea/oac_backend",
-  "version": "1.0.46",
+  "version": "1.0.48",
   "description": "Backend service for the OAC project",
   "main": "src/index.js",
   "bin": {
@@ -29,6 +29,7 @@
     "crypto": "1.0.1",
     "express": "5.1.0",
     "express-rate-limit": "8.1.0",
+    "fast-xml-parser": "^5.3.3",
     "get-port": "7.1.0",
     "knex": "3.1.0",
     "libxmljs2": "0.37.0",

package/src/controllers/fuseki.js

@@ -3,16 +3,10 @@ const multer = require('multer');
 const router = express.Router();
 const path = require('path');
 const fs = require('fs');
-const config = require('../config')
-const configFuseki = config.fuseki || {
-    "protocol": "http",
-    "host": "127.0.0.1",
-    "port": "3030",
-    "dataset": "oac"
-}
-const fusekiUrlDataset = `${configFuseki.protocol}://${configFuseki.host}:${configFuseki.port}/${configFuseki.dataset}`;
-const fusekiUrl = `${fusekiUrlDataset}/sparql`;
-const fusekiUrlUpdate = `${fusekiUrlDataset}/update`;
+const {
+    fusekiUrl,
+    fusekiUrlUpdate
+} = require('../models/fusekiConfig');
 const axios = require('axios');
 const Fuseki = require('../models/fuseki');
 const { Parser, transformMode } = require('../models/vocabolaries/parser');
@@ -75,33 +69,61 @@ router.post('/upload/vocabularies', upload.array('files'), (req, res) => {
         deleteFiles(uploadedFiles)
         return res.status(400).json({ message: 'Uploaded XML file is not valid' });
     }
-    VocabParser.insertQuery(xmlFile.path).then(query => {
+    VocabParser.insertQuery(xmlFile.path).then(queries => {
         //console.log("Query to insert vocabularies: ", query);
-        
-        deleteFiles(uploadedFiles)
-        
-        axios.post(fusekiUrlUpdate, query, {
-            headers: {
-                'Content-Type': 'application/sparql-update',
-                'Accept': 'application/sparql-results+json'
+        let results = Array(queries.length, null);
+        let checkCompleted = function(){
+            console.log(results)
+            deleteFiles(uploadedFiles)
+            let failed = results.filter(r => r.status === false);
+            console.log(failed);
+            if(failed.length > 0){
+                let message = `Error inserting vocabularies in ${failed.length} files.`;
+                return res.status(500).json({ 
+                    message,
+                    files: uploadedFiles,
+                    results
+                });
+            }else{
+                res.json({
+                    message: 'File correctly uploaded and vocabularies updated in the triple store',
+                    files: uploadedFiles
+                });
             }
+        }
+        let fusekiCall = function(index){
+            return new Promise((resolve, reject) => {
+                let query = queries[index];
+                try{
+                    axios.post(fusekiUrlUpdate, query, {
+                        headers: {
+                            'Content-Type': 'application/sparql-update',
+                            'Accept': 'application/sparql-results+json'
+                        }
+                    })
+                    .then(() => {
+                        resolve({
+                            index, success: true, message: 'Vocabulary inserted correctly'
+                        });
+                    }).catch(error => {
+                        let msg = (error.response?.status + error.response?.data) || error.message
+                        resolve({
+                            index, success: false, message: `Error from SPARQL end-point: ${msg}`
+                        });
+                    });
+                }catch(e){
+                    resolve({
+                        index, success: false, message: `Error: ${e}`
+                    });
+                }
+            })
+        }
+        setTimeout(async ()=>{
+            for(let index=0; index<queries.length; index++){
+                results[index] = await fusekiCall(index);
+            }
+            checkCompleted();
         })
-        .then(response => {
-            res.json({
-                message: 'File correctly uploaded and vocabularies updated: ' + response.data,
-                files: uploadedFiles
-            });
-        }).catch(error => {
-            let message = (error.response?.status + error.response?.data) || error.message
-            console.log(message);
-            //fs.writeFileSync('/home/nicole/Scaricati/spqr_error.txt', query);   
-            res.status(500).json({ 
-                message: 'Error from SPARQL end-point: ' + message, 
-                files: uploadedFiles,
-                query
-            });
-        });
-        
     }).catch(err => {
         deleteFiles(uploadedFiles)
         console.error('Error transforming XML:', err);

package/src/controllers/ontology.js

@@ -7,6 +7,12 @@ const Converter = require('../models/converter');
 const Validator = require('../models/validator');
 const tmp = require('tmp');
 const Investigations = require('../models/investigations');
+const {
+    fusekiUrlDataset,
+    fusekiUrl,
+    fusekiUrlUpdate
+} = require('../models/fusekiConfig');
+const axios = require('axios');
 
 let SCHEMAS = {}
 
@@ -112,19 +118,43 @@ router.post('/validate', (req, res) => {
 router.post('/form/save', (req, res) => {
     let dataset = req.body.turtle;
     let uuid = req.body.uuid;
-    Investigations.save({
-        uuid, dataset, format: 'turtle'
-    }).then( () => {
-        res.json({
-            success: true
-        });
-    }).catch( (err) => {
-        console.log("Error saving investigation: ", err);
+    try{
+        let updateQuery = Converter.turtle2Sparql(dataset);
+        Investigations.save({
+            uuid, dataset, format: 'turtle'
+        }).then( () => {
+            axios.post(fusekiUrlUpdate, updateQuery, {
+                headers: {
+                    'Content-Type': 'application/sparql-update',
+                    'Accept': 'application/sparql-results+json'
+                }
+            }).then(response => {
+                console.log(response.data);
+                res.status(200).json({
+                    success: true
+                });
+            }).catch(error => {
+                //TODO: rollback investigation save
+                let message = (error.response?.status + error.response?.data) || error.message
+                res.status(500).json({ 
+                    message: 'Error from SPARQL end-point: ' + message, 
+                    success: false
+                });
+            });                
+        }).catch( (err) => {
+            console.log("Error saving investigation: ", err);
+            res.json({
+                success: false,
+                message: `Error: ${err}`
+            });
+        }); 
+    }catch(e){
         res.json({
             success: false,
-            message: `Error: ${err}`
+            message: `Error: ${e.message}`
         });
-    });    
+        return;
+    }       
 });
 
 router.get('/form/:uuid', (req, res) => {

package/src/models/converter.js

@@ -17,6 +17,83 @@ class Converter {
         })
     }
 
+    static termToSparql(term) {
+        if (!term) return '';
+        const t = term.termType || term.type; // some versions use .type
+        const value = term.value;
+        if (t === 'NamedNode' || t === 'IRI') {
+            return `<${value}>`;
+        }
+        if (t === 'BlankNode' || t === 'Blank') {
+            return `_:${value}`;
+        }
+        if (t === 'Literal' || t === 'literal') {
+            // escape per basic N-Triples rules
+            const esc = value
+                .replace(/\\/g, '\\\\')
+                .replace(/"/g, '\\"')
+                .replace(/\r/g, '\\r')
+                .replace(/\n/g, '\\n');
+            const lang = term.language;
+            const dt = term.datatype && term.datatype.value;
+            if (lang) return `"${esc}"@${lang}`;
+            if (dt && dt !== 'http://www.w3.org/2001/XMLSchema#string') 
+                return `"${esc}"^^<${dt}>`;
+            return `"${esc}"`;
+        }
+        return `${value}`;
+    }
+
+    static turtle2Sparql(turtle, opts={}){
+        const graph = opts.graph || null; // if null -> default graph
+        const parser = new Parser();
+        const quads = parser.parse(turtle);
+        const termToSparql = Converter.termToSparql;
+         // group objects by subject+predicate
+        const groups = new Map(); // key -> { subj, pred, objects: Set() }
+        for (const q of quads) {
+            const s = q.subject;
+            const p = q.predicate;
+            const o = q.object;
+            const key = `${termToSparql(s)} ${termToSparql(p)}`;
+            if (!groups.has(key)) groups.set(key, { subj: s, pred: p, objects: new Set() });
+            groups.get(key).objects.add(termToSparql(o));
+        }
+
+        // build SPARQL update parts
+        const parts = [];
+        for (const [, { subj, pred, objects }] of groups) {
+            const sStr = termToSparql(subj);
+            const pStr = termToSparql(pred);
+
+            // DELETE WHERE: remove any existing object for the subject/predicate
+            // we use a variable ?o to delete any existing triples with same s,p
+            let deleteBlock;
+            if (graph) {
+                deleteBlock = `DELETE WHERE { GRAPH <${graph}> { ${sStr} ${pStr} ?o } };`;
+            } else {
+                deleteBlock = `DELETE WHERE { ${sStr} ${pStr} ?o } ;`;
+            }
+
+            // INSERT DATA: insert the objects we parsed. If multiple objects, join with comma.
+            const objs = Array.from(objects);
+            const objectsList = objs.join(' ,\n      '); // pretty print
+            let insertBlock;
+            if (graph) {
+                insertBlock = `INSERT DATA { GRAPH <${graph}> { ${sStr} ${pStr} ${objectsList} . } };`;
+            } else {
+                insertBlock = `INSERT DATA { ${sStr} ${pStr} ${objectsList} . } ;`;
+            }
+
+            // append as one atomic unit (delete then insert)
+            parts.push(`${deleteBlock}\n${insertBlock}`);
+        }
+
+        // join with double newline for readability
+        return parts.join('\n\n');
+        
+    }
+
     static async turtle2RdfXml(inTurtlePath, outRdfXmlPath) {
         return new Promise((resolve, reject) => {
             const command = `rapper -i turtle -o rdfxml "${inTurtlePath}" > "${outRdfXmlPath}"`; 

package/src/models/fusekiConfig.js

@@ -0,0 +1,16 @@
+const config = require('../config')
+const configFuseki = config.fuseki || {
+    "protocol": "http",
+    "host": "127.0.0.1",
+    "port": "3030",
+    "dataset": "oac"
+}
+const fusekiUrlDataset = `${configFuseki.protocol}://${configFuseki.host}:${configFuseki.port}/${configFuseki.dataset}`;
+const fusekiUrl = `${fusekiUrlDataset}/sparql`;
+const fusekiUrlUpdate = `${fusekiUrlDataset}/update`;
+
+module.exports = {
+    fusekiUrlDataset,
+    fusekiUrl,
+    fusekiUrlUpdate
+}

package/src/models/vocabolaries/parser

@@ -5,7 +5,7 @@ const { exec } = require('child_process');
 const stripBom = require('strip-bom').default;
 const config = require('../../config');
 const VocabPrefix = config.fuseki.vocabularies.prefix || 'diagnostica';
-
+const Splitter = require('./splitter');
 
 const transformMode = {
     default: 'default',
@@ -75,46 +75,73 @@ class Parser{
             }catch(e){
                 reject(e)        
             }
-            
-            /*    
-            let xmlData = this._getXmlData(xmlPath);
-            const xml = xmlParser.xmlParse(xmlData);
-            xslt.xsltProcess(xml, this.xslt)
-            .then(result => {
-                var terms = result.split('
')
-                resolve(terms.map(line => line.trim()).filter(line => line.length > 0));
-            }).catch(err => {
-                reject(err)
-            });
-            */
         });
     }
 
     insertQuery(xmlPath, className='crm:E55_Type'){
         return new Promise((resolve, reject) => {
-            this.transform(xmlPath, transformMode.forInsert).then(terms => {
-                let query = `
-                PREFIX crm: <http://www.cidoc-crm.org/cidoc-crm/>
-                PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+            const xmlFolder = path.dirname(xmlPath);
+            const xmlFile = path.basename(xmlPath);
+            const splitter = new Splitter(xmlFolder, xmlFile);
+            const files = splitter.splitFiles();
+            let queries = new Array(files.length);
+            let cleanTempFiles = function(queries){
+                for(let file of files){
+                    try{
+                        fs.unlinkSync(file.path);
+                    }catch(e){
+                        //console.log(`Error deleting temp file ${file.path}: ${e.message}`);
+                    }
+                }
+                resolve(queries);
+            }
+            let checkCompleted = function(index, query, status, message){
+                console.log(`Completed ${index+1}/${files.length}`);
+                files[index].status = status;
+                files[index].message = message;
+                if(status) queries[index]=query;
+                let allDone = files.every(f => f.status !== null);
+                if(allDone){
+                    cleanTempFiles(queries);
+                }
+            }
+            let transformFn = function(instance, index, className, xmlPath){
+                return function(){
+                    instance.transform(xmlPath, transformMode.forInsert).then(terms => {
+                        let query = `
+                        PREFIX crm:     <http://www.cidoc-crm.org/cidoc-crm/> 
+                        PREFIX basecpm: <http://ontome.net/ns/cpm/> 
+                        PREFIX j.0:     <http://www.cidoc-crm.org/extensions/crmsci/> 
+                        PREFIX crmsci:  <http://www.ics.forth.gr/isl/CRMsci> 
+                        PREFIX base:    <http://www.ics.forth.gr/isl/CRMinf/> 
+                        PREFIX rdfs:    <http://www.w3.org/2000/01/rdf-schema#>
 
-                INSERT {
-                    ?term a ${className} ;
-                    rdfs:label ?label ;
-                    crm:P127_has_broader_term ?broader .
+                        INSERT {
+                            ?term a ${className} ;
+                            rdfs:label ?label ;
+                            crm:P127_has_broader_term ?broader .
+                        }
+                        WHERE {
+                            VALUES (?term ?label ?broader) {
+                                ${terms.join(' \n')}
+                            }
+                            FILTER NOT EXISTS {
+                                ?term rdfs:label ?label .
+                            }
+                        }`
+                        checkCompleted(index, query, true, null);
+                    }).catch(err => {
+                        console.log(err);
+                        checkCompleted(index, null, false, `${err}`);
+                    });
                 }
-                WHERE {
-                    VALUES (?term ?label ?broader) {
-                        ${terms.join(' \n')}
-                    }
-                    FILTER NOT EXISTS {
-                        ?term rdfs:label ?label .
-                    }
-                }`
-                resolve(query);
-            }).catch(err => {
-                console.log(err);
-                reject(err);
-            });
+            }
+
+            for(let index=0; index<files.length; index++){
+                let fn = transformFn(this, index, files[index].class, files[index].path);
+                fn();
+            }
+            
         })
     }
 

package/src/models/vocabolaries/splitter.js

@@ -0,0 +1,69 @@
+const fs = require('fs');
+const path = require('path');
+const { XMLParser, XMLBuilder } = require('fast-xml-parser');
+
+class Splitter {
+
+    constructor(xmlFolder, xmlName, skipWrite=false) {
+        this.xmlFolder = xmlFolder;
+        this.xmlName = xmlName;
+        this.skipWrite = skipWrite;
+        const xmlPath = path.join(this.xmlFolder, this.xmlName);
+        this.xml = fs.readFileSync(xmlPath, 'utf8');
+        // Parser
+        this.parser = new XMLParser({
+            ignoreAttributes: false,
+            attributeNamePrefix: '@_'
+        });
+    }
+
+    splitByClass() {
+        const data = this.parser.parse(this.xml);
+        const vocabularies = data.vocabularies.vocabulary;
+
+        // Normalizza a array
+        const vocabArray = Array.isArray(vocabularies)
+            ? vocabularies
+            : [vocabularies];
+
+        // Raggruppa per class
+        const grouped = {};
+        for (const vocab of vocabArray) {
+            const cls = vocab['@_class'] || 'NO_CLASS';
+            if (!grouped[cls]) grouped[cls] = [];
+            grouped[cls].push(vocab);
+        }
+
+        return grouped;
+    }
+
+    splitFiles() {
+        const grouped = this.splitByClass();
+        const builder = new XMLBuilder({
+            ignoreAttributes: false,
+            attributeNamePrefix: '@_',
+            format: true,
+            indentBy: '  '
+        });
+        let index = 0
+        let results = [];
+        for (const cls in grouped) {
+            const outObj = {
+                vocabularies: {
+                    vocabulary: grouped[cls]
+                }
+            };
+            index++;
+            const xmlOut = builder.build(outObj);
+            const xmlOutPath = path.join(this.xmlFolder, `${this.xmlName}_split_${index}.xml`);    
+            if(!this.skipWrite)
+                fs.writeFileSync(xmlOutPath, xmlOut, 'utf8');
+            results.push({ class: cls, path: xmlOutPath, status:null, message:null });
+        }
+        return results.sort((a, b) => a.class.localeCompare(b.class));
+    }
+
+}
+
+
+module.exports = Splitter;

package/test/models/converter.test.js

@@ -27,4 +27,13 @@ describe('Converter', () => {
     });
   });
 
+it('should convert a turtle string to sparql UPSERT', (done) => {
+    let inputTurtle = fs.readFileSync(__dirname + '/example-investigation-01.ttl', 'utf8');
+    let sparql = converter.turtle2Sparql(inputTurtle)
+    expect(sparql).to.be.a('string');
+    expect(sparql.length).to.be.greaterThan(0);
+    console.log(sparql);
+    done();
+  });
+
 })

package/test/models/example-investigation-01.ttl

@@ -0,0 +1,32 @@
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix ex: <http://example.org/shapes/>.
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix owl: <http://www.w3.org/2002/07/owl#>.
+@prefix crm: <http://www.cidoc-crm.org/cidoc-crm/>.
+@prefix basecpm: <http://ontome.net/ns/cpm/>.
+@prefix base: <http://www.ics.forth.gr/isl/CRMinf/>.
+@prefix cpm: <http://ontome.net/ns/cpm/>.
+@prefix crmsci: <http://www.cidoc-crm.org/extensions/crmsci/>.
+@prefix pref: <http://diagnostica/>.
+@prefix skos: <http://www.w3.org/2004/02/skos/core#>.
+
+pref:3cc24c8c-ce06-40ef-bdce-9da2c4c5394b ex:P48haspreferredidentifier01 <http://indagine/3>;
+    a crm:E42_Identifier.
+pref:1adba9d5-c83b-41a8-b97f-c6e5b168658a a crm:E7_Activity;
+    crm:P48_has_preferred_identifier pref:3cc24c8c-ce06-40ef-bdce-9da2c4c5394b;
+    crm:P17_was_motivated_by pref:f30ead0e-edcc-4f6f-868e-8bc6f9d13813;
+    crm:P14_carried_out_by pref:57701c6c-ece4-472b-bc24-b75c1c254568;
+    <http://purl.org/dc/terms/conformsTo> ex:E7ActivityShape.
+pref:40a75ae6-daa4-4ae1-bd41-2e07175f6f75 a crm:E55_Type;
+    ex:P2hastype02 <http://diagnostica/vocabularies/quesito-diagnostico/tecnologia-di-produzione>.
+pref:f30ead0e-edcc-4f6f-868e-8bc6f9d13813 a base:I12_Adopted_Belief;
+    crm:P2_has_type pref:40a75ae6-daa4-4ae1-bd41-2e07175f6f75.
+pref:95e957ad-55b5-45d3-bf62-6a292959c4f9 a crm:E41_Appellation;
+    ex:ente_richiedente "Università";
+    ex:schedatore "Christian".
+pref:956f0c5a-6fc5-4a79-b399-6c6288fb000f a crm:E55_Type;
+    crm:P1_is_defined_by pref:95e957ad-55b5-45d3-bf62-6a292959c4f9.
+pref:57701c6c-ece4-472b-bc24-b75c1c254568 a crm:E29_Actor;
+    crm:P2_has_type pref:956f0c5a-6fc5-4a79-b399-6c6288fb000f.

package/test/models/vocabolaries/parser.test.js

@@ -44,7 +44,10 @@ describe('Vocabolaries.Parsers', () => {
   it('should get the insert query for the vocabolaries.xml file', async () => {
     const parser = Parser.GET_INSTANCE();
     var query = await parser.insertQuery(__dirname + '/vocabolaries.xml');
-    expect(query.length).to.be.equal(40619);
+    expect(query.length).to.be.equal(3);
+    expect(query[0].length).to.be.equal(5279);
+    expect(query[1].length).to.be.equal(2665);
+    expect(query[2].length).to.be.equal(35150);
   });
 
 });

package/test/models/vocabolaries/splitter.test.js

@@ -0,0 +1,36 @@
+const chai = require('chai');
+const expect = chai.expect;
+const request = require('supertest');
+const Splitter = require('../../../src/models/vocabolaries/splitter');
+
+
+describe('Vocabolaries.Splitter', () => {
+  
+  beforeEach(() => {
+    
+  });
+
+  it('should create a parser instance', () => {
+    const splitter = new Splitter(__dirname, 'vocabolaries_2.xml');
+    expect(splitter).to.be.an.instanceof(Splitter);
+  });
+
+  it('should get group of classes', () => {
+    const splitter = new Splitter(__dirname, 'vocabolaries_2.xml');
+    const groups = splitter.splitByClass();
+    expect(Object.keys(groups).length).to.be.equal(6);
+  });
+
+  it('should get group of classes', () => {
+    const splitter = new Splitter(__dirname, 'vocabolaries_2.xml', true);
+    const files = splitter.splitFiles();
+    expect(files.length).to.be.equal(6);
+    expect(files[0].class).to.be.equal("base:I2_Belief");
+    expect(files[1].class).to.be.equal("basecpm:CP42_Material_Decay");
+    expect(files[2].class).to.be.equal("basecpm:CP43_Structural_Damage");
+    expect(files[3].class).to.be.equal("crm:E29_Design_or_Procedure");
+    expect(files[4].class).to.be.equal("crm:E55_Type");
+    expect(files[5].class).to.be.equal("crm:E58_Measurement_Unit");
+  });
+  
+});