cassproject 5.0.7 → 5.0.8

package/README.md

@@ -50,6 +50,10 @@ Development unit tests presume you have a CaSS Repository running on `localhost:
 
 # Changelog
 
+## 5.0.8
+* Security updates
+* CTDL-ASN import fixes
+
 ## 5.0.7
 * Security updates
 * Assertions with the same registration will no longer propagate to the same inferred assertions.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cassproject",
-	"version": "5.0.7",
+	"version": "5.0.8",
 	"description": "Competency and Skills Service",
 	"main": "index.js",
 	"scripts": {
@@ -193,18 +193,18 @@
 		"chai": "4.5.0",
 		"concurrently": "^9.2.1",
 		"convert-hrtime": "^5.0.0",
-		"cypress": "^15.7.1",
+		"cypress": "^15.8.1",
 		"cypress-fail-fast": "^7.1.1",
-		"eslint": "^9.39.1",
+		"eslint": "^9.39.2",
 		"fake-indexeddb": "^6.2.5",
 		"mocha": "^11.7.5",
 		"node-polyfill-webpack-plugin": "^4.1.0",
 		"nodemon": "^3.1.11",
 		"nyc": "^17.1.0",
-		"sinon": "^21.0.0",
+		"sinon": "^21.0.1",
 		"url-polyfill": "^1.1.14",
 		"wait-on": "^9.0.3",
-		"webpack": "^5.103.0",
+		"webpack": "^5.104.1",
 		"webpack-cli": "^6.0.1"
 	},
 	"packageManager": "npm@11.3.0+sha512.96eb611483f49c55f7fa74df61b588de9e213f80a256728e6798ddc67176c7b07e4a1cfc7de8922422cbce02543714367037536955221fa451b0c4fefaf20c66"

package/src/org/cass/importer/CTDLASNCSVConceptImport.js

@@ -131,6 +131,7 @@ module.exports = class CTDLASNCSVConceptImport {
 			encoding: "UTF-8",
 			complete: async function(results) {
 				let tabularData = results["data"];
+				let errors = []; // Collect all errors to display at once
 				try {
 					for (let data of tabularData) {
 						for (let [key, value] of Object.entries(data)) {
@@ -148,30 +149,37 @@ module.exports = class CTDLASNCSVConceptImport {
 						validationRules.hierarchyRules
 					);
 					if (hierarchyError) {
-						failure(hierarchyError);
-						return;
+						errors.push(hierarchyError);
 					}
 				}
 				const terms = JSON.parse(JSON.stringify((await EcRemote.getExpectingObject("https://schema.cassproject.org/0.4/jsonld1.1/ceasn2cassConceptsTerms"))));
 				let schemeArray = [];
 				let concepts = [];
+				let rowsWithErrors = new Set(); // Track rows that have errors
 				for (let each = 0; each < tabularData.length; each++) {
 					let pretranslatedE = tabularData[each];
 					// Skip extra lines if found in file
 					if (!pretranslatedE) {
 						continue;
 					}
+
+					// Validate basic required fields and collect errors
+					let rowHasError = false;
 					if (!pretranslatedE["@id"]) {
-						failure(`Row ${each + 2}: is missing an @id`);
-						return;
+						errors.push(`Row ${each + 2}: is missing an @id`);
+						rowHasError = true;
 					}
 					if (!pretranslatedE["@type"]) {
-						failure(`Row ${each + 2}: is missing a @type`);
-						return;
+						errors.push(`Row ${each + 2}: is missing a @type`);
+						rowHasError = true;
 					}
-					if (!pretranslatedE["@id"].startsWith('http') && !pretranslatedE["@id"].startsWith('ce-')) {
-						failure(`row ${each + 2}: @id must be a valid URI or start with 'ce-'`)
-						return;
+					if (pretranslatedE["@id"] && !pretranslatedE["@id"].startsWith('http') && !pretranslatedE["@id"].startsWith('ce-')) {
+						errors.push(`Row ${each + 2}: @id must be a valid URI or start with 'ce-'`);
+						rowHasError = true;
+					}
+					if (rowHasError) {
+						rowsWithErrors.add(each);
+						continue;
 					}
 					if (
 						pretranslatedE["@type"].toLowerCase().startsWith('sample') || pretranslatedE["@type"].toLowerCase().startsWith('instruction')
@@ -181,15 +189,20 @@ module.exports = class CTDLASNCSVConceptImport {
 					if (pretranslatedE["@type"] == "skos:ConceptScheme") {
 						// Validate required properties (only if validation rules provided)
 						if (validationRules && validationRules.requiredProps) {
-							const validationError = CTDLASNCSVConceptImport.validateRequiredProperties(
+							const validationErrors = CTDLASNCSVConceptImport.validateRequiredProperties(
 								pretranslatedE,
 								"skos:ConceptScheme",
 								each + 2,
 								validationRules.requiredProps
 							);
-							if (validationError) {
-								failure(validationError);
-								return;
+							if (validationErrors) {
+								if (Array.isArray(validationErrors)) {
+									errors.push(...validationErrors);
+								} else {
+									errors.push(validationErrors);
+								}
+								rowsWithErrors.add(each);
+								continue;
 							}
 						}
 						let translator = new EcLinkedData(null, null);
@@ -263,15 +276,20 @@ module.exports = class CTDLASNCSVConceptImport {
 					} else if (pretranslatedE["@type"] == "skos:Concept") {
 						// Validate required properties (only if validation rules provided)
 						if (validationRules && validationRules.requiredProps) {
-							const validationError = CTDLASNCSVConceptImport.validateRequiredProperties(
+							const validationErrors = CTDLASNCSVConceptImport.validateRequiredProperties(
 								pretranslatedE,
 								"skos:Concept",
 								each + 2,
 								validationRules.requiredProps
 							);
-							if (validationError) {
-								failure(validationError);
-								return;
+							if (validationErrors) {
+								if (Array.isArray(validationErrors)) {
+									errors.push(...validationErrors);
+								} else {
+									errors.push(validationErrors);
+								}
+								rowsWithErrors.add(each);
+								continue;
 							}
 						}
 						let translator = new EcLinkedData(null, null);
@@ -302,8 +320,9 @@ module.exports = class CTDLASNCSVConceptImport {
 						let f = new EcConcept();
 						f.copyFrom(e);
 						if (e["id"] == null) {
-							failure(`Row ${each + 2}: Concept is missing an id`);
-							return;
+							errors.push(`Row ${each + 2}: Concept is missing an id`);
+							rowsWithErrors.add(each);
+							continue;
 						}
 						if (
 							EcConcept.template != null &&
@@ -446,15 +465,22 @@ module.exports = class CTDLASNCSVConceptImport {
 						pretranslatedE["@type"] == null ||
 						pretranslatedE["@type"] == ""
 					) {
-						failure(`Row ${each + 2}: Missing or empty @type`);
-						return;
+						errors.push(`Row ${each + 2}: Missing or empty @type`);
+						rowsWithErrors.add(each);
+						continue;
 					} else {
-						failure(
+						errors.push(
 							`Row ${each + 2}: Found unknown type: ` + pretranslatedE["@type"]
 						);
-						return;
+						rowsWithErrors.add(each);
+						continue;
 					}
 				}
+				// If there are any errors, report them all at once
+				if (errors.length > 0) {
+					failure(errors);
+					return;
+				}
 				success(schemeArray, concepts);
 			},
 			error: failure
@@ -475,6 +501,7 @@ module.exports = class CTDLASNCSVConceptImport {
 			encoding: "UTF-8",
 			complete: async function(results) {
 				let tabularData = results["data"];
+				let errors = []; // Collect all errors to display at once
 				try {
 					for (let data of tabularData) {
 						for (let [key, value] of Object.entries(data)) {
@@ -492,26 +519,33 @@ module.exports = class CTDLASNCSVConceptImport {
 						validationRules.hierarchyRules
 					);
 					if (hierarchyError) {
-						failure(hierarchyError);
-						return;
+						errors.push(hierarchyError);
 					}
 				}
 				const terms = JSON.parse(JSON.stringify((await EcRemote.getExpectingObject("https://schema.cassproject.org/0.4/jsonld1.1/ceasn2cassConceptsTerms"))));
 				let schemeArray = [];
 				let concepts = [];
+				let rowsWithErrors = new Set(); // Track rows that have errors
 				for (let each = 0; each < tabularData.length; each++) {
 					let pretranslatedE = tabularData[each];
 					// Skip extra lines if found in file
 					if (!pretranslatedE) {
 						continue;
 					}
+
+					// Validate basic required fields and collect errors
+					let rowHasError = false;
 					if (!pretranslatedE["@id"]) {
-						failure(`Row ${each + 2}: is missing an @id`);
-						return;
+						errors.push(`Row ${each + 2}: is missing an @id`);
+						rowHasError = true;
 					}
 					if (!pretranslatedE["@type"]) {
-						failure(`Row ${each + 2}: is missing a @type`);
-						return;
+						errors.push(`Row ${each + 2}: is missing a @type`);
+						rowHasError = true;
+					}
+					if (rowHasError) {
+						rowsWithErrors.add(each);
+						continue;
 					}
 					if (
 						pretranslatedE["@type"].toLowerCase().startsWith('sample') || pretranslatedE["@type"].toLowerCase().startsWith('instruction')
@@ -521,15 +555,20 @@ module.exports = class CTDLASNCSVConceptImport {
 					if (pretranslatedE["@type"] == "asn:ProgressionModel") {
 						// Validate required properties (only if validation rules provided)
 						if (validationRules && validationRules.requiredProps) {
-							const validationError = CTDLASNCSVConceptImport.validateRequiredProperties(
+							const validationErrors = CTDLASNCSVConceptImport.validateRequiredProperties(
 								pretranslatedE,
 								"asn:ProgressionModel",
 								each + 2,
 								validationRules.requiredProps
 							);
-							if (validationError) {
-								failure(validationError);
-								return;
+							if (validationErrors) {
+								if (Array.isArray(validationErrors)) {
+									errors.push(...validationErrors);
+								} else {
+									errors.push(validationErrors);
+								}
+								rowsWithErrors.add(each);
+								continue;
 							}
 						}
 						let translator = new EcLinkedData(null, null);
@@ -599,15 +638,20 @@ module.exports = class CTDLASNCSVConceptImport {
 					} else if (pretranslatedE["@type"] == "asn:ProgressionLevel") {
 						// Validate required properties (only if validation rules provided)
 						if (validationRules && validationRules.requiredProps) {
-							const validationError = CTDLASNCSVConceptImport.validateRequiredProperties(
+							const validationErrors = CTDLASNCSVConceptImport.validateRequiredProperties(
 								pretranslatedE,
 								"asn:ProgressionLevel",
 								each + 2,
 								validationRules.requiredProps
 							);
-							if (validationError) {
-								failure(validationError);
-								return;
+							if (validationErrors) {
+								if (Array.isArray(validationErrors)) {
+									errors.push(...validationErrors);
+								} else {
+									errors.push(validationErrors);
+								}
+								rowsWithErrors.add(each);
+								continue;
 							}
 						}
 						let translator = new EcLinkedData(null, null);
@@ -638,8 +682,9 @@ module.exports = class CTDLASNCSVConceptImport {
 						let f = new EcConcept();
 						f.copyFrom(e);
 						if (e["id"] == null) {
-							failure(`Row ${each + 2}: Concept is missing an id`);
-							return;
+							errors.push(`Row ${each + 2}: Concept is missing an id`);
+							rowsWithErrors.add(each);
+							continue;
 						}
 						if (
 							EcConcept.template != null &&
@@ -747,15 +792,22 @@ module.exports = class CTDLASNCSVConceptImport {
 						pretranslatedE["@type"] == null ||
 						pretranslatedE["@type"] == ""
 					) {
-						failure(`Row ${each + 2}: Missing or empty @type`);
-						return;
+						errors.push(`Row ${each + 2}: Missing or empty @type`);
+						rowsWithErrors.add(each);
+						continue;
 					} else {
-						failure(
+						errors.push(
 							`Row ${each + 2}: Found unknown type: ` + pretranslatedE["@type"]
 						);
-						return;
+						rowsWithErrors.add(each);
+						continue;
 					}
 				}
+				// If there are any errors, report them all at once
+				if (errors.length > 0) {
+					failure(errors);
+					return;
+				}
 				success(schemeArray, concepts);
 			},
 			error: failure

package/src/org/cass/importer/CTDLASNCSVImport.js

@@ -155,6 +155,7 @@ module.exports = class CTDLASNCSVImport {
 			encoding: "UTF-8",
 			complete: async function(results) {
 				let tabularData = results["data"];
+				let errors = []; // Collect all errors to display at once
 				try {
 					for (let data of tabularData) {
 						for (let [key, value] of Object.entries(data)) {
@@ -172,8 +173,7 @@ module.exports = class CTDLASNCSVImport {
 						validationRules.hierarchyRules
 					);
 					if (hierarchyError) {
-						failure(hierarchyError);
-						return;
+						errors.push(hierarchyError);
 					}
 				}
 
@@ -185,6 +185,7 @@ module.exports = class CTDLASNCSVImport {
 				let competencyRows = {};
 				let relations = [];
 				let relationById = {};
+				let rowsWithErrors = new Set(); // Track rows that have errors to avoid duplicate processing
 				for (let i = 0; i < tabularData.length; i++) {
 					if (!tabularData[i]) {
 						continue;
@@ -194,7 +195,7 @@ module.exports = class CTDLASNCSVImport {
 					if (!pretranslatedE) {
 						continue;
 					}
-					if (pretranslatedE["@type"].toLowerCase() && 
+					if (pretranslatedE["@type"] && pretranslatedE["@type"].toLowerCase() &&
 						(pretranslatedE["@type"].toLowerCase().startsWith('sample') || pretranslatedE["@type"].toLowerCase().startsWith('instruction'))
 					) {
 						continue;
@@ -211,17 +212,24 @@ module.exports = class CTDLASNCSVImport {
 							continue;
 						}
 					}
+
+					// Validate basic required fields and collect errors
+					let rowHasError = false;
 					if (!pretranslatedE["@id"]) {
-						failure(`Row ${i + 2}: is missing an @id`)
-						return;
+						errors.push(`Row ${i + 2}: is missing an @id`);
+						rowHasError = true;
 					}
 					if (!pretranslatedE["@type"]) {
-						failure(`Row ${i + 2}: is missing a @type`);
-						return;
+						errors.push(`Row ${i + 2}: is missing a @type`);
+						rowHasError = true;
+					}
+					if (pretranslatedE["@id"] && !pretranslatedE["@id"].startsWith('http') && !pretranslatedE["@id"].startsWith('ce-')) {
+						errors.push(`Row ${i + 2}: @id must be a valid URI or start with 'ce-'`);
+						rowHasError = true;
 					}
-					if (!pretranslatedE["@id"].startsWith('http') && !pretranslatedE["@id"].startsWith('ce-')) {
-						failure(`row ${i + 2}: @id must be a valid URI or start with 'ce-'`)
-						return;
+					if (rowHasError) {
+						rowsWithErrors.add(i);
+						continue;
 					}
 					if (
 						pretranslatedE["@type"] ==
@@ -229,15 +237,20 @@ module.exports = class CTDLASNCSVImport {
 					) {
 						// Validate required properties (only if validation rules provided)
 						if (validationRules && validationRules.requiredProps) {
-							const validationError = CTDLASNCSVImport.validateRequiredProperties(
+							const validationErrors = CTDLASNCSVImport.validateRequiredProperties(
 								pretranslatedE,
 								"ceasn:CompetencyFramework",
 								i + 2,
 								validationRules.requiredProps
 							);
-							if (validationError) {
-								failure(validationError);
-								return;
+							if (validationErrors) {
+								if (Array.isArray(validationErrors)) {
+									errors.push(...validationErrors);
+								} else {
+									errors.push(validationErrors);
+								}
+								rowsWithErrors.add(i);
+								continue;
 							}
 						}
 
@@ -251,7 +264,9 @@ module.exports = class CTDLASNCSVImport {
 						try {
 							let existing = await EcRepository.get(translator.id);
 							if (existing && existing.type !== 'Framework') {
-								return failure(`Row ${i + 2}: ${translator.id} already exists as a ${existing.type}`);
+								errors.push(`Row ${i + 2}: ${translator.id} already exists as a ${existing.type}`);
+								rowsWithErrors.add(i);
+								continue;
 							}
 						} catch (e) {
 							console.error(e);
@@ -371,15 +386,20 @@ module.exports = class CTDLASNCSVImport {
 					) {
 					// Validate required properties (only if validation rules provided)
 					if (validationRules && validationRules.requiredProps) {
-						const validationError = CTDLASNCSVImport.validateRequiredProperties(
+						const validationErrors = CTDLASNCSVImport.validateRequiredProperties(
 							pretranslatedE,
 							"ceasn:Competency",
 							i + 2,
 							validationRules.requiredProps
 						);
-						if (validationError) {
-							failure(validationError);
-							return;
+						if (validationErrors) {
+							if (Array.isArray(validationErrors)) {
+								errors.push(...validationErrors);
+							} else {
+								errors.push(validationErrors);
+							}
+							rowsWithErrors.add(i);
+							continue;
 						}
 					}
 
@@ -418,8 +438,9 @@ module.exports = class CTDLASNCSVImport {
 						let f = new EcCompetency();
 						f.copyFrom(e);
 						if (e["id"] == null) {
-							failure(`Row ${i+2}: Competency is missing an id`);
-							return;
+							errors.push(`Row ${i+2}: Competency is missing an id`);
+							rowsWithErrors.add(i);
+							continue;
 						}
 						if (e["ceasn:isPartOf"] != null) {
 							let shortId = EcRemoteLinkedData.trimVersionFromUrl(e["ceasn:isPartOf"]);
@@ -461,10 +482,11 @@ module.exports = class CTDLASNCSVImport {
 								}
 							}
 							if (!done) {
-								failure(
+								errors.push(
 									`Row ${i+2}: Could not find framework:${e["type"]} (Possibly missing ceasn:isPartOf or ceasn:isChildOf)`
 								);
-								return;
+								rowsWithErrors.add(i);
+								continue;
 							}
 							if (parent != null) {
 								if (parent["type"] == "Framework") {
@@ -479,18 +501,20 @@ module.exports = class CTDLASNCSVImport {
 										)
 									].competency.push(f.shortId());
 								} else {
-									failure(
+									errors.push(
 										`Row ${i+2}: Object cannot trace to framework:` +
 											e["type"]
 									);
-									return;
+									rowsWithErrors.add(i);
+									continue;
 								}
 							} else {
-								failure(
+								errors.push(
 									`Row ${i+2}: Object has no framework:` +
 										e["type"]
 								);
-								return;
+								rowsWithErrors.add(i);
+								continue;
 							}
 						}
 						if (
@@ -682,15 +706,22 @@ module.exports = class CTDLASNCSVImport {
 						pretranslatedE["@type"] == null ||
 						pretranslatedE["@type"] == ""
 					) {
-						failure(`Row ${i+2}: Missing or empty @type`);
-						return;
+						errors.push(`Row ${i+2}: Missing or empty @type`);
+						rowsWithErrors.add(i);
+						continue;
 					} else {
-						failure(
+						errors.push(
 							`Row ${i+2}: Found unknown type:` + pretranslatedE["@type"]
 						);
-						return;
+						rowsWithErrors.add(i);
+						continue;
 					}
 				}
+				// If there are any errors, report them all at once
+				if (errors.length > 0) {
+					failure(errors);
+					return;
+				}
 				success(frameworkArray, competencies, relations);
 			},
 			error: failure
@@ -712,6 +743,7 @@ module.exports = class CTDLASNCSVImport {
 			encoding: "UTF-8",
 			complete: async function(results) {
 				let tabularData = results["data"];
+				let errors = []; // Collect all errors to display at once
 				try {
 					for (let data of tabularData) {
 						for (let [key, value] of Object.entries(data)) {
@@ -729,8 +761,7 @@ module.exports = class CTDLASNCSVImport {
 						validationRules.hierarchyRules
 					);
 					if (hierarchyError) {
-						failure(hierarchyError);
-						return;
+						errors.push(hierarchyError);
 					}
 				}
 
@@ -742,23 +773,31 @@ module.exports = class CTDLASNCSVImport {
 				let competencyRows = {};
 				let relations = [];
 				let relationById = {};
+				let rowsWithErrors = new Set(); // Track rows that have errors
 				for (let i = 0; i < tabularData.length; i++) {
 					let pretranslatedE = tabularData[i];
 					// Skip extra lines if found in file
 					if (!pretranslatedE) {
 						continue;
 					}
+
+					// Validate basic required fields and collect errors
+					let rowHasError = false;
 					if (!pretranslatedE["@id"]) {
-						failure(`Row ${i + 2}: is missing an @id`);
-						return;
+						errors.push(`Row ${i + 2}: is missing an @id`);
+						rowHasError = true;
 					}
-					if (!pretranslatedE["@id"].startsWith('http') && !pretranslatedE["@id"].startsWith('ce-')) {
-						failure(`row ${i + 2}: @id must be a valid URI or start with 'ce-'`)
-						return;
+					if (pretranslatedE["@id"] && !pretranslatedE["@id"].startsWith('http') && !pretranslatedE["@id"].startsWith('ce-')) {
+						errors.push(`Row ${i + 2}: @id must be a valid URI or start with 'ce-'`);
+						rowHasError = true;
 					}
 					if (!pretranslatedE["@type"]) {
-						failure(`Row ${i + 2}: is missing a @type`);
-						return;
+						errors.push(`Row ${i + 2}: is missing a @type`);
+						rowHasError = true;
+					}
+					if (rowHasError) {
+						rowsWithErrors.add(i);
+						continue;
 					}
 					if (
 						pretranslatedE["@type"].toLowerCase().startsWith('sample') || pretranslatedE["@type"].toLowerCase().startsWith('instruction')
@@ -783,15 +822,20 @@ module.exports = class CTDLASNCSVImport {
 					) {
 						// Validate required properties (only if validation rules provided)
 						if (validationRules && validationRules.requiredProps) {
-							const validationError = CTDLASNCSVImport.validateRequiredProperties(
+							const validationErrors = CTDLASNCSVImport.validateRequiredProperties(
 								pretranslatedE,
 								"ceterms:Collection",
 								i + 2,
 								validationRules.requiredProps
 							);
-							if (validationError) {
-								failure(validationError);
-								return;
+							if (validationErrors) {
+								if (Array.isArray(validationErrors)) {
+									errors.push(...validationErrors);
+								} else {
+									errors.push(validationErrors);
+								}
+								rowsWithErrors.add(i);
+								continue;
 							}
 						}
 						let translator = new EcLinkedData(null, null);
@@ -867,15 +911,20 @@ module.exports = class CTDLASNCSVImport {
 					) {
 						// Validate required properties (only if validation rules provided)
 						if (validationRules && validationRules.requiredProps) {
-							const validationError = CTDLASNCSVImport.validateRequiredProperties(
+							const validationErrors = CTDLASNCSVImport.validateRequiredProperties(
 								pretranslatedE,
 								"ceterms:Collection:Competency",
 								i + 2,
 								validationRules.requiredProps
 							);
-							if (validationError) {
-								failure(validationError);
-								return;
+							if (validationErrors) {
+								if (Array.isArray(validationErrors)) {
+									errors.push(...validationErrors);
+								} else {
+									errors.push(validationErrors);
+								}
+								rowsWithErrors.add(i);
+								continue;
 							}
 						}
 						let translator = new EcLinkedData(null, null);
@@ -901,8 +950,9 @@ module.exports = class CTDLASNCSVImport {
 						let f = new EcCompetency();
 						f.copyFrom(e);
 						if (e["id"] == null) {
-							failure(`Row ${i+2}: Competency is missing an id`);
-							return;
+							errors.push(`Row ${i+2}: Competency is missing an id`);
+							rowsWithErrors.add(i);
+							continue;
 						}
 						if (e["ceterms:isMemberOf"] != null) {
 							if (!EcArray.isArray(e["ceterms:isMemberOf"])) {
@@ -1095,15 +1145,22 @@ module.exports = class CTDLASNCSVImport {
 						pretranslatedE["@type"] == null ||
 						pretranslatedE["@type"] == ""
 					) {
-						failure(`Row ${i+2}: Missing or empty @type`);
-						return;
+						errors.push(`Row ${i+2}: Missing or empty @type`);
+						rowsWithErrors.add(i);
+						continue;
 					} else {
-						failure(
+						errors.push(
 							`Row ${i+2}: Found unknown type:` + pretranslatedE["@type"]
 						);
-						return;
+						rowsWithErrors.add(i);
+						continue;
 					}
 				}
+				// If there are any errors, report them all at once
+				if (errors.length > 0) {
+					failure(errors);
+					return;
+				}
 				success(frameworkArray, competencies, relations);
 			},
 			error: failure