cassproject 1.5.36 → 1.5.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +4 -1
  2. package/package.json +8 -8
  3. package/src/org/cass/importer/CTDLASNCSVImport.js +59 -4
package/README.md CHANGED
@@ -35,7 +35,7 @@ Development unit tests presume you have a CaSS Repository running on `localhost:
35
35
 
36
36
  ## Publish checklist
37
37
 
38
- * Review dependencies, autocomplete version numbers to latest versions.
38
+ * `npm upgrade --save` Review dependencies, autocomplete version numbers to latest versions.
39
39
  * Increment version number in package.json and yuidoc.json.
40
40
  * Update changelog.
41
41
  * `npm install`
@@ -48,6 +48,9 @@ Development unit tests presume you have a CaSS Repository running on `localhost:
48
48
 
49
49
  # Changelog
50
50
 
51
+ ## 1.5.37
52
+ * Can now search for and de-duplicate competencies on import.
53
+
51
54
  ## 1.5.35
52
55
  * If --force-fips is enabled, always tries to use SHA-256 instead of crashing.
53
56
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "cassproject",
3
- "version": "1.5.36",
3
+ "version": "1.5.37",
4
4
  "description": "Competency and Skills Service",
5
5
  "main": "index.js",
6
6
  "scripts": {
@@ -9,11 +9,11 @@
9
9
  "multitest": "concurrently --kill-others --kill-others-on-fail \"npm run test15\" \"npm run test14\" \"npm run test13\" \"npm run test12\"",
10
10
  "testCassTest": "npm run testkill && docker run -d --name cass-test -p80:80 -e CASS_LOOPBACK cass-test && wait-on http://localhost/api/ping && npm run testNode18 && npm run testNode18Fips && npm run testNode16 && npm run testNode15 && npm run testNode14 && npm run testNode13 && npm run testNode12 && npm run testCypressEdge && npm run testCypress && npm run testkill",
11
11
  "testDevHttps": "npm run testkill && docker run -d --name cass-test -p443:80 -e CASS_LOOPBACK -e HTTPS=true cassproject/cass:dev && wait-on https://localhost/api/ping && npm run testNode18HttpsFips && npm run testNode18Https && npm run testNode16Https && npm run testNode15Https && npm run testNode14Https && npm run testNode13Https && npm run testNode12Https && npm run testCypressEdgeHttps && npm run testCypressHttps",
12
- "test15HttpsFips": "npm run testkill && docker run -d --name cass-test -p443:443 -e CASS_LOOPBACK -e HTTPS=true cassproject/cass:1.5.34 && wait-on https://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypressHttps\" \"npm run testNode20Https\" \"npm run testNode20HttpsFips\" \"npm run testNode20HttpsForceFips\" \"npm run testNode18Https\" \"npm run testNode18HttpsFips\" \"npm run testNode16Https\" && npm run testkill",
13
- "test15Https11Fips": "npm run testkill && docker run -d --name cass-test -p443:443 -e CASS_LOOPBACK -e HTTPS=true -e HTTP2=false cassproject/cass:1.5.34 && wait-on https://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypressHttps\" \"npm run testNode20Https\" \"npm run testNode20HttpsFips\" \"npm run testNode20HttpsForceFips\" \"npm run testNode18Https\" \"npm run testNode18HttpsFips\" \"npm run testNode16Https\" && npm run testkill",
12
+ "test15HttpsFips": "npm run testkill && docker run -d --name cass-test -p443:443 -e CASS_LOOPBACK -e HTTPS=true cassproject/cass:1.5.35 && wait-on https://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypressHttps\" \"npm run testNode20Https\" \"npm run testNode20HttpsFips\" \"npm run testNode20HttpsForceFips\" \"npm run testNode18Https\" \"npm run testNode18HttpsFips\" \"npm run testNode16Https\" && npm run testkill",
13
+ "test15Https11Fips": "npm run testkill && docker run -d --name cass-test -p443:443 -e CASS_LOOPBACK -e HTTPS=true -e HTTP2=false cassproject/cass:1.5.35 && wait-on https://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypressHttps\" \"npm run testNode20Https\" \"npm run testNode20HttpsFips\" \"npm run testNode20HttpsForceFips\" \"npm run testNode18Https\" \"npm run testNode18HttpsFips\" \"npm run testNode16Https\" && npm run testkill",
14
14
  "test15Https": "npm run testkill && docker run -d --name cass-test -p443:443 -e CASS_LOOPBACK -e HTTPS=true cassproject/cass:1.5.32 && wait-on https://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypressHttps\" \"npm run testNode20Https\" \"npm run testNode20HttpsFips\" \"npm run testNode18Https\" \"npm run testNode18HttpsFips\" \"npm run testNode16Https\" && npm run testkill",
15
15
  "test15Https11": "npm run testkill && docker run -d --name cass-test -p443:443 -e CASS_LOOPBACK -e HTTPS=true -e HTTP2=false cassproject/cass:1.5.32 && wait-on https://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypressHttps\" \"npm run testNode20Https\" \"npm run testNode20HttpsFips\" \"npm run testNode18Https\" \"npm run testNode18HttpsFips\" \"npm run testNode16Https\" && npm run testkill",
16
- "test15Fips": "export CASS_LOOPBACK=http://localhost/api/|| set CASS_LOOPBACK=http://localhost/api/&& npm run testkill15 && docker run -d -e CASS_LOOPBACK --name cass-test15 -p80:80 cassproject/cass:1.5.34 && wait-on http://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypress\" \"npm run testNode20\" \"npm run testNode20Fips\" \"npm run testNode20ForceFips\" \"npm run testNode18\" \"npm run testNode18Fips\" \"npm run testNode16\" && npm run testkill15",
16
+ "test15Fips": "export CASS_LOOPBACK=http://localhost/api/|| set CASS_LOOPBACK=http://localhost/api/&& npm run testkill15 && docker run -d -e CASS_LOOPBACK --name cass-test15 -p80:80 cassproject/cass:1.5.35 && wait-on http://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypress\" \"npm run testNode20\" \"npm run testNode20Fips\" \"npm run testNode20ForceFips\" \"npm run testNode18\" \"npm run testNode18Fips\" \"npm run testNode16\" && npm run testkill15",
17
17
  "test15": "export CASS_LOOPBACK=http://localhost/api/|| set CASS_LOOPBACK=http://localhost/api/&& npm run testkill15 && docker run -d -e CASS_LOOPBACK --name cass-test15 -p80:80 cassproject/cass:1.5.32 && wait-on http://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypress\" \"npm run testNode20\" \"npm run testNode20Fips\" \"npm run testNode18\" \"npm run testNode18Fips\" \"npm run testNode16\" && npm run testkill15",
18
18
  "test14": "npm run testkill14 && docker run -d -e CASS_LOOPBACK --name cass-test14 -p80:80 cassproject/cass:1.4.4 && wait-on http://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypress\" \"npm run testNode16\" && npm run testkill14",
19
19
  "test13": "npm run testkill13 && docker run -d -e CASS_LOOPBACK --name cass-test13 -p80:80 cassproject/cass:1.3.18 && wait-on http://localhost/api/ping && concurrently --kill-others-on-fail \"npm run testCypress\" \"npm run testNode16\" && npm run testkill13",
@@ -139,18 +139,18 @@
139
139
  },
140
140
  "homepage": "https://github.com/cassproject/cass-npm#readme",
141
141
  "devDependencies": {
142
- "@babel/core": "^7.22.20",
142
+ "@babel/core": "^7.23.0",
143
143
  "@babel/preset-env": "^7.22.20",
144
144
  "@cypress/browserify-preprocessor": "^3.0.2",
145
145
  "@cypress/vite-dev-server": "^5.0.6",
146
146
  "@cypress/webpack-preprocessor": "^6.0.0",
147
147
  "babel-eslint": "^10.1.0",
148
148
  "babel-plugin-transform-remove-strict-mode": "^0.0.2",
149
- "chai": "^4.3.8",
149
+ "chai": "^4.3.10",
150
150
  "concurrently": "^8.2.1",
151
151
  "convert-hrtime": "^5.0.0",
152
- "cypress": "^13.2.0",
153
- "eslint": "^8.49.0",
152
+ "cypress": "^13.3.0",
153
+ "eslint": "^8.50.0",
154
154
  "mocha": "^10.2.0",
155
155
  "node-polyfill-webpack-plugin": "^2.0.1",
156
156
  "nodemon": "^3.0.1",
package/src/org/cass/importer/CTDLASNCSVImport.js CHANGED
@@ -23,11 +23,48 @@ module.exports = class CTDLASNCSVImport {
23
23
  let frameworkCounter = 0;
24
24
  let competencyCounter = 0;
25
25
  let collectionCounter = 0;
26
+ let duplicates = [];
27
+ let uniqueRows = [];
26
28
  let typeCol = nameToCol["@type"];
29
+ let uniqueRowIndexes = [];
27
30
  if (typeCol == null) {
28
31
  this.error("No @type in CSV.");
29
32
  return;
30
33
  }
34
+ // Search for duplicate competencies with different CTIDs
35
+ if (tabularData[0]) {
36
+ const colId = tabularData[0].findIndex((element) => element.toLowerCase().contains("@id"));
37
+ const colCtid = tabularData[0].findIndex((element) => element.toLowerCase().contains("ceterms:ctid"));
38
+ const colCompetencyText = tabularData[0].findIndex((element) => element.toLowerCase().contains("ceasn:competencytext"));
39
+ const colCodedNotation = tabularData[0].findIndex((element) => element.toLowerCase().contains("ceasn:codednotation"));
40
+ if (colCtid >= 0) {
41
+ for (let i = 1; i < tabularData.length; i++) {
42
+ const row = tabularData[i].filter((element, j) => (j !== colCtid) && (j !== colId));
43
+ const existing = uniqueRowIndexes.findIndex((uniqueRow) => uniqueRow.every((each, k) => each === row[k]))
44
+ if (existing < 0) {
45
+ uniqueRowIndexes.push(row);
46
+ uniqueRows.push({
47
+ competencyText: colCompetencyText >= 0 ? tabularData[i][colCompetencyText] : undefined,
48
+ ctid: tabularData[i][colCtid],
49
+ codedNotation: colCodedNotation >= 0 ? tabularData[i][colCodedNotation] : undefined,
50
+ line: i
51
+ });
52
+ } else {
53
+ const originalAlreadyAdded = duplicates.find((duplicate) => duplicate.line === uniqueRows[existing].line);
54
+ if (!originalAlreadyAdded) {
55
+ duplicates.push(uniqueRows[existing]);
56
+ }
57
+ duplicates.push({
58
+ competencyText: colCompetencyText >= 0 ? tabularData[i][colCompetencyText] : undefined,
59
+ ctid: tabularData[i][colCtid],
60
+ codedNotation: colCodedNotation >= 0 ? tabularData[i][colCodedNotation] : undefined,
61
+ line: i
62
+ });
63
+ }
64
+ }
65
+ duplicates.sort((a, b) => a.competencyText < b.competencyText ? -1 : 1);
66
+ }
67
+ }
31
68
  for (let i = 0; i < tabularData.length; i++) {
32
69
  if (i == 0) continue;
33
70
  let col = tabularData[i];
@@ -53,7 +90,7 @@ module.exports = class CTDLASNCSVImport {
53
90
  return;
54
91
  }
55
92
  }
56
- success(frameworkCounter, competencyCounter, collectionCounter);
93
+ success(frameworkCounter, competencyCounter, collectionCounter, duplicates);
57
94
  },
58
95
  error: failure
59
96
  });
@@ -66,7 +103,8 @@ module.exports = class CTDLASNCSVImport {
66
103
  ceo,
67
104
  endpoint,
68
105
  eim,
69
- collectionsFlag
106
+ collectionsFlag,
107
+ skip
70
108
  ) {
71
109
  if (eim === undefined || eim == null)
72
110
  eim = EcIdentityManager.default;
@@ -80,7 +118,7 @@ module.exports = class CTDLASNCSVImport {
80
118
  failure("Invalid file type");
81
119
  }
82
120
  if (collectionsFlag) {
83
- return this.importCollectionsAndCompetencies(repo, file, success, failure, ceo, endpoint, eim);
121
+ return this.importCollectionsAndCompetencies(repo, file, success, failure, ceo, endpoint, eim, skip);
84
122
  }
85
123
  Papa.parse(file, {
86
124
  header: true,
@@ -518,7 +556,8 @@ module.exports = class CTDLASNCSVImport {
518
556
  failure,
519
557
  ceo,
520
558
  endpoint,
521
- eim
559
+ eim,
560
+ skip
522
561
  ) {
523
562
  Papa.parse(file, {
524
563
  header: true,
@@ -550,6 +589,12 @@ module.exports = class CTDLASNCSVImport {
550
589
  }
551
590
  delete pretranslatedE["ceterms:CTID"];
552
591
  }
592
+ // Skip competency if ctid is specified
593
+ if (skip && Array.isArray(skip) && skip.length > 0) {
594
+ if (skip.find((element) => element.ctid ? element.ctid.includes(pretranslatedE["ceterms:ctid"]) : element === pretranslatedE["ceterms:ctid"])) {
595
+ continue;
596
+ }
597
+ }
553
598
  if (
554
599
  pretranslatedE["@type"] ==
555
600
  "ceterms:Collection"
@@ -608,6 +653,16 @@ module.exports = class CTDLASNCSVImport {
608
653
  frameworkRows[f.shortId()] = e;
609
654
  frameworkArray.push(f);
610
655
  f.competency = f["ceterms:hasMember"] || [];
656
+ // Remove skipped competencies
657
+ if (skip && Array.isArray(skip) && skip.length > 0 && f.competency) {
658
+ skip.forEach((element) => {
659
+ const id = (element.ctid ? element.ctid : element).replace('ce-', '');
660
+ const index = f.competency.findIndex((comp) => comp.includes(id));
661
+ if (index) {
662
+ f.competency.splice(index, 1);
663
+ }
664
+ });
665
+ }
611
666
  delete f["ceterms:hasMember"];
612
667
  f.relation = [];
613
668
  f.subType = "Collection";