@mitre/inspec-objects 0.0.29 → 0.0.31

package/.github/workflows/push-to-npm.yml

@@ -0,0 +1,33 @@
+name: Push @mitre/inspec-objects to NPM
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-deploy:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@master
+
+      - name: setup node
+        uses: actions/setup-node@v1
+        with:
+          node-version: "16.x"
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Install project dependencies
+        run: npm ci
+
+      - name: Remove testing resources
+        run: rm -rf test
+
+      - name: Build
+        run: npm run build
+
+      - name: Pack all items that are published as packages
+        run: npm pack
+
+      - name: Publish SAF CLI to NPM
+        run: npm publish --access public mitre-inspec-objects-*.tgz
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/lib/parsers/json.d.ts

@@ -3,4 +3,4 @@ import Profile from "../objects/profile";
 export declare function processEvaluation(evaluationInput: ContextualizedEvaluation): Profile;
 export declare function processProfileJSON(profileInput: ContextualizedProfile): Profile;
 export declare function processExecJSON(execJSON: ExecJSON.Execution): Profile;
-export declare function processJSON(json: string): Profile;
+export declare function processInSpecProfile(json: string): Profile;

package/lib/parsers/json.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.processJSON = exports.processExecJSON = exports.processProfileJSON = exports.processEvaluation = void 0;
+exports.processInSpecProfile = exports.processExecJSON = exports.processProfileJSON = exports.processEvaluation = void 0;
 const tslib_1 = require("tslib");
 const inspecjs_1 = require("inspecjs");
 const lodash_1 = tslib_1.__importDefault(require("lodash"));
@@ -72,7 +72,7 @@ function processExecJSON(execJSON) {
     return processEvaluation((0, inspecjs_1.contextualizeEvaluation)(execJSON));
 }
 exports.processExecJSON = processExecJSON;
-function processJSON(json) {
+function processInSpecProfile(json) {
     const convertedFile = (0, inspecjs_1.convertFile)(json, true);
     let profile = new profile_1.default();
     if (convertedFile["1_0_ExecJson"]) {
@@ -87,4 +87,4 @@ function processJSON(json) {
     profile.controls = lodash_1.default.sortBy(profile.controls, "id");
     return profile;
 }
-exports.processJSON = processJSON;
+exports.processInSpecProfile = processInSpecProfile;

package/lib/parsers/xccdf.js

@@ -40,6 +40,7 @@ exports.extractAllComplexChecks = extractAllComplexChecks;
 function ensureDecodedXMLStringValue(input) {
     return lodash_1.default.get(input, '[0].#text') ? lodash_1.default.get(input, '[0].#text') : input;
 }
+// Moving the newline removal to diff library rather than processXCCDF level
 function processXCCDF(xml, removeNewlines = false, useRuleId, ovalDefinitions) {
     const parsedXML = (0, xccdf_1.convertEncodedXmlIntoJson)(xml);
     const rules = extractAllRules(parsedXML.Benchmark[0].Group);

package/lib/utilities/diff.d.ts

@@ -2,8 +2,8 @@ import Profile from "../objects/profile";
 import { ProfileDiff } from "../types/diff";
 import winston from "winston";
 export declare function removeNewlines(control?: Record<string, unknown>): Record<string, unknown>;
-export declare function simplifyDiff(diffData: Record<string, unknown>): Record<string, unknown>;
+export declare function ignoreFormattingDiff(diffData: Record<string, unknown>): Record<string, unknown>;
 export declare function diffProfile(fromProfile: Profile, toProfile: Profile, logger: winston.Logger): {
-    simplified: ProfileDiff;
-    originalDiff: Record<string, unknown>;
+    ignoreFormattingDiff: ProfileDiff;
+    rawDiff: Record<string, unknown>;
 };

package/lib/utilities/diff.js

@@ -1,10 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.diffProfile = exports.simplifyDiff = exports.removeNewlines = void 0;
+exports.diffProfile = exports.ignoreFormattingDiff = exports.removeNewlines = void 0;
 const tslib_1 = require("tslib");
 const json_diff_1 = require("json-diff");
 const lodash_1 = tslib_1.__importDefault(require("lodash"));
 const update_1 = require("./update");
+const global_1 = require("./global");
 function removeNewlines(control) {
     if (!control) {
         return {};
@@ -20,14 +21,15 @@ function removeNewlines(control) {
     });
 }
 exports.removeNewlines = removeNewlines;
-function simplifyDiff(diffData) {
+// Goal is to use a linter for the formatting and compare characters without whitespaces here
+function ignoreFormattingDiff(diffData) {
     return lodash_1.default.transform(diffData, (result, diffValue, key) => {
         if (lodash_1.default.has(diffValue, "__new")) {
             // Remove any trailing space
             if (typeof lodash_1.default.get(diffValue, "__new") === "string" &&
                 typeof lodash_1.default.get(diffValue, "__old") === "string") {
-                if (lodash_1.default.get(diffValue, "__new").trim() !==
-                    lodash_1.default.get(diffValue, "__old").trim()) {
+                if ((0, global_1.removeWhitespace)(lodash_1.default.get(diffValue, "__new")) !==
+                    (0, global_1.removeWhitespace)(lodash_1.default.get(diffValue, "__old"))) {
                     lodash_1.default.set(result, key, lodash_1.default.get(diffValue, "__new"));
                 }
             }
@@ -41,7 +43,7 @@ function simplifyDiff(diffData) {
                 .filter((value) => value);
         }
         else if (typeof diffValue === "object") {
-            result[key] = simplifyDiff(diffValue);
+            result[key] = ignoreFormattingDiff(diffValue);
         }
         else if (key.endsWith("__deleted")) {
             return undefined;
@@ -51,19 +53,22 @@ function simplifyDiff(diffData) {
         }
     });
 }
-exports.simplifyDiff = simplifyDiff;
+exports.ignoreFormattingDiff = ignoreFormattingDiff;
 function diffProfile(fromProfile, toProfile, logger) {
+    var _a;
     const profileDiff = {
         addedControlIDs: [],
         removedControlIDs: [],
-        renamedControlIds: {},
+        renamedControlIDs: {},
+        changedControlIDs: [],
         addedControls: {},
         changedControls: {},
     };
     const originalDiff = {
         addedControlIDs: [],
         removedControlIDs: [],
-        renamedControlIds: {},
+        renamedControlIDs: {},
+        changedControlIDs: [],
         addedControls: {},
         changedControls: {},
     };
@@ -72,9 +77,11 @@ function diffProfile(fromProfile, toProfile, logger) {
         .sort();
     const toControlIDs = toProfile.controls.map((control) => control.id).sort();
     // Find new controls
-    const controlIDDiff = (0, json_diff_1.diff)(fromControlIDs, toControlIDs);
+    const controlIDDiff = (_a = (0, json_diff_1.diff)(fromControlIDs, toControlIDs)) === null || _a === void 0 ? void 0 : _a.filter((item) => !(item.length === 1 && item[0] === " "));
     // Contains the new IDs
     const changedControlIds = [];
+    // a diffValue has an entry for both what was subtracted ("-")
+    // and what was added ("+") -- need to handle both
     controlIDDiff === null || controlIDDiff === void 0 ? void 0 : controlIDDiff.forEach((diffValue) => {
         if (diffValue[0] === "-") {
             const existingControl = fromProfile.controls.find((control) => control.id === diffValue[1]);
@@ -82,13 +89,18 @@ function diffProfile(fromProfile, toProfile, logger) {
             if (existingControl) {
                 const newControl = (0, update_1.findUpdatedControlByAllIdentifiers)(existingControl, toProfile.controls);
                 if (newControl && newControl.id !== existingControl.id) {
-                    profileDiff.renamedControlIds[existingControl.id] = newControl.id;
-                    originalDiff.renamedControlIds[existingControl.id] = newControl.id;
+                    profileDiff.renamedControlIDs[existingControl.id] = newControl.id;
+                    originalDiff.renamedControlIDs[existingControl.id] = newControl.id;
                     changedControlIds.push(newControl.id.toLowerCase());
                     const controlDiff = lodash_1.default.omit((0, json_diff_1.diff)(existingControl, newControl), "code__deleted");
-                    profileDiff.changedControls[newControl.id] = simplifyDiff(controlDiff);
+                    // logger.info("CONTROL DIFF:" + JSON.stringify(controlDiff, null, 2))
+                    const renamedControlIgnoredFormatting = ignoreFormattingDiff(controlDiff);
+                    logger.info(JSON.stringify(renamedControlIgnoredFormatting));
+                    profileDiff.changedControls[newControl.id] = renamedControlIgnoredFormatting;
+                    profileDiff.changedControlIDs.push(newControl.id);
                     originalDiff.changedControls[newControl.id] = controlDiff;
-                    logger.verbose(`Control ${existingControl.id} has been upgraded to ${newControl.id}`);
+                    originalDiff.changedControlIDs.push(newControl.id);
+                    logger.verbose(`Control ${existingControl.id} has been updated to ${newControl.id}`);
                 }
                 else {
                     profileDiff.removedControlIDs.push(diffValue[1]);
@@ -99,11 +111,17 @@ function diffProfile(fromProfile, toProfile, logger) {
                 logger.error(`Unable to find existing control ${diffValue[1]}`);
             }
         }
-        else if (diffValue[0] === "+" && !changedControlIds.includes(diffValue[1].toLowerCase())) {
+        else if (diffValue[0] === "+" && !changedControlIds.includes(diffValue[1].toLowerCase()) && diffValue[1]) {
+            logger.info(JSON.stringify(diffValue));
+            logger.info(JSON.stringify(changedControlIds));
             profileDiff.addedControlIDs.push(diffValue[1]);
             originalDiff.addedControlIDs.push(diffValue[1]);
         }
     });
+    // take the list of renamed controls out of the list of added controls
+    // (a control is not "new" if it was renamed)
+    profileDiff.addedControlIDs = profileDiff.addedControlIDs.filter((item) => !Object.values(profileDiff.renamedControlIDs).includes(item));
+    originalDiff.addedControlIDs = originalDiff.addedControlIDs.filter((item) => !Object.values(originalDiff.renamedControlIDs).includes(item));
     // Add new controls to addedControls
     profileDiff.addedControlIDs.forEach((addedControl) => {
         const newControl = toProfile.controls.find((control) => addedControl === control.id);
@@ -118,11 +136,13 @@ function diffProfile(fromProfile, toProfile, logger) {
         if (toControl) {
             const controlDiff = lodash_1.default.omit((0, json_diff_1.diff)(fromControl, toControl), "code__deleted");
             if (controlDiff) {
-                profileDiff.changedControls[toControl.id] = simplifyDiff(controlDiff);
+                profileDiff.changedControls[toControl.id] = ignoreFormattingDiff(controlDiff);
+                profileDiff.changedControlIDs.push(toControl.id);
                 originalDiff.changedControls[toControl.id] = controlDiff;
+                originalDiff.changedControlIDs.push(toControl.id);
             }
         }
     }
-    return { simplified: profileDiff, originalDiff: originalDiff };
+    return { ignoreFormattingDiff: profileDiff, rawDiff: originalDiff };
 }
 exports.diffProfile = diffProfile;

package/lib/utilities/diffMarkdown.d.ts

@@ -1,5 +1,5 @@
 import { ProfileDiff } from "../types/diff";
 export declare function createDiffMarkdown(diff: {
-    simplified: ProfileDiff;
-    originalDiff: any;
+    ignoreFormattingDiff: ProfileDiff;
+    rawDiff: any;
 }): string;

package/lib/utilities/diffMarkdown.js

@@ -11,7 +11,7 @@ function getUpdatedCheckForId(id, profile) {
 }
 function createDiffMarkdown(diff) {
     const renderableDiffData = {
-        addedControls: Object.values(diff.simplified.addedControls),
+        addedControls: Object.values(diff.ignoreFormattingDiff.addedControls),
         hasRenamedControls: false,
         renamedControls: [],
         updatedChecks: [],
@@ -20,14 +20,14 @@ function createDiffMarkdown(diff) {
         updatedTitles: [],
         updatedDescriptions: [],
     };
-    Object.entries(diff.simplified.renamedControlIds).forEach(([oldId, newId]) => {
+    Object.entries(diff.ignoreFormattingDiff.renamedControlIDs).forEach(([oldId, newId]) => {
         renderableDiffData.hasRenamedControls = true;
         renderableDiffData.renamedControls.push({
             oldId: oldId,
             newId: newId,
         });
     });
-    Object.entries(diff.originalDiff.changedControls).forEach(([id, controlDiff]) => {
+    Object.entries(diff.rawDiff.changedControls).forEach(([id, controlDiff]) => {
         var _a, _b;
         if ((_a = controlDiff.descs) === null || _a === void 0 ? void 0 : _a.check) {
             const oldCheck = lodash_1.default.get(controlDiff.descs.check, "__old");

package/lib/utilities/global.d.ts

@@ -1,5 +1,6 @@
 export declare function wrap(s: string, lineLength?: number): string;
 export declare function unformatText(s: string): string;
+export declare function removeWhitespace(input: string): string;
 declare const escapeQuotes: (s: string) => string;
 declare const escapeDoubleQuotes: (s: string) => string;
 declare const wrapAndEscapeQuotes: (s: string, lineLength?: number) => string;

package/lib/utilities/global.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.hasPath = exports.getFirstPath = exports.removeNewlinePlaceholders = exports.wrapAndEscapeQuotes = exports.escapeDoubleQuotes = exports.escapeQuotes = exports.unformatText = exports.wrap = void 0;
+exports.hasPath = exports.getFirstPath = exports.removeNewlinePlaceholders = exports.wrapAndEscapeQuotes = exports.escapeDoubleQuotes = exports.escapeQuotes = exports.removeWhitespace = exports.unformatText = exports.wrap = void 0;
 const tslib_1 = require("tslib");
 const lodash_1 = tslib_1.__importDefault(require("lodash"));
 // Breaks lines down to lineLength number of characters
@@ -40,6 +40,10 @@ function unformatText(s) {
     return s.replace(/\n/g, ' ').replace(/\\n/g, ' ').replace(/( +|\t)/g, ' ');
 }
 exports.unformatText = unformatText;
+function removeWhitespace(input) {
+    return input.replace(/\s/gi, '');
+}
+exports.removeWhitespace = removeWhitespace;
 const escapeQuotes = (s) => s.replace(/\\/g, "\\\\").replace(/'/g, "\\'"); // Escape backslashes and quotes
 exports.escapeQuotes = escapeQuotes;
 const escapeDoubleQuotes = (s) => s.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); // Escape backslashes and double quotes

package/lib/utilities/update.d.ts

@@ -6,8 +6,8 @@ import { OvalDefinitionValue } from '../types/oval';
 export declare type UpdatedProfileReturn = {
     profile: Profile;
     diff: {
-        simplified: ProfileDiff;
-        originalDiff: Record<string, unknown>;
+        ignoreFormattingDiff: ProfileDiff;
+        rawDiff: Record<string, unknown>;
     };
     markdown: string;
 };

package/lib/utilities/update.js

@@ -38,14 +38,18 @@ function getExistingDescribeFromControl(control) {
     if (control.code) {
         let existingDescribeBlock = '';
         let currentQuoteEscape = '';
+        let percentBlockRegexp = /%[qQriIwWxs]?(?<lDelimiter>[\(\[\{\<])/;
+        let inPercentBlock = false;
         let inQuoteBlock = false;
         let inMetadataValueOverride = false;
         let indentedMetadataOverride = false;
         let inDescribeBlock = false;
         let mostSpacesSeen = 0;
+        let lDelimiter = '(';
+        let rDelimiter = ')';
         control.code.split('\n').forEach((line) => {
-            const wordSplit = line.trim().split(' ');
-            const spaces = line.substring(0, line.indexOf(wordSplit[0])).length;
+            const wordArray = line.trim().split(' ');
+            const spaces = line.substring(0, line.indexOf(wordArray[0])).length;
             if (spaces - mostSpacesSeen > 10) {
                 indentedMetadataOverride = true;
             }
@@ -53,24 +57,51 @@ function getExistingDescribeFromControl(control) {
                 mostSpacesSeen = spaces;
                 indentedMetadataOverride = false;
             }
-            if (control.id === 'SV-204392') {
-                console.log(line);
-                console.log(inDescribeBlock);
-            }
-            if ((!inQuoteBlock && !inMetadataValueOverride && !indentedMetadataOverride) || inDescribeBlock) {
-                // Get the number of spaces at the beggining of the current line
-                if (spaces >= 2) {
-                    const firstWord = wordSplit[0];
+            if ((!inPercentBlock && !inQuoteBlock && !inMetadataValueOverride && !indentedMetadataOverride) || inDescribeBlock) {
+                if (inDescribeBlock && wordArray.length === 1 && wordArray.includes('')) {
+                    existingDescribeBlock += '\n';
+                }
+                // Get the number of spaces at the beginning of the current line
+                else if (spaces >= 2) {
+                    const firstWord = wordArray[0];
                     if (knownInSpecKeywords.indexOf(firstWord.toLowerCase()) === -1 || (knownInSpecKeywords.indexOf(firstWord.toLowerCase()) !== -1 && spaces > 2) || inDescribeBlock) {
                         inDescribeBlock = true;
                         existingDescribeBlock += line + '\n';
                     }
                 }
             }
-            wordSplit.forEach((word, index) => {
-                const charSplit = word.split('');
-                charSplit.forEach((char, index) => {
-                    if (char === '"' && charSplit[index - 1] !== '\\') {
+            wordArray.forEach((word, index) => {
+                //console.log(`LDELIMITER:   \"${lDelimiter}\"   RDELIMITER:   \"${rDelimiter}\"`)
+                let percentBlockMatch = percentBlockRegexp.exec(word);
+                if (percentBlockMatch && inPercentBlock === false) {
+                    inPercentBlock = true;
+                    lDelimiter = percentBlockMatch.groups.lDelimiter || '(';
+                    switch (lDelimiter) {
+                        case '{': {
+                            rDelimiter = '}';
+                            break;
+                        }
+                        case '[': {
+                            rDelimiter = ']';
+                            break;
+                        }
+                        case '<': {
+                            rDelimiter = '>';
+                            break;
+                        }
+                        default: {
+                            break;
+                        }
+                    }
+                }
+                const charArray = word.split('');
+                charArray.forEach((char, index) => {
+                    if (inPercentBlock) {
+                        if (char === rDelimiter && charArray[index - 1] !== '\\' && !inQuoteBlock) {
+                            inPercentBlock = false;
+                        }
+                    }
+                    if (char === '"' && charArray[index - 1] !== '\\') {
                         if (!currentQuoteEscape || !inQuoteBlock) {
                             currentQuoteEscape = '"';
                         }
@@ -78,7 +109,7 @@ function getExistingDescribeFromControl(control) {
                             inQuoteBlock = !inQuoteBlock;
                         }
                     }
-                    else if (char === "'" && charSplit[index - 1] !== '\\') {
+                    else if (char === "'" && charArray[index - 1] !== '\\') {
                         if (!currentQuoteEscape || !inQuoteBlock) {
                             currentQuoteEscape = "'";
                         }
@@ -89,7 +120,8 @@ function getExistingDescribeFromControl(control) {
                 });
             });
         });
-        return existingDescribeBlock;
+        // Take off the extra newline at the end
+        return existingDescribeBlock.slice(0, -1);
     }
     else {
         return '';
@@ -131,8 +163,8 @@ function updateProfile(from, using, logger) {
     // Find the diff
     const diff = (0, diff_1.diffProfile)(from, using, logger);
     // Add the new controls
-    diff.simplified.addedControlIDs.forEach(id => {
-        const addedControl = diff.simplified.addedControls[id];
+    diff.ignoreFormattingDiff.addedControlIDs.forEach(id => {
+        const addedControl = diff.ignoreFormattingDiff.addedControls[id];
         if (addedControl) {
             logger.debug(`New Control: ${addedControl.id} - ${addedControl.title}`);
             to.controls.push(addedControl);
@@ -145,7 +177,7 @@ function updateProfile(from, using, logger) {
     for (const existingControl of from.controls) {
         const updatedControl = findUpdatedControlByAllIdentifiers(existingControl, using.controls);
         if (updatedControl) {
-            const controlDiff = diff.simplified.changedControls[updatedControl.id];
+            const controlDiff = diff.ignoreFormattingDiff.changedControls[updatedControl.id];
             if (controlDiff) {
                 to.controls.push(updateControl(existingControl, controlDiff, logger));
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mitre/inspec-objects",
-  "version": "0.0.29",
+  "version": "0.0.31",
   "description": "Typescript objects for normalizing between InSpec profiles and XCCDF benchmarks",
   "main": "lib/index.js",
   "publishConfig": {