@featurevisor/core 1.2.1 → 1.2.2

package/lib/tester/testProject.js

@@ -41,10 +41,12 @@ var fs = require("fs");
 var testSegment_1 = require("./testSegment");
 var testFeature_1 = require("./testFeature");
 var cliFormat_1 = require("./cliFormat");
+var prettyDuration_1 = require("./prettyDuration");
+var printTestResult_1 = require("./printTestResult");
 function testProject(deps, options) {
     if (options === void 0) { options = {}; }
     return __awaiter(this, void 0, void 0, function () {
-        var rootDirectoryPath, projectConfig, datasource, hasError, testFiles, patterns, _i, testFiles_1, testFile, testFilePath, t, test_1, segmentHasError, test_2, featureHasError;
+        var rootDirectoryPath, projectConfig, datasource, hasError, testFiles, startTime, patterns, passedTestsCount, failedTestsCount, passedAssertionsCount, failedAssertionsCount, _i, testFiles_1, testFile, testFilePath, t, test_1, testResult, test_2, testResult, diffInMs, testSpecsMessage, testAssertionsMessage;
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
@@ -63,10 +65,15 @@ function testProject(deps, options) {
                         hasError = true;
                         return [2 /*return*/, hasError];
                     }
+                    startTime = Date.now();
                     patterns = {
                         keyPattern: options.keyPattern ? new RegExp(options.keyPattern) : undefined,
                         assertionPattern: options.assertionPattern ? new RegExp(options.assertionPattern) : undefined,
                     };
+                    passedTestsCount = 0;
+                    failedTestsCount = 0;
+                    passedAssertionsCount = 0;
+                    failedAssertionsCount = 0;
                     _i = 0, testFiles_1 = testFiles;
                     _a.label = 2;
                 case 2:
@@ -81,12 +88,19 @@ function testProject(deps, options) {
                     if (patterns.keyPattern && !patterns.keyPattern.test(test_1.segment)) {
                         return [3 /*break*/, 8];
                     }
-                    console.log(cliFormat_1.CLI_FORMAT_BOLD, "\nTesting: ".concat(testFilePath.replace(rootDirectoryPath, "")));
                     return [4 /*yield*/, (0, testSegment_1.testSegment)(datasource, test_1, patterns)];
                 case 4:
-                    segmentHasError = _a.sent();
-                    if (segmentHasError) {
+                    testResult = _a.sent();
+                    (0, printTestResult_1.printTestResult)(testResult, testFilePath, rootDirectoryPath);
+                    if (!testResult.passed) {
                         hasError = true;
+                        failedTestsCount++;
+                        failedAssertionsCount += testResult.assertions.filter(function (a) { return !a.passed; }).length;
+                        passedAssertionsCount += testResult.assertions.length - failedAssertionsCount;
+                    }
+                    else {
+                        passedTestsCount++;
+                        passedAssertionsCount += testResult.assertions.length;
                     }
                     return [3 /*break*/, 8];
                 case 5:
@@ -95,12 +109,19 @@ function testProject(deps, options) {
                     if (patterns.keyPattern && !patterns.keyPattern.test(test_2.feature)) {
                         return [3 /*break*/, 8];
                     }
-                    console.log(cliFormat_1.CLI_FORMAT_BOLD, "\nTesting: ".concat(testFilePath.replace(rootDirectoryPath, "")));
                     return [4 /*yield*/, (0, testFeature_1.testFeature)(datasource, projectConfig, test_2, options, patterns)];
                 case 6:
-                    featureHasError = _a.sent();
-                    if (featureHasError) {
+                    testResult = _a.sent();
+                    (0, printTestResult_1.printTestResult)(testResult, testFilePath, rootDirectoryPath);
+                    if (!testResult.passed) {
                         hasError = true;
+                        failedTestsCount++;
+                        failedAssertionsCount += testResult.assertions.filter(function (a) { return !a.passed; }).length;
+                        passedAssertionsCount += testResult.assertions.length - failedAssertionsCount;
+                    }
+                    else {
+                        passedTestsCount++;
+                        passedAssertionsCount += testResult.assertions.length;
                     }
                     return [3 /*break*/, 8];
                 case 7:
@@ -111,14 +132,19 @@ function testProject(deps, options) {
                     _i++;
                     return [3 /*break*/, 2];
                 case 9:
-                    console.log("");
+                    diffInMs = Date.now() - startTime;
+                    console.log("\n---\n");
+                    testSpecsMessage = "Test specs: ".concat(passedTestsCount, " passed, ").concat(failedTestsCount, " failed");
+                    testAssertionsMessage = "Assertions: ".concat(passedAssertionsCount, " passed, ").concat(failedAssertionsCount, " failed");
                     if (hasError) {
-                        console.log(cliFormat_1.CLI_FORMAT_RED, "Some tests failed");
+                        console.log(cliFormat_1.CLI_FORMAT_RED, testSpecsMessage);
+                        console.log(cliFormat_1.CLI_FORMAT_RED, testAssertionsMessage);
                     }
                     else {
-                        console.log(cliFormat_1.CLI_FORMAT_GREEN, "All tests passed");
+                        console.log(cliFormat_1.CLI_FORMAT_GREEN, testSpecsMessage);
+                        console.log(cliFormat_1.CLI_FORMAT_GREEN, testAssertionsMessage);
                     }
-                    console.log("");
+                    console.log(cliFormat_1.CLI_FORMAT_BOLD, "Time:       ".concat((0, prettyDuration_1.prettyDuration)(diffInMs)));
                     return [2 /*return*/, hasError];
             }
         });

package/lib/tester/testProject.js.map

@@ -1 +1 @@
-{"version":3,"file":"testProject.js","sourceRoot":"","sources":["../../src/tester/testProject.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uBAAyB;AAIzB,6CAA4C;AAC5C,6CAA4C;AAC5C,yCAAgF;AAUhF,SAAsB,WAAW,CAC/B,IAAkB,EAClB,OAAgC;IAAhC,wBAAA,EAAA,YAAgC;;;;;;oBAExB,iBAAiB,GAAgC,IAAI,kBAApC,EAAE,aAAa,GAAiB,IAAI,cAArB,EAAE,UAAU,GAAK,IAAI,WAAT,CAAU;oBAE1D,QAAQ,GAAG,KAAK,CAAC;oBAErB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;wBACpD,OAAO,CAAC,KAAK,CAAC,0CAAmC,aAAa,CAAC,kBAAkB,CAAE,CAAC,CAAC;wBACrF,QAAQ,GAAG,IAAI,CAAC;wBAEhB,sBAAO,QAAQ,EAAC;qBACjB;oBAEiB,qBAAM,UAAU,CAAC,SAAS,EAAE,EAAA;;oBAAxC,SAAS,GAAG,SAA4B;oBAE9C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;wBAC1B,OAAO,CAAC,KAAK,CAAC,6BAAsB,aAAa,CAAC,kBAAkB,CAAE,CAAC,CAAC;wBACxE,QAAQ,GAAG,IAAI,CAAC;wBAEhB,sBAAO,QAAQ,EAAC;qBACjB;oBAEK,QAAQ,GAAG;wBACf,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS;wBAC3E,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS;qBAC9F,CAAC;0BAE8B,EAAT,uBAAS;;;yBAAT,CAAA,uBAAS,CAAA;oBAArB,QAAQ;oBACX,YAAY,GAAG,UAAU,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;oBAEhD,qBAAM,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAA;;oBAAvC,CAAC,GAAG,SAAmC;yBAExC,CAAiB,CAAC,OAAO,EAA1B,wBAA0B;oBAEtB,SAAO,CAAgB,CAAC;oBAE9B,IAAI,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,MAAI,CAAC,OAAO,CAAC,EAAE;wBAClE,wBAAS;qBACV;oBAED,OAAO,CAAC,GAAG,CAAC,2BAAe,EAAE,qBAAc,YAAY,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAE,CAAC,CAAC;oBAElE,qBAAM,IAAA,yBAAW,EAAC,UAAU,EAAE,MAAI,EAAE,QAAQ,CAAC,EAAA;;oBAA/D,eAAe,GAAG,SAA6C;oBAErE,IAAI,eAAe,EAAE;wBACnB,QAAQ,GAAG,IAAI,CAAC;qBACjB;;;yBACS,CAAiB,CAAC,OAAO,EAA1B,wBAA0B;oBAE7B,SAAO,CAAgB,CAAC;oBAE9B,IAAI,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,MAAI,CAAC,OAAO,CAAC,EAAE;wBAClE,wBAAS;qBACV;oBAED,OAAO,CAAC,GAAG,CAAC,2BAAe,EAAE,qBAAc,YAAY,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAE,CAAC,CAAC;oBAElE,qBAAM,IAAA,yBAAW,EAAC,UAAU,EAAE,aAAa,EAAE,MAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAA;;oBAAvF,eAAe,GAAG,SAAqE;oBAE7F,IAAI,eAAe,EAAE;wBACnB,QAAQ,GAAG,IAAI,CAAC;qBACjB;;;oBAED,OAAO,CAAC,KAAK,CAAC,6BAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAE,CAAC,CAAC;oBAC5D,QAAQ,GAAG,IAAI,CAAC;;;oBArCG,IAAS,CAAA;;;oBAyChC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAChB,IAAI,QAAQ,EAAE;wBACZ,OAAO,CAAC,GAAG,CAAC,0BAAc,EAAE,mBAAmB,CAAC,CAAC;qBAClD;yBAAM;wBACL,OAAO,CAAC,GAAG,CAAC,4BAAgB,EAAE,kBAAkB,CAAC,CAAC;qBACnD;oBACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAEhB,sBAAO,QAAQ,EAAC;;;;CACjB;AA/ED,kCA+EC"}
+{"version":3,"file":"testProject.js","sourceRoot":"","sources":["../../src/tester/testProject.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uBAAyB;AAIzB,6CAA4C;AAC5C,6CAA4C;AAC5C,yCAAgF;AAEhF,mDAAkD;AAClD,qDAAoD;AASpD,SAAsB,WAAW,CAC/B,IAAkB,EAClB,OAAgC;IAAhC,wBAAA,EAAA,YAAgC;;;;;;oBAExB,iBAAiB,GAAgC,IAAI,kBAApC,EAAE,aAAa,GAAiB,IAAI,cAArB,EAAE,UAAU,GAAK,IAAI,WAAT,CAAU;oBAE1D,QAAQ,GAAG,KAAK,CAAC;oBAErB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;wBACpD,OAAO,CAAC,KAAK,CAAC,0CAAmC,aAAa,CAAC,kBAAkB,CAAE,CAAC,CAAC;wBACrF,QAAQ,GAAG,IAAI,CAAC;wBAEhB,sBAAO,QAAQ,EAAC;qBACjB;oBAEiB,qBAAM,UAAU,CAAC,SAAS,EAAE,EAAA;;oBAAxC,SAAS,GAAG,SAA4B;oBAE9C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;wBAC1B,OAAO,CAAC,KAAK,CAAC,6BAAsB,aAAa,CAAC,kBAAkB,CAAE,CAAC,CAAC;wBACxE,QAAQ,GAAG,IAAI,CAAC;wBAEhB,sBAAO,QAAQ,EAAC;qBACjB;oBAEK,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAEvB,QAAQ,GAAG;wBACf,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS;wBAC3E,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS;qBAC9F,CAAC;oBAEE,gBAAgB,GAAG,CAAC,CAAC;oBACrB,gBAAgB,GAAG,CAAC,CAAC;oBAErB,qBAAqB,GAAG,CAAC,CAAC;oBAC1B,qBAAqB,GAAG,CAAC,CAAC;0BAEE,EAAT,uBAAS;;;yBAAT,CAAA,uBAAS,CAAA;oBAArB,QAAQ;oBACX,YAAY,GAAG,UAAU,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;oBAEhD,qBAAM,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAA;;oBAAvC,CAAC,GAAG,SAAmC;yBAExC,CAAiB,CAAC,OAAO,EAA1B,wBAA0B;oBAEtB,SAAO,CAAgB,CAAC;oBAE9B,IAAI,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,MAAI,CAAC,OAAO,CAAC,EAAE;wBAClE,wBAAS;qBACV;oBAEkB,qBAAM,IAAA,yBAAW,EAAC,UAAU,EAAE,MAAI,EAAE,QAAQ,CAAC,EAAA;;oBAA1D,UAAU,GAAG,SAA6C;oBAChE,IAAA,iCAAe,EAAC,UAAU,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC;oBAE7D,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;wBACtB,QAAQ,GAAG,IAAI,CAAC;wBAChB,gBAAgB,EAAE,CAAC;wBAEnB,qBAAqB,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,CAAC,MAAM,EAAT,CAAS,CAAC,CAAC,MAAM,CAAC;wBAC/E,qBAAqB,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,qBAAqB,CAAC;qBAC/E;yBAAM;wBACL,gBAAgB,EAAE,CAAC;wBAEnB,qBAAqB,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC;qBACvD;;;yBACS,CAAiB,CAAC,OAAO,EAA1B,wBAA0B;oBAE7B,SAAO,CAAgB,CAAC;oBAE9B,IAAI,QAAQ,CAAC,UAAU,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,MAAI,CAAC,OAAO,CAAC,EAAE;wBAClE,wBAAS;qBACV;oBAEkB,qBAAM,IAAA,yBAAW,EAAC,UAAU,EAAE,aAAa,EAAE,MAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAA;;oBAAlF,UAAU,GAAG,SAAqE;oBACxF,IAAA,iCAAe,EAAC,UAAU,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC;oBAE7D,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;wBACtB,QAAQ,GAAG,IAAI,CAAC;wBAChB,gBAAgB,EAAE,CAAC;wBAEnB,qBAAqB,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,CAAC,MAAM,EAAT,CAAS,CAAC,CAAC,MAAM,CAAC;wBAC/E,qBAAqB,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,qBAAqB,CAAC;qBAC/E;yBAAM;wBACL,gBAAgB,EAAE,CAAC;wBAEnB,qBAAqB,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC;qBACvD;;;oBAED,OAAO,CAAC,KAAK,CAAC,6BAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAE,CAAC,CAAC;oBAC5D,QAAQ,GAAG,IAAI,CAAC;;;oBAnDG,IAAS,CAAA;;;oBAuD1B,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAExC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;oBAEjB,gBAAgB,GAAG,sBAAe,gBAAgB,sBAAY,gBAAgB,YAAS,CAAC;oBACxF,qBAAqB,GAAG,sBAAe,qBAAqB,sBAAY,qBAAqB,YAAS,CAAC;oBAC7G,IAAI,QAAQ,EAAE;wBACZ,OAAO,CAAC,GAAG,CAAC,0BAAc,EAAE,gBAAgB,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,0BAAc,EAAE,qBAAqB,CAAC,CAAC;qBACpD;yBAAM;wBACL,OAAO,CAAC,GAAG,CAAC,4BAAgB,EAAE,gBAAgB,CAAC,CAAC;wBAChD,OAAO,CAAC,GAAG,CAAC,4BAAgB,EAAE,qBAAqB,CAAC,CAAC;qBACtD;oBAED,OAAO,CAAC,GAAG,CAAC,2BAAe,EAAE,sBAAe,IAAA,+BAAc,EAAC,QAAQ,CAAC,CAAE,CAAC,CAAC;oBAExE,sBAAO,QAAQ,EAAC;;;;CACjB;AA7GD,kCA6GC"}

package/lib/tester/testSegment.d.ts

@@ -1,3 +1,3 @@
-import { TestSegment } from "@featurevisor/types";
+import { TestSegment, TestResult } from "@featurevisor/types";
 import { Datasource } from "../datasource";
-export declare function testSegment(datasource: Datasource, test: TestSegment, patterns: any): Promise<boolean>;
+export declare function testSegment(datasource: Datasource, test: TestSegment, patterns: any): Promise<TestResult>;

package/lib/tester/testSegment.js

@@ -38,24 +38,31 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.testSegment = void 0;
 var sdk_1 = require("@featurevisor/sdk");
-var cliFormat_1 = require("./cliFormat");
 var matrix_1 = require("./matrix");
 function testSegment(datasource, test, patterns) {
     return __awaiter(this, void 0, void 0, function () {
-        var hasError, segmentKey, segmentExists, parsedSegment, conditions;
+        var testStartTime, segmentKey, testResult, segmentExists, parsedSegment, conditions;
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
-                    hasError = false;
+                    testStartTime = Date.now();
                     segmentKey = test.segment;
-                    console.log(cliFormat_1.CLI_FORMAT_BOLD, "  Segment \"".concat(segmentKey, "\":"));
+                    testResult = {
+                        type: "segment",
+                        key: segmentKey,
+                        // to be updated later
+                        notFound: false,
+                        duration: 0,
+                        passed: true,
+                        assertions: [],
+                    };
                     return [4 /*yield*/, datasource.segmentExists(segmentKey)];
                 case 1:
                     segmentExists = _a.sent();
                     if (!segmentExists) {
-                        console.error(cliFormat_1.CLI_FORMAT_RED, "  Segment does not exist: ".concat(segmentKey));
-                        hasError = true;
-                        return [2 /*return*/, hasError];
+                        testResult.notFound = true;
+                        testResult.passed = false;
+                        return [2 /*return*/, testResult];
                     }
                     return [4 /*yield*/, datasource.readSegment(segmentKey)];
                 case 2:
@@ -64,19 +71,35 @@ function testSegment(datasource, test, patterns) {
                     test.assertions.forEach(function (assertion, aIndex) {
                         var assertions = (0, matrix_1.getSegmentAssertionsFromMatrix)(aIndex, assertion);
                         assertions.forEach(function (assertion) {
+                            var assertionStartTime = Date.now();
+                            var testResultAssertion = {
+                                description: assertion.description,
+                                duration: 0,
+                                passed: true,
+                                errors: [],
+                            };
                             if (patterns.assertionPattern && !patterns.assertionPattern.test(assertion.description)) {
                                 return;
                             }
-                            console.log(assertion.description);
                             var expected = assertion.expectedToMatch;
                             var actual = (0, sdk_1.allConditionsAreMatched)(conditions, assertion.context);
-                            if (actual !== expected) {
-                                hasError = true;
-                                console.error(cliFormat_1.CLI_FORMAT_RED, "    Segment failed: expected \"".concat(expected, "\", got \"").concat(actual, "\""));
+                            var passed = actual === expected;
+                            if (!passed) {
+                                var testResultAssertionError = {
+                                    type: "segment",
+                                    expected: expected,
+                                    actual: actual,
+                                };
+                                testResultAssertion.errors.push(testResultAssertionError);
+                                testResult.passed = false;
+                                testResultAssertion.passed = passed;
                             }
+                            testResult.assertions.push(testResultAssertion);
+                            testResultAssertion.duration = Date.now() - assertionStartTime;
                         });
                     });
-                    return [2 /*return*/, hasError];
+                    testResult.duration = Date.now() - testStartTime;
+                    return [2 /*return*/, testResult];
             }
         });
     });

package/lib/tester/testSegment.js.map

@@ -1 +1 @@
-{"version":3,"file":"testSegment.js","sourceRoot":"","sources":["../../src/tester/testSegment.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,yCAA4D;AAI5D,yCAA8D;AAC9D,mCAA0D;AAE1D,SAAsB,WAAW,CAC/B,UAAsB,EACtB,IAAiB,EACjB,QAAQ;;;;;;oBAEJ,QAAQ,GAAG,KAAK,CAAC;oBAEf,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC;oBAEhC,OAAO,CAAC,GAAG,CAAC,2BAAe,EAAE,sBAAc,UAAU,QAAI,CAAC,CAAC;oBAErC,qBAAM,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,EAAA;;oBAA1D,aAAa,GAAG,SAA0C;oBAEhE,IAAI,CAAC,aAAa,EAAE;wBAClB,OAAO,CAAC,KAAK,CAAC,0BAAc,EAAE,oCAA6B,UAAU,CAAE,CAAC,CAAC;wBACzE,QAAQ,GAAG,IAAI,CAAC;wBAEhB,sBAAO,QAAQ,EAAC;qBACjB;oBAEqB,qBAAM,UAAU,CAAC,WAAW,CAAC,UAAU,CAAC,EAAA;;oBAAxD,aAAa,GAAG,SAAwC;oBACxD,UAAU,GAAG,aAAa,CAAC,UAAqC,CAAC;oBAEvE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,SAAS,EAAE,MAAM;wBACjD,IAAM,UAAU,GAAG,IAAA,uCAA8B,EAAC,MAAM,EAAE,SAAS,CAAC,CAAC;wBAErE,UAAU,CAAC,OAAO,CAAC,UAAU,SAAS;4BACpC,IAAI,QAAQ,CAAC,gBAAgB,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE;gCACvF,OAAO;6BACR;4BAED,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;4BAEnC,IAAM,QAAQ,GAAG,SAAS,CAAC,eAAe,CAAC;4BAC3C,IAAM,MAAM,GAAG,IAAA,6BAAuB,EAAC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;4BAEtE,IAAI,MAAM,KAAK,QAAQ,EAAE;gCACvB,QAAQ,GAAG,IAAI,CAAC;gCAEhB,OAAO,CAAC,KAAK,CACX,0BAAc,EACd,yCAAiC,QAAQ,uBAAW,MAAM,OAAG,CAC9D,CAAC;6BACH;wBACH,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;oBAEH,sBAAO,QAAQ,EAAC;;;;CACjB;AAhDD,kCAgDC"}
+{"version":3,"file":"testSegment.js","sourceRoot":"","sources":["../../src/tester/testSegment.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA,yCAA4D;AAI5D,mCAA0D;AAE1D,SAAsB,WAAW,CAC/B,UAAsB,EACtB,IAAiB,EACjB,QAAQ;;;;;;oBAEF,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC3B,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC;oBAE1B,UAAU,GAAe;wBAC7B,IAAI,EAAE,SAAS;wBACf,GAAG,EAAE,UAAU;wBAEf,sBAAsB;wBACtB,QAAQ,EAAE,KAAK;wBACf,QAAQ,EAAE,CAAC;wBACX,MAAM,EAAE,IAAI;wBACZ,UAAU,EAAE,EAAE;qBACf,CAAC;oBAEoB,qBAAM,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,EAAA;;oBAA1D,aAAa,GAAG,SAA0C;oBAEhE,IAAI,CAAC,aAAa,EAAE;wBAClB,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC;wBAC3B,UAAU,CAAC,MAAM,GAAG,KAAK,CAAC;wBAE1B,sBAAO,UAAU,EAAC;qBACnB;oBAEqB,qBAAM,UAAU,CAAC,WAAW,CAAC,UAAU,CAAC,EAAA;;oBAAxD,aAAa,GAAG,SAAwC;oBACxD,UAAU,GAAG,aAAa,CAAC,UAAqC,CAAC;oBAEvE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,SAAS,EAAE,MAAM;wBACjD,IAAM,UAAU,GAAG,IAAA,uCAA8B,EAAC,MAAM,EAAE,SAAS,CAAC,CAAC;wBAErE,UAAU,CAAC,OAAO,CAAC,UAAU,SAAS;4BACpC,IAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;4BACtC,IAAM,mBAAmB,GAAwB;gCAC/C,WAAW,EAAE,SAAS,CAAC,WAAqB;gCAC5C,QAAQ,EAAE,CAAC;gCACX,MAAM,EAAE,IAAI;gCACZ,MAAM,EAAE,EAAE;6BACX,CAAC;4BAEF,IAAI,QAAQ,CAAC,gBAAgB,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE;gCACvF,OAAO;6BACR;4BAED,IAAM,QAAQ,GAAG,SAAS,CAAC,eAAe,CAAC;4BAC3C,IAAM,MAAM,GAAG,IAAA,6BAAuB,EAAC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;4BACtE,IAAM,MAAM,GAAG,MAAM,KAAK,QAAQ,CAAC;4BAEnC,IAAI,CAAC,MAAM,EAAE;gCACX,IAAM,wBAAwB,GAA6B;oCACzD,IAAI,EAAE,SAAS;oCACf,QAAQ,UAAA;oCACR,MAAM,QAAA;iCACP,CAAC;gCAED,mBAAmB,CAAC,MAAqC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;gCAC1F,UAAU,CAAC,MAAM,GAAG,KAAK,CAAC;gCAC1B,mBAAmB,CAAC,MAAM,GAAG,MAAM,CAAC;6BACrC;4BAED,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;4BAChD,mBAAmB,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;wBACjE,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;oBAEH,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC;oBAEjD,sBAAO,UAAU,EAAC;;;;CACnB;AAvED,kCAuEC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@featurevisor/core",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "description": "Core package of Featurevisor for Node.js usage",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -44,9 +44,9 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@featurevisor/sdk": "^1.2.0",
-    "@featurevisor/site": "^1.2.1",
-    "@featurevisor/types": "^1.2.0",
+    "@featurevisor/sdk": "^1.2.2",
+    "@featurevisor/site": "^1.2.2",
+    "@featurevisor/types": "^1.2.2",
     "axios": "^1.3.4",
     "joi": "^17.8.3",
     "js-yaml": "^4.1.0",
@@ -57,5 +57,5 @@
     "@types/js-yaml": "^4.0.5",
     "@types/tar": "^6.1.4"
   },
-  "gitHead": "2cb917005c4ab458ab4901d5d45a7db55a92a3cd"
+  "gitHead": "65f5422bceb755a6d6a2e5b0c025b41486612c73"
 }

package/src/tester/matrix.ts

@@ -105,7 +105,7 @@ export function getFeatureAssertionsFromMatrix(
 ): FeatureAssertion[] {
   if (!assertionWithMatrix.matrix) {
     const assertion = { ...assertionWithMatrix };
-    assertion.description = `  Assertion #${aIndex + 1}: (${assertion.environment}) ${
+    assertion.description = `Assertion #${aIndex + 1}: (${assertion.environment}) ${
       assertion.description || `at ${assertion.at}%`
     }`;
 
@@ -118,7 +118,7 @@ export function getFeatureAssertionsFromMatrix(
   for (let cIndex = 0; cIndex < combinations.length; cIndex++) {
     const combination = combinations[cIndex];
     const assertion = applyCombinationToFeatureAssertion(combination, assertionWithMatrix);
-    assertion.description = `  Assertion #${aIndex + 1}: (${assertion.environment}) ${
+    assertion.description = `Assertion #${aIndex + 1}: (${assertion.environment}) ${
       assertion.description || `at ${assertion.at}%`
     }`;
 
@@ -161,7 +161,7 @@ export function getSegmentAssertionsFromMatrix(
 ): SegmentAssertion[] {
   if (!assertionWithMatrix.matrix) {
     const assertion = { ...assertionWithMatrix };
-    assertion.description = `  Assertion #${aIndex + 1}: ${
+    assertion.description = `Assertion #${aIndex + 1}: ${
       assertion.description || `#${aIndex + 1}`
     }`;
 
@@ -174,7 +174,7 @@ export function getSegmentAssertionsFromMatrix(
   for (let cIndex = 0; cIndex < combinations.length; cIndex++) {
     const combination = combinations[cIndex];
     const assertion = applyCombinationToSegmentAssertion(combination, assertionWithMatrix);
-    assertion.description = `  Assertion #${aIndex + 1}: ${
+    assertion.description = `Assertion #${aIndex + 1}: ${
       assertion.description || `#${aIndex + 1}`
     }`;
 

package/src/tester/prettyDuration.ts

@@ -0,0 +1,34 @@
+export function prettyDuration(diffInMs) {
+  let diff = Math.abs(diffInMs);
+
+  if (diff === 0) {
+    return `0ms`;
+  }
+
+  const ms = diff % 1000;
+  diff = (diff - ms) / 1000;
+  const secs = diff % 60;
+  diff = (diff - secs) / 60;
+  const mins = diff % 60;
+  const hrs = (diff - mins) / 60;
+
+  let result = "";
+
+  if (hrs) {
+    result += ` ${hrs}h`;
+  }
+
+  if (mins) {
+    result += ` ${mins}m`;
+  }
+
+  if (secs) {
+    result += ` ${secs}s`;
+  }
+
+  if (ms) {
+    result += ` ${ms}ms`;
+  }
+
+  return result.trim();
+}

package/src/tester/printTestResult.ts

@@ -0,0 +1,53 @@
+import { TestResult } from "@featurevisor/types";
+
+import { CLI_FORMAT_BOLD, CLI_FORMAT_RED } from "./cliFormat";
+import { prettyDuration } from "./prettyDuration";
+
+export function printTestResult(testResult: TestResult, testFilePath, rootDirectoryPath) {
+  console.log("");
+
+  const title = `Testing: ${testFilePath.replace(rootDirectoryPath, "")} (${prettyDuration(
+    testResult.duration,
+  )})`;
+  console.log(title);
+
+  if (testResult.notFound) {
+    console.log(CLI_FORMAT_RED, `  => ${testResult.type} ${testResult.key} not found`);
+
+    return;
+  }
+
+  console.log(CLI_FORMAT_BOLD, `  ${testResult.type} "${testResult.key}":`);
+
+  testResult.assertions.forEach(function (assertion) {
+    if (assertion.passed) {
+      console.log(`  ✔ ${assertion.description} (${prettyDuration(assertion.duration)})`);
+    } else {
+      console.log(
+        CLI_FORMAT_RED,
+        `  ✘ ${assertion.description} (${prettyDuration(assertion.duration)})`,
+      );
+
+      assertion.errors?.forEach(function (error) {
+        if (error.message) {
+          console.log(CLI_FORMAT_RED, `    => ${error.message}`);
+
+          return;
+        }
+
+        if (error.type === "variable") {
+          const variableKey = (error.details as any).variableKey;
+
+          console.log(CLI_FORMAT_RED, `    => variable key: ${variableKey}`);
+          console.log(CLI_FORMAT_RED, `       => expected: ${error.expected}`);
+          console.log(CLI_FORMAT_RED, `       => received: ${error.actual}`);
+        } else {
+          console.log(
+            CLI_FORMAT_RED,
+            `    => ${error.type}: expected "${error.expected}", received "${error.actual}"`,
+          );
+        }
+      });
+    }
+  });
+}

package/src/tester/testFeature.ts

@@ -1,4 +1,9 @@
-import { TestFeature } from "@featurevisor/types";
+import {
+  TestFeature,
+  TestResult,
+  TestResultAssertion,
+  TestResultAssertionError,
+} from "@featurevisor/types";
 import { createInstance, MAX_BUCKETED_NUMBER } from "@featurevisor/sdk";
 
 import { Datasource } from "../datasource";
@@ -8,7 +13,6 @@ import { SCHEMA_VERSION } from "../config";
 
 import { checkIfArraysAreEqual } from "./checkIfArraysAreEqual";
 import { checkIfObjectsAreEqual } from "./checkIfObjectsAreEqual";
-import { CLI_FORMAT_BOLD, CLI_FORMAT_RED } from "./cliFormat";
 import { getFeatureAssertionsFromMatrix } from "./matrix";
 
 export async function testFeature(
@@ -17,24 +21,39 @@ export async function testFeature(
   test: TestFeature,
   options: { verbose?: boolean; showDatafile?: boolean } = {},
   patterns,
-): Promise<boolean> {
-  let hasError = false;
+): Promise<TestResult> {
+  const testStartTime = Date.now();
   const featureKey = test.feature;
 
-  console.log(CLI_FORMAT_BOLD, `  Feature "${featureKey}":`);
+  const testResult: TestResult = {
+    type: "feature",
+    key: featureKey,
+
+    // to be updated later
+    notFound: false,
+    duration: 0,
+    passed: true,
+    assertions: [],
+  };
 
   for (let aIndex = 0; aIndex < test.assertions.length; aIndex++) {
     const assertions = getFeatureAssertionsFromMatrix(aIndex, test.assertions[aIndex]);
 
     for (let bIndex = 0; bIndex < assertions.length; bIndex++) {
+      const assertionStartTime = Date.now();
       const assertion = assertions[bIndex];
 
+      const testResultAssertion: TestResultAssertion = {
+        description: assertion.description as string,
+        duration: 0,
+        passed: true,
+        errors: [],
+      };
+
       if (patterns.assertionPattern && !patterns.assertionPattern.test(assertion.description)) {
         continue;
       }
 
-      console.log(assertion.description);
-
       const requiredChain = await datasource.getRequiredFeaturesChain(test.feature);
       const featuresToInclude = Array.from(requiredChain);
 
@@ -68,17 +87,27 @@ export async function testFeature(
         sdk.setLogLevels(["debug", "info", "warn", "error"]);
       }
 
+      const feature = await datasource.readFeature(featureKey);
+      if (!feature) {
+        testResult.notFound = true;
+        testResult.passed = false;
+
+        return testResult;
+      }
+
       // isEnabled
       if ("expectedToBeEnabled" in assertion) {
         const isEnabled = sdk.isEnabled(featureKey, assertion.context);
 
         if (isEnabled !== assertion.expectedToBeEnabled) {
-          hasError = true;
-
-          console.error(
-            CLI_FORMAT_RED,
-            `    isEnabled failed: expected "${assertion.expectedToBeEnabled}", received "${isEnabled}"`,
-          );
+          testResult.passed = false;
+          testResultAssertion.passed = false;
+
+          (testResultAssertion.errors as TestResultAssertionError[]).push({
+            type: "flag",
+            expected: assertion.expectedToBeEnabled,
+            actual: isEnabled,
+          });
         }
       }
 
@@ -87,26 +116,18 @@ export async function testFeature(
         const variation = sdk.getVariation(featureKey, assertion.context);
 
         if (variation !== assertion.expectedVariation) {
-          hasError = true;
-
-          console.error(
-            CLI_FORMAT_RED,
-            `    Variation failed: expected "${assertion.expectedVariation}", received "${variation}"`,
-          );
+          testResult.passed = false;
+          testResultAssertion.passed = false;
+
+          (testResultAssertion.errors as TestResultAssertionError[]).push({
+            type: "variation",
+            expected: assertion.expectedVariation,
+            actual: variation,
+          });
         }
       }
 
       // variables
-      const feature = await datasource.readFeature(featureKey);
-
-      if (!feature) {
-        hasError = true;
-
-        console.error(CLI_FORMAT_RED, `    Feature "${featureKey}" failed: feature not found`);
-
-        continue;
-      }
-
       if (typeof assertion.expectedVariables === "object") {
         Object.keys(assertion.expectedVariables).forEach(function (variableKey) {
           const expectedValue =
@@ -118,12 +139,15 @@ export async function testFeature(
           const variableSchema = feature.variablesSchema?.find((v) => v.key === variableKey);
 
           if (!variableSchema) {
-            hasError = true;
+            testResult.passed = false;
+            testResultAssertion.passed = false;
 
-            console.error(
-              CLI_FORMAT_RED,
-              `    Variable "${variableKey}" failed: variable schema not found in feature`,
-            );
+            (testResultAssertion.errors as TestResultAssertionError[]).push({
+              type: "variable",
+              expected: assertion.expectedVariation,
+              actual: undefined,
+              message: `schema for variable "${variableKey}" not found in feature`,
+            });
 
             return;
           }
@@ -144,12 +168,18 @@ export async function testFeature(
             }
 
             if (!passed) {
-              console.error(CLI_FORMAT_RED, `    Variable "${variableKey}" failed:`);
-              console.error(
-                CLI_FORMAT_RED,
-                `      expected: ${JSON.stringify(parsedExpectedValue)}`,
-              );
-              console.error(CLI_FORMAT_RED, `      received: ${JSON.stringify(actualValue)}`);
+              testResult.passed = false;
+              testResultAssertion.passed = false;
+
+              (testResultAssertion.errors as TestResultAssertionError[]).push({
+                type: "variable",
+                expected:
+                  typeof expectedValue !== "string" ? JSON.stringify(expectedValue) : expectedValue,
+                actual: typeof actualValue !== "string" ? JSON.stringify(actualValue) : actualValue,
+                details: {
+                  variableKey,
+                },
+              });
             }
           } else {
             // other types
@@ -162,19 +192,28 @@ export async function testFeature(
             }
 
             if (!passed) {
-              console.error(CLI_FORMAT_RED, `    Variable "${variableKey}" failed:`);
-              console.error(CLI_FORMAT_RED, `      expected: ${JSON.stringify(expectedValue)}`);
-              console.error(CLI_FORMAT_RED, `      received: ${JSON.stringify(actualValue)}`);
+              testResult.passed = false;
+              testResultAssertion.passed = false;
+
+              (testResultAssertion.errors as TestResultAssertionError[]).push({
+                type: "variable",
+                expected: expectedValue as string,
+                actual: actualValue as string,
+                details: {
+                  variableKey,
+                },
+              });
             }
           }
-
-          if (!passed) {
-            hasError = true;
-          }
         });
       }
+
+      testResultAssertion.duration = Date.now() - assertionStartTime;
+      testResult.assertions.push(testResultAssertion);
     }
   }
 
-  return hasError;
+  testResult.duration = Date.now() - testStartTime;
+
+  return testResult;
 }