@featurevisor/core 1.35.3 → 2.0.1

package/src/tester/testFeature.ts

@@ -1,26 +1,29 @@
-import {
+import type {
   DatafileContent,
   TestFeature,
   TestResult,
   TestResultAssertion,
   TestResultAssertionError,
 } from "@featurevisor/types";
-import { createInstance, MAX_BUCKETED_NUMBER } from "@featurevisor/sdk";
+import {
+  createInstance,
+  FeaturevisorInstance,
+  LogLevel,
+  MAX_BUCKETED_NUMBER,
+  OverrideOptions,
+} from "@featurevisor/sdk";
 
 import { Datasource } from "../datasource";
 import { ProjectConfig } from "../config";
 
-import { checkIfArraysAreEqual } from "./checkIfArraysAreEqual";
-import { checkIfObjectsAreEqual } from "./checkIfObjectsAreEqual";
-import { getFeatureAssertionsFromMatrix } from "./matrix";
+import { checkIfArraysAreEqual, checkIfObjectsAreEqual } from "./helpers";
 import type { DatafileContentByEnvironment } from "./testProject";
 
 export async function testFeature(
   datasource: Datasource,
   projectConfig: ProjectConfig,
   test: TestFeature,
-  options: { verbose?: boolean; showDatafile?: boolean } = {},
-  patterns,
+  options: { verbose?: boolean; quiet?: boolean; showDatafile?: boolean; [key: string]: any } = {},
   datafileContentByEnvironment: DatafileContentByEnvironment,
 ): Promise<TestResult> {
   const testStartTime = Date.now();
@@ -38,166 +41,309 @@ export async function testFeature(
   };
 
   for (let aIndex = 0; aIndex < test.assertions.length; aIndex++) {
-    const assertions = getFeatureAssertionsFromMatrix(aIndex, test.assertions[aIndex]);
+    const assertionStartTime = Date.now();
+    const assertion = test.assertions[aIndex];
+
+    const testResultAssertion: TestResultAssertion = {
+      description: assertion.description as string,
+      duration: 0,
+      passed: true,
+      errors: [],
+    };
+
+    const datafileContent = datafileContentByEnvironment.get(assertion.environment || false);
+
+    if (options.showDatafile) {
+      console.log("");
+      console.log(JSON.stringify(datafileContent, null, 2));
+      console.log("");
+    }
 
-    for (let bIndex = 0; bIndex < assertions.length; bIndex++) {
-      const assertionStartTime = Date.now();
-      const assertion = assertions[bIndex];
+    let logLevel: LogLevel = "warn";
+    if (options.verbose) {
+      logLevel = "debug";
+    } else if (options.quiet) {
+      logLevel = "fatal";
+    }
 
-      const testResultAssertion: TestResultAssertion = {
-        description: assertion.description as string,
-        duration: 0,
-        passed: true,
-        errors: [],
-      };
+    const sdk: FeaturevisorInstance = createInstance({
+      datafile: datafileContent as DatafileContent,
+      sticky: assertion.sticky ? assertion.sticky : {},
+      hooks: [
+        {
+          name: "tester",
+          bucketValue: ({ bucketValue }) => {
+            if (typeof assertion.at !== "undefined") {
+              return assertion.at * (MAX_BUCKETED_NUMBER / 100);
+            }
 
-      if (patterns.assertionPattern && !patterns.assertionPattern.test(assertion.description)) {
-        continue;
-      }
+            return bucketValue;
+          },
+        },
+      ],
+      logLevel,
+    });
+
+    const feature = await datasource.readFeature(featureKey);
+    if (!feature) {
+      testResult.notFound = true;
+      testResult.passed = false;
+
+      return testResult;
+    }
+
+    if (assertion.context) {
+      sdk.setContext(assertion.context);
+    }
+
+    /**
+     * expectedToBeEnabled
+     */
+    function testExpectedToBeEnabled(sdk, assertion, details = {}) {
+      const isEnabled = sdk.isEnabled(featureKey, assertion.context || {});
 
-      const datafileContent = datafileContentByEnvironment.get(assertion.environment || false);
+      if (isEnabled !== assertion.expectedToBeEnabled) {
+        testResult.passed = false;
+        testResultAssertion.passed = false;
 
-      if (options.showDatafile) {
-        console.log("");
-        console.log(JSON.stringify(datafileContent, null, 2));
-        console.log("");
+        (testResultAssertion.errors as TestResultAssertionError[]).push({
+          type: "flag",
+          expected: assertion.expectedToBeEnabled,
+          actual: isEnabled,
+          details,
+        });
       }
+    }
 
-      const sdk = createInstance({
-        datafile: datafileContent,
-        configureBucketValue: () => {
-          return assertion.at * (MAX_BUCKETED_NUMBER / 100);
-        },
-      });
+    if ("expectedToBeEnabled" in assertion) {
+      testExpectedToBeEnabled(sdk, assertion);
+    }
 
-      if (options.verbose) {
-        sdk.setLogLevels(["debug", "info", "warn", "error"]);
+    /**
+     * expectedVariation
+     */
+    function testExpectedVariation(sdk, assertion, details = {}) {
+      const overrideOptions: OverrideOptions = {};
+      if (assertion.defaultVariationValue) {
+        overrideOptions.defaultVariationValue = assertion.defaultVariationValue;
       }
 
-      const feature = await datasource.readFeature(featureKey);
-      if (!feature) {
-        testResult.notFound = true;
+      const variation = sdk.getVariation(featureKey, assertion.context || {}, overrideOptions);
+
+      if (variation !== assertion.expectedVariation) {
         testResult.passed = false;
+        testResultAssertion.passed = false;
 
-        return testResult;
+        (testResultAssertion.errors as TestResultAssertionError[]).push({
+          type: "variation",
+          expected: assertion.expectedVariation,
+          actual: variation,
+          details,
+        });
       }
+    }
 
-      // isEnabled
-      if ("expectedToBeEnabled" in assertion) {
-        const isEnabled = sdk.isEnabled(featureKey, assertion.context);
-
-        if (isEnabled !== assertion.expectedToBeEnabled) {
-          testResult.passed = false;
-          testResultAssertion.passed = false;
+    if ("expectedVariation" in assertion) {
+      testExpectedVariation(sdk, assertion);
+    }
 
-          (testResultAssertion.errors as TestResultAssertionError[]).push({
-            type: "flag",
-            expected: assertion.expectedToBeEnabled,
-            actual: isEnabled,
-          });
+    /**
+     * expectedVariables
+     */
+    function testExpectedVariables(sdk, assertion, details = {}) {
+      Object.keys(assertion.expectedVariables).forEach(function (variableKey) {
+        const expectedValue =
+          assertion.expectedVariables && assertion.expectedVariables[variableKey];
+
+        const overrideOptions: OverrideOptions = {};
+        if (assertion.defaultVariableValues && assertion.defaultVariableValues[variableKey]) {
+          overrideOptions.defaultVariableValue = assertion.defaultVariableValues[variableKey];
         }
-      }
 
-      // variation
-      if ("expectedVariation" in assertion) {
-        const variation = sdk.getVariation(featureKey, assertion.context);
+        const actualValue = sdk.getVariable(
+          featureKey,
+          variableKey,
+          assertion.context || {},
+          overrideOptions,
+        );
+
+        let passed;
+
+        const variableSchema = feature.variablesSchema?.[variableKey];
 
-        if (variation !== assertion.expectedVariation) {
+        if (!variableSchema) {
           testResult.passed = false;
           testResultAssertion.passed = false;
 
           (testResultAssertion.errors as TestResultAssertionError[]).push({
-            type: "variation",
+            type: "variable",
             expected: assertion.expectedVariation,
-            actual: variation,
+            actual: undefined,
+            message: `schema for variable "${variableKey}" not found in feature`,
           });
-        }
-      }
 
-      // variables
-      if (typeof assertion.expectedVariables === "object") {
-        Object.keys(assertion.expectedVariables).forEach(function (variableKey) {
-          const expectedValue =
-            assertion.expectedVariables && assertion.expectedVariables[variableKey];
-          const actualValue = sdk.getVariable(featureKey, variableKey, assertion.context);
+          return;
+        }
 
-          let passed;
+        if (variableSchema.type === "json") {
+          // JSON type
+          const parsedExpectedValue =
+            typeof expectedValue === "string" ? JSON.parse(expectedValue as string) : expectedValue;
 
-          const variableSchema = feature.variablesSchema?.find((v) => v.key === variableKey);
+          if (Array.isArray(actualValue)) {
+            passed = checkIfArraysAreEqual(parsedExpectedValue, actualValue);
+          } else if (typeof actualValue === "object") {
+            passed = checkIfObjectsAreEqual(parsedExpectedValue, actualValue);
+          } else {
+            passed = JSON.stringify(parsedExpectedValue) === JSON.stringify(actualValue);
+          }
 
-          if (!variableSchema) {
+          if (!passed) {
             testResult.passed = false;
             testResultAssertion.passed = false;
 
             (testResultAssertion.errors as TestResultAssertionError[]).push({
               type: "variable",
-              expected: assertion.expectedVariation,
-              actual: undefined,
-              message: `schema for variable "${variableKey}" not found in feature`,
+              expected:
+                typeof expectedValue !== "string" ? JSON.stringify(expectedValue) : expectedValue,
+              actual: typeof actualValue !== "string" ? JSON.stringify(actualValue) : actualValue,
+              details: {
+                ...details,
+                variableKey,
+              },
             });
+          }
+        } else {
+          // other types
+          if (typeof expectedValue === "object") {
+            passed = checkIfObjectsAreEqual(expectedValue, actualValue);
+          } else if (Array.isArray(expectedValue)) {
+            passed = checkIfArraysAreEqual(expectedValue, actualValue);
+          } else {
+            passed = expectedValue === actualValue;
+          }
 
-            return;
+          if (!passed) {
+            testResult.passed = false;
+            testResultAssertion.passed = false;
+
+            (testResultAssertion.errors as TestResultAssertionError[]).push({
+              type: "variable",
+              expected: expectedValue as string,
+              actual: actualValue as string,
+              details: {
+                ...details,
+                variableKey,
+              },
+            });
           }
+        }
+      });
+    }
 
-          if (variableSchema.type === "json") {
-            // JSON type
-            const parsedExpectedValue =
-              typeof expectedValue === "string"
-                ? JSON.parse(expectedValue as string)
-                : expectedValue;
-
-            if (Array.isArray(actualValue)) {
-              passed = checkIfArraysAreEqual(parsedExpectedValue, actualValue);
-            } else if (typeof actualValue === "object") {
-              passed = checkIfObjectsAreEqual(parsedExpectedValue, actualValue);
-            } else {
-              passed = JSON.stringify(parsedExpectedValue) === JSON.stringify(actualValue);
-            }
+    if (typeof assertion.expectedVariables === "object") {
+      testExpectedVariables(sdk, assertion);
+    }
 
-            if (!passed) {
-              testResult.passed = false;
-              testResultAssertion.passed = false;
-
-              (testResultAssertion.errors as TestResultAssertionError[]).push({
-                type: "variable",
-                expected:
-                  typeof expectedValue !== "string" ? JSON.stringify(expectedValue) : expectedValue,
-                actual: typeof actualValue !== "string" ? JSON.stringify(actualValue) : actualValue,
-                details: {
-                  variableKey,
-                },
-              });
-            }
-          } else {
-            // other types
-            if (typeof expectedValue === "object") {
-              passed = checkIfObjectsAreEqual(expectedValue, actualValue);
-            } else if (Array.isArray(expectedValue)) {
-              passed = checkIfArraysAreEqual(expectedValue, actualValue);
-            } else {
-              passed = expectedValue === actualValue;
-            }
+    /**
+     * expectedEvaluations
+     */
+    function testExpectedEvaluations(sdk, assertion, rootDetails = {}) {
+      function testEvaluation(type, evaluation, expected, details = {}) {
+        for (const [key, value] of Object.entries(expected)) {
+          if (evaluation[key] !== value) {
+            testResult.passed = false;
+            testResultAssertion.passed = false;
 
-            if (!passed) {
-              testResult.passed = false;
-              testResultAssertion.passed = false;
-
-              (testResultAssertion.errors as TestResultAssertionError[]).push({
-                type: "variable",
-                expected: expectedValue as string,
-                actual: actualValue as string,
-                details: {
-                  variableKey,
-                },
-              });
-            }
+            (testResultAssertion.errors as TestResultAssertionError[]).push({
+              type: "evaluation",
+              expected: value as string | number | boolean | null | undefined,
+              actual: evaluation[key],
+              details: {
+                ...rootDetails,
+                ...details,
+                evaluationType: type,
+                evaluationKey: key,
+              },
+            });
           }
-        });
+        }
       }
 
-      testResultAssertion.duration = Date.now() - assertionStartTime;
-      testResult.assertions.push(testResultAssertion);
+      if (assertion.expectedEvaluations.flag) {
+        const evaluation = sdk.evaluateFlag(featureKey, assertion.context || {});
+        testEvaluation("flag", evaluation, assertion.expectedEvaluations.flag);
+      }
+
+      if (assertion.expectedEvaluations.variation) {
+        const evaluation = sdk.evaluateVariation(featureKey, assertion.context || {});
+        testEvaluation("variation", evaluation, assertion.expectedEvaluations.variation);
+      }
+
+      if (assertion.expectedEvaluations.variables) {
+        const variableKeys = Object.keys(assertion.expectedEvaluations.variables);
+
+        for (const variableKey of variableKeys) {
+          const evaluation = sdk.evaluateVariable(featureKey, variableKey, assertion.context || {});
+          testEvaluation(
+            "variable",
+            evaluation,
+            assertion.expectedEvaluations.variables[variableKey],
+            { variableKey },
+          );
+        }
+      }
     }
+
+    if (assertion.expectedEvaluations) {
+      testExpectedEvaluations(sdk, assertion);
+    }
+
+    /**
+     * children
+     */
+    if (Array.isArray(assertion.children)) {
+      let childIndex = 0;
+
+      for (const child of assertion.children) {
+        const childSdk = sdk.spawn(child.context || {}, {
+          sticky: assertion.sticky || {},
+        });
+
+        // expectedToBeEnabled
+        if (typeof child.expectedToBeEnabled !== "undefined") {
+          testExpectedToBeEnabled(childSdk, child, {
+            childIndex,
+          });
+        }
+
+        // expectedVariation
+        if (typeof child.expectedVariation !== "undefined") {
+          testExpectedVariation(childSdk, child, {
+            childIndex,
+          });
+        }
+
+        // expectedVariables
+        if (typeof child.expectedVariables === "object") {
+          testExpectedVariables(childSdk, child, {
+            childIndex,
+          });
+        }
+
+        // expectedEvaluations
+        if (typeof child.expectedEvaluations === "object") {
+          testExpectedEvaluations(childSdk, child, {
+            childIndex,
+          });
+        }
+
+        childIndex++;
+      }
+    }
+
+    testResultAssertion.duration = Date.now() - assertionStartTime;
+    testResult.assertions.push(testResultAssertion);
   }
 
   testResult.duration = Date.now() - testStartTime;

package/src/tester/testProject.ts

@@ -1,7 +1,6 @@
 import * as fs from "fs";
-import * as path from "path";
 
-import { TestSegment, TestFeature, DatafileContent } from "@featurevisor/types";
+import type { TestSegment, TestFeature, Test, DatafileContent } from "@featurevisor/types";
 
 import { testSegment } from "./testSegment";
 import { testFeature } from "./testFeature";
@@ -13,6 +12,7 @@ import { printTestResult } from "./printTestResult";
 import { buildDatafile } from "../builder";
 import { SCHEMA_VERSION } from "../config";
 import { Plugin } from "../cli";
+import { listEntities } from "../list";
 
 export interface TestProjectOptions {
   keyPattern?: string;
@@ -24,11 +24,6 @@ export interface TestProjectOptions {
   inflate?: number;
 }
 
-export interface TestPatterns {
-  keyPattern?: RegExp;
-  assertionPattern?: RegExp;
-}
-
 export interface ExecutionResult {
   passed: boolean;
   assertionsCount: {
@@ -40,22 +35,18 @@ export interface ExecutionResult {
 export type DatafileContentByEnvironment = Map<string | false, DatafileContent>;
 
 export async function executeTest(
-  testFile: string,
+  test: Test,
   deps: Dependencies,
   options: TestProjectOptions,
-  patterns: TestPatterns,
   datafileContentByEnvironment: DatafileContentByEnvironment,
 ): Promise<ExecutionResult | undefined> {
   const { datasource, projectConfig, rootDirectoryPath } = deps;
 
-  const relativeTestFilePath = path
-    .join(projectConfig.testsDirectoryPath, datasource.getTestSpecName(testFile))
-    .replace(rootDirectoryPath + path.sep, "");
-
-  const t = await datasource.readTest(testFile);
+  const extension = datasource.getExtension();
+  const relativeTestFilePath = test.key + (extension ? `.${extension}` : "");
 
-  const tAsSegment = t as TestSegment;
-  const tAsFeature = t as TestFeature;
+  const tAsSegment = test as TestSegment;
+  const tAsFeature = test as TestFeature;
   const key = tAsSegment.segment || tAsFeature.feature;
   const type = tAsSegment.segment ? "segment" : "feature";
 
@@ -68,26 +59,21 @@ export async function executeTest(
   };
 
   if (!key) {
-    console.error(`  => Invalid test: ${JSON.stringify(t)}`);
+    console.error(`  => Invalid test: ${JSON.stringify(test)}`);
     executionResult.passed = false;
 
     return executionResult;
   }
 
-  if (patterns.keyPattern && !patterns.keyPattern.test(key)) {
-    return;
-  }
-
   let testResult;
   if (type === "segment") {
-    testResult = await testSegment(datasource, tAsSegment, patterns);
+    testResult = await testSegment(datasource, tAsSegment, options);
   } else {
     testResult = await testFeature(
       datasource,
       projectConfig,
       tAsFeature,
       options,
-      patterns,
       datafileContentByEnvironment,
     );
   }
@@ -117,9 +103,9 @@ export async function executeTest(
 
 export async function testProject(
   deps: Dependencies,
-  options: TestProjectOptions = {},
+  testOptions: TestProjectOptions = {},
 ): Promise<boolean> {
-  const { projectConfig, datasource } = deps;
+  const { rootDirectoryPath, projectConfig, datasource, options } = deps;
 
   let hasError = false;
 
@@ -130,22 +116,8 @@ export async function testProject(
     return hasError;
   }
 
-  const testFiles = await datasource.listTests();
-
-  if (testFiles.length === 0) {
-    console.error(`No tests found in: ${projectConfig.testsDirectoryPath}`);
-    hasError = true;
-
-    return hasError;
-  }
-
   const startTime = Date.now();
 
-  const patterns: TestPatterns = {
-    keyPattern: options.keyPattern ? new RegExp(options.keyPattern) : undefined,
-    assertionPattern: options.assertionPattern ? new RegExp(options.assertionPattern) : undefined,
-  };
-
   let passedTestsCount = 0;
   let failedTestsCount = 0;
 
@@ -170,7 +142,7 @@ export async function testProject(
         existingState,
       );
 
-      datafileContentByEnvironment.set(environment, datafileContent);
+      datafileContentByEnvironment.set(environment, datafileContent as DatafileContent);
     }
   }
 
@@ -189,17 +161,24 @@ export async function testProject(
       existingState,
     );
 
-    datafileContentByEnvironment.set(false, datafileContent);
+    datafileContentByEnvironment.set(false, datafileContent as DatafileContent);
   }
 
-  for (const testFile of testFiles) {
-    const executionResult = await executeTest(
-      testFile,
-      deps,
-      options,
-      patterns,
-      datafileContentByEnvironment,
-    );
+  const tests = await listEntities<Test>(
+    {
+      rootDirectoryPath,
+      projectConfig,
+      datasource,
+      options: {
+        ...testOptions,
+        applyMatrix: true,
+      },
+    },
+    "test",
+  );
+
+  for (const test of tests) {
+    const executionResult = await executeTest(test, deps, options, datafileContentByEnvironment);
 
     if (!executionResult) {
       continue;