@featurevisor/core 0.47.7 → 0.48.0

package/src/linter.ts

@@ -1,13 +1,12 @@
 // for use in node only
 import * as fs from "fs";
-import * as path from "path";
 
 import * as Joi from "joi";
 
-import { getYAMLFiles, parseYaml } from "./utils";
+import { Datasource } from "./datasource/datasource";
 
 import { ProjectConfig } from "./config";
-import { ParsedFeature, FeatureKey, Required } from "@featurevisor/types";
+import { FeatureKey, Required } from "@featurevisor/types";
 
 export function getAttributeJoiSchema() {
   const attributeJoiSchema = Joi.object({
@@ -112,7 +111,11 @@ export function getSegmentJoiSchema(projectConfig: ProjectConfig, conditionsJoiS
   return segmentJoiSchema;
 }
 
-export function getGroupJoiSchema(projectConfig: ProjectConfig, availableFeatureKeys: string[]) {
+export function getGroupJoiSchema(
+  projectConfig: ProjectConfig,
+  datasource: Datasource,
+  availableFeatureKeys: string[],
+) {
   const groupJoiSchema = Joi.object({
     description: Joi.string().required(),
     slots: Joi.array()
@@ -134,13 +137,14 @@ export function getGroupJoiSchema(projectConfig: ProjectConfig, availableFeature
 
           if (slot.feature) {
             const featureKey = slot.feature;
-            const featurePath = path.join(projectConfig.featuresDirectoryPath, `${featureKey}.yml`);
-            const parsedFeature = parseYaml(fs.readFileSync(featurePath, "utf8")) as ParsedFeature;
+            const featureExists = datasource.entityExists("feature", featureKey);
 
-            if (!parsedFeature) {
+            if (!featureExists) {
               throw new Error(`feature ${featureKey} not found`);
             }
 
+            const parsedFeature = datasource.readFeature(featureKey);
+
             const environmentKeys = Object.keys(parsedFeature.environments);
             for (const environmentKey of environmentKeys) {
               const environment = parsedFeature.environments[environmentKey];
@@ -439,7 +443,7 @@ export function printJoiError(e: Joi.ValidationError) {
 }
 
 function checkForCircularDependencyInRequired(
-  featuresDirectoryPath: string,
+  datasource: Datasource,
   featureKey: FeatureKey,
   required?: Required[],
   chain: FeatureKey[] = [],
@@ -461,19 +465,17 @@ function checkForCircularDependencyInRequired(
       throw new Error(`circular dependency found: ${chain.join(" -> ")}`);
     }
 
-    const requiredFeaturePath = path.join(featuresDirectoryPath, `${requiredKey}.yml`);
+    const requiredFeatureExists = datasource.entityExists("feature", requiredKey);
 
-    if (!fs.existsSync(requiredFeaturePath)) {
+    if (!requiredFeatureExists) {
       throw new Error(`required feature "${requiredKey}" not found`);
     }
 
-    const requiredParsedFeature = parseYaml(
-      fs.readFileSync(requiredFeaturePath, "utf8"),
-    ) as ParsedFeature;
+    const requiredParsedFeature = datasource.readFeature(requiredKey);
 
     if (requiredParsedFeature.required) {
       checkForCircularDependencyInRequired(
-        featuresDirectoryPath,
+        datasource,
         featureKey,
         requiredParsedFeature.required,
         chain,
@@ -484,19 +486,20 @@ function checkForCircularDependencyInRequired(
 
 export async function lintProject(projectConfig: ProjectConfig): Promise<boolean> {
   let hasError = false;
+  const datasource = new Datasource(projectConfig);
 
   const availableAttributeKeys: string[] = [];
   const availableSegmentKeys: string[] = [];
   const availableFeatureKeys: string[] = [];
 
   // lint attributes
-  console.log("Linting attributes...\n");
-  const attributeFilePaths = getYAMLFiles(path.join(projectConfig.attributesDirectoryPath));
+  const attributes = datasource.listAttributes();
+  console.log(`Linting ${attributes.length} attributes...\n`);
+
   const attributeJoiSchema = getAttributeJoiSchema();
 
-  for (const filePath of attributeFilePaths) {
-    const key = path.basename(filePath, ".yml");
-    const parsed = parseYaml(fs.readFileSync(filePath, "utf8")) as any;
+  for (const key of attributes) {
+    const parsed = datasource.readAttribute(key);
     availableAttributeKeys.push(key);
 
     try {
@@ -515,14 +518,14 @@ export async function lintProject(projectConfig: ProjectConfig): Promise<boolean
   }
 
   // lint segments
-  console.log("\nLinting segments...\n");
-  const segmentFilePaths = getYAMLFiles(path.join(projectConfig.segmentsDirectoryPath));
+  const segments = datasource.listSegments();
+  console.log(`\nLinting ${segments.length} segments...\n`);
+
   const conditionsJoiSchema = getConditionsJoiSchema(projectConfig, availableAttributeKeys);
   const segmentJoiSchema = getSegmentJoiSchema(projectConfig, conditionsJoiSchema);
 
-  for (const filePath of segmentFilePaths) {
-    const key = path.basename(filePath, ".yml");
-    const parsed = parseYaml(fs.readFileSync(filePath, "utf8")) as any;
+  for (const key of segments) {
+    const parsed = datasource.readSegment(key);
     availableSegmentKeys.push(key);
 
     try {
@@ -541,14 +544,16 @@ export async function lintProject(projectConfig: ProjectConfig): Promise<boolean
   }
 
   // lint groups
-  console.log("\nLinting groups...\n");
+
   if (fs.existsSync(projectConfig.groupsDirectoryPath)) {
-    const groupFilePaths = getYAMLFiles(path.join(projectConfig.groupsDirectoryPath));
-    const groupJoiSchema = getGroupJoiSchema(projectConfig, availableFeatureKeys);
+    const groups = datasource.listGroups();
+    console.log(`\nLinting ${groups.length} groups...\n`);
+
+    // @TODO: feature it slots can be from availableFeatureKeys only
+    const groupJoiSchema = getGroupJoiSchema(projectConfig, datasource, availableFeatureKeys);
 
-    for (const filePath of groupFilePaths) {
-      const key = path.basename(filePath, ".yml");
-      const parsed = parseYaml(fs.readFileSync(filePath, "utf8")) as any;
+    for (const key of groups) {
+      const parsed = datasource.readGroup(key);
 
       try {
         await groupJoiSchema.validateAsync(parsed);
@@ -569,8 +574,9 @@ export async function lintProject(projectConfig: ProjectConfig): Promise<boolean
   // @TODO: feature cannot exist in multiple groups
 
   // lint features
-  console.log("\nLinting features...\n");
-  const featureFilePaths = getYAMLFiles(path.join(projectConfig.featuresDirectoryPath));
+  const features = datasource.listFeatures();
+  console.log(`\nLinting ${features.length} features...\n`);
+
   const featureJoiSchema = getFeatureJoiSchema(
     projectConfig,
     conditionsJoiSchema,
@@ -578,9 +584,8 @@ export async function lintProject(projectConfig: ProjectConfig): Promise<boolean
     availableFeatureKeys,
   );
 
-  for (const filePath of featureFilePaths) {
-    const key = path.basename(filePath, ".yml");
-    const parsed = parseYaml(fs.readFileSync(filePath, "utf8")) as any;
+  for (const key of features) {
+    const parsed = datasource.readFeature(key);
     availableFeatureKeys.push(key);
 
     try {
@@ -599,11 +604,7 @@ export async function lintProject(projectConfig: ProjectConfig): Promise<boolean
 
     if (parsed.required) {
       try {
-        checkForCircularDependencyInRequired(
-          projectConfig.featuresDirectoryPath,
-          key,
-          parsed.required,
-        );
+        checkForCircularDependencyInRequired(datasource, key, parsed.required);
       } catch (e) {
         console.log("  =>", key);
         console.log("     => Error:", e.message);
@@ -613,18 +614,18 @@ export async function lintProject(projectConfig: ProjectConfig): Promise<boolean
   }
 
   // lint tests
-  console.log("\nLinting tests...\n");
   if (fs.existsSync(projectConfig.testsDirectoryPath)) {
-    const testFilePaths = getYAMLFiles(path.join(projectConfig.testsDirectoryPath));
+    const tests = datasource.listTests();
+    console.log(`\nLinting ${tests.length} tests...\n`);
+
     const testsJoiSchema = getTestsJoiSchema(
       projectConfig,
       availableFeatureKeys,
       availableSegmentKeys,
     );
 
-    for (const filePath of testFilePaths) {
-      const key = path.basename(filePath, ".yml");
-      const parsed = parseYaml(fs.readFileSync(filePath, "utf8")) as any;
+    for (const key of tests) {
+      const parsed = datasource.readTest(key);
 
       try {
         await testsJoiSchema.validateAsync(parsed);

package/src/site.ts

@@ -6,9 +6,6 @@ import { execSync } from "child_process";
 import * as mkdirp from "mkdirp";
 
 import {
-  Attribute,
-  ParsedFeature,
-  Segment,
   HistoryEntry,
   LastModified,
   SearchIndex,
@@ -19,11 +16,8 @@ import {
 } from "@featurevisor/types";
 
 import { ProjectConfig } from "./config";
-import {
-  parseYaml,
-  extractAttributeKeysFromConditions,
-  extractSegmentKeysFromGroupSegments,
-} from "./utils";
+import { Datasource } from "./datasource/datasource";
+import { extractAttributeKeysFromConditions, extractSegmentKeysFromGroupSegments } from "./utils";
 
 function getRelativePaths(rootDirectoryPath, projectConfig: ProjectConfig) {
   const relativeFeaturesPath = path.relative(
@@ -104,7 +98,7 @@ export function generateHistory(rootDirectoryPath, projectConfig: ProjectConfig)
         const fileName = lineSplit.pop() as string;
         const relativeDir = lineSplit.join(path.sep);
 
-        const key = fileName.replace(".yml", "");
+        const key = fileName.replace("." + projectConfig.parser, "");
 
         let type = "feature" as "attribute" | "segment" | "feature";
 
@@ -249,6 +243,7 @@ export function generateSiteSearchIndex(
       features: [],
     },
   };
+  const datasource = new Datasource(projectConfig);
 
   /**
    * Links
@@ -267,15 +262,15 @@ export function generateSiteSearchIndex(
     result.links = {
       attribute: repoDetails.blobUrl.replace(
         "{{blobPath}}",
-        prefix + relativeAttributesPath + "/{{key}}.yml",
+        prefix + relativeAttributesPath + "/{{key}}." + datasource.getExtension(),
       ),
       segment: repoDetails.blobUrl.replace(
         "{{blobPath}}",
-        prefix + relativeSegmentsPath + "/{{key}}.yml",
+        prefix + relativeSegmentsPath + "/{{key}}." + datasource.getExtension(),
       ),
       feature: repoDetails.blobUrl.replace(
         "{{blobPath}}",
-        prefix + relativeFeaturesPath + "/{{key}}.yml",
+        prefix + relativeFeaturesPath + "/{{key}}." + datasource.getExtension(),
       ),
       commit: repoDetails.commitUrl,
     };
@@ -296,145 +291,127 @@ export function generateSiteSearchIndex(
   } = {};
 
   // features
-  const featureFiles = fs.readdirSync(projectConfig.featuresDirectoryPath);
-  featureFiles
-    .filter((fileName) => fileName.endsWith(".yml"))
-    .forEach((fileName) => {
-      const filePath = path.join(projectConfig.featuresDirectoryPath, fileName);
-      const entityName = fileName.replace(".yml", "");
-
-      const fileContent = fs.readFileSync(filePath, "utf8");
-      const parsed = parseYaml(fileContent) as ParsedFeature;
-
-      if (Array.isArray(parsed.variations)) {
-        parsed.variations.forEach((variation) => {
-          if (!variation.variables) {
-            return;
-          }
+  const featureFiles = datasource.listFeatures();
+  featureFiles.forEach((entityName) => {
+    const parsed = datasource.readFeature(entityName);
+
+    if (Array.isArray(parsed.variations)) {
+      parsed.variations.forEach((variation) => {
+        if (!variation.variables) {
+          return;
+        }
 
-          variation.variables.forEach((v) => {
-            if (v.overrides) {
-              v.overrides.forEach((o) => {
-                if (o.conditions) {
-                  extractAttributeKeysFromConditions(o.conditions).forEach((attributeKey) => {
-                    if (!attributesUsedInFeatures[attributeKey]) {
-                      attributesUsedInFeatures[attributeKey] = new Set();
-                    }
-
-                    attributesUsedInFeatures[attributeKey].add(entityName);
-                  });
-                }
-
-                if (o.segments && o.segments !== "*") {
-                  extractSegmentKeysFromGroupSegments(o.segments).forEach((segmentKey) => {
-                    if (!segmentsUsedInFeatures[segmentKey]) {
-                      segmentsUsedInFeatures[segmentKey] = new Set();
-                    }
-
-                    segmentsUsedInFeatures[segmentKey].add(entityName);
-                  });
-                }
-              });
-            }
-          });
-        });
-      }
+        variation.variables.forEach((v) => {
+          if (v.overrides) {
+            v.overrides.forEach((o) => {
+              if (o.conditions) {
+                extractAttributeKeysFromConditions(o.conditions).forEach((attributeKey) => {
+                  if (!attributesUsedInFeatures[attributeKey]) {
+                    attributesUsedInFeatures[attributeKey] = new Set();
+                  }
+
+                  attributesUsedInFeatures[attributeKey].add(entityName);
+                });
+              }
 
-      Object.keys(parsed.environments).forEach((environmentKey) => {
-        const env = parsed.environments[environmentKey];
+              if (o.segments && o.segments !== "*") {
+                extractSegmentKeysFromGroupSegments(o.segments).forEach((segmentKey) => {
+                  if (!segmentsUsedInFeatures[segmentKey]) {
+                    segmentsUsedInFeatures[segmentKey] = new Set();
+                  }
 
-        env.rules.forEach((rule) => {
-          if (rule.segments && rule.segments !== "*") {
-            extractSegmentKeysFromGroupSegments(rule.segments).forEach((segmentKey) => {
-              if (!segmentsUsedInFeatures[segmentKey]) {
-                segmentsUsedInFeatures[segmentKey] = new Set();
+                  segmentsUsedInFeatures[segmentKey].add(entityName);
+                });
               }
-
-              segmentsUsedInFeatures[segmentKey].add(entityName);
             });
           }
         });
+      });
+    }
 
-        if (env.force) {
-          env.force.forEach((force) => {
-            if (force.segments && force.segments !== "*") {
-              extractSegmentKeysFromGroupSegments(force.segments).forEach((segmentKey) => {
-                if (!segmentsUsedInFeatures[segmentKey]) {
-                  segmentsUsedInFeatures[segmentKey] = new Set();
-                }
+    Object.keys(parsed.environments).forEach((environmentKey) => {
+      const env = parsed.environments[environmentKey];
 
-                segmentsUsedInFeatures[segmentKey].add(entityName);
-              });
+      env.rules.forEach((rule) => {
+        if (rule.segments && rule.segments !== "*") {
+          extractSegmentKeysFromGroupSegments(rule.segments).forEach((segmentKey) => {
+            if (!segmentsUsedInFeatures[segmentKey]) {
+              segmentsUsedInFeatures[segmentKey] = new Set();
             }
 
-            if (force.conditions) {
-              extractAttributeKeysFromConditions(force.conditions).forEach((attributeKey) => {
-                if (!attributesUsedInFeatures[attributeKey]) {
-                  attributesUsedInFeatures[attributeKey] = new Set();
-                }
-
-                attributesUsedInFeatures[attributeKey].add(entityName);
-              });
-            }
+            segmentsUsedInFeatures[segmentKey].add(entityName);
           });
         }
       });
 
-      result.entities.features.push({
-        ...parsed,
-        key: entityName,
-        lastModified: getLastModifiedFromHistory(fullHistory, "feature", entityName),
-      });
+      if (env.force) {
+        env.force.forEach((force) => {
+          if (force.segments && force.segments !== "*") {
+            extractSegmentKeysFromGroupSegments(force.segments).forEach((segmentKey) => {
+              if (!segmentsUsedInFeatures[segmentKey]) {
+                segmentsUsedInFeatures[segmentKey] = new Set();
+              }
+
+              segmentsUsedInFeatures[segmentKey].add(entityName);
+            });
+          }
+
+          if (force.conditions) {
+            extractAttributeKeysFromConditions(force.conditions).forEach((attributeKey) => {
+              if (!attributesUsedInFeatures[attributeKey]) {
+                attributesUsedInFeatures[attributeKey] = new Set();
+              }
+
+              attributesUsedInFeatures[attributeKey].add(entityName);
+            });
+          }
+        });
+      }
     });
 
+    result.entities.features.push({
+      ...parsed,
+      key: entityName,
+      lastModified: getLastModifiedFromHistory(fullHistory, "feature", entityName),
+    });
+  });
+
   // segments
-  const segmentFiles = fs.readdirSync(projectConfig.segmentsDirectoryPath);
-  segmentFiles
-    .filter((fileName) => fileName.endsWith(".yml"))
-    .forEach((fileName) => {
-      const filePath = path.join(projectConfig.segmentsDirectoryPath, fileName);
-      const entityName = fileName.replace(".yml", "");
-
-      const fileContent = fs.readFileSync(filePath, "utf8");
-      const parsed = parseYaml(fileContent) as Segment;
-
-      extractAttributeKeysFromConditions(parsed.conditions as Condition | Condition[]).forEach(
-        (attributeKey) => {
-          if (!attributesUsedInSegments[attributeKey]) {
-            attributesUsedInSegments[attributeKey] = new Set();
-          }
+  const segmentFiles = datasource.listSegments();
+  segmentFiles.forEach((entityName) => {
+    const parsed = datasource.readSegment(entityName);
+
+    extractAttributeKeysFromConditions(parsed.conditions as Condition | Condition[]).forEach(
+      (attributeKey) => {
+        if (!attributesUsedInSegments[attributeKey]) {
+          attributesUsedInSegments[attributeKey] = new Set();
+        }
 
-          attributesUsedInSegments[attributeKey].add(entityName);
-        },
-      );
+        attributesUsedInSegments[attributeKey].add(entityName);
+      },
+    );
 
-      result.entities.segments.push({
-        ...parsed,
-        key: entityName,
-        lastModified: getLastModifiedFromHistory(fullHistory, "segment", entityName),
-        usedInFeatures: Array.from(segmentsUsedInFeatures[entityName] || []),
-      });
+    result.entities.segments.push({
+      ...parsed,
+      key: entityName,
+      lastModified: getLastModifiedFromHistory(fullHistory, "segment", entityName),
+      usedInFeatures: Array.from(segmentsUsedInFeatures[entityName] || []),
     });
+  });
 
   // attributes
-  const attributeFiles = fs.readdirSync(projectConfig.attributesDirectoryPath);
-  attributeFiles
-    .filter((fileName) => fileName.endsWith(".yml"))
-    .forEach((fileName) => {
-      const filePath = path.join(projectConfig.attributesDirectoryPath, fileName);
-      const entityName = fileName.replace(".yml", "");
-
-      const fileContent = fs.readFileSync(filePath, "utf8");
-      const parsed = parseYaml(fileContent) as Attribute;
-
-      result.entities.attributes.push({
-        ...parsed,
-        key: entityName,
-        lastModified: getLastModifiedFromHistory(fullHistory, "attribute", entityName),
-        usedInFeatures: Array.from(attributesUsedInFeatures[entityName] || []),
-        usedInSegments: Array.from(attributesUsedInSegments[entityName] || []),
-      });
+  const attributeFiles = datasource.listAttributes();
+  attributeFiles.forEach((entityName) => {
+    const parsed = datasource.readAttribute(entityName);
+
+    result.entities.attributes.push({
+      ...parsed,
+      key: entityName,
+      lastModified: getLastModifiedFromHistory(fullHistory, "attribute", entityName),
+      usedInFeatures: Array.from(attributesUsedInFeatures[entityName] || []),
+      usedInSegments: Array.from(attributesUsedInSegments[entityName] || []),
     });
+  });
 
   return result;
 }

package/src/tester.ts

@@ -1,12 +1,11 @@
 import * as fs from "fs";
-import * as path from "path";
 
-import { DatafileContent, Spec, Segment, Condition } from "@featurevisor/types";
+import { DatafileContent, Condition } from "@featurevisor/types";
 import { createInstance, allConditionsAreMatched, MAX_BUCKETED_NUMBER } from "@featurevisor/sdk";
 
 import { ProjectConfig } from "./config";
-import { parseYaml } from "./utils";
 import { getDatafilePath } from "./builder";
+import { Datasource } from "./datasource/datasource";
 
 // @TODO: make it better
 export function checkIfArraysAreEqual(a, b) {
@@ -49,6 +48,7 @@ export function checkIfObjectsAreEqual(a, b) {
 
 export function testProject(rootDirectoryPath: string, projectConfig: ProjectConfig): boolean {
   let hasError = false;
+  const datasource = new Datasource(projectConfig);
 
   if (!fs.existsSync(projectConfig.testsDirectoryPath)) {
     console.error(`Tests directory does not exist: ${projectConfig.testsDirectoryPath}`);
@@ -57,9 +57,7 @@ export function testProject(rootDirectoryPath: string, projectConfig: ProjectCon
     return hasError;
   }
 
-  const testFiles = fs
-    .readdirSync(projectConfig.testsDirectoryPath)
-    .filter((f) => f.endsWith(".yml"));
+  const testFiles = datasource.listTests();
 
   if (testFiles.length === 0) {
     console.error(`No tests found in: ${projectConfig.testsDirectoryPath}`);
@@ -69,11 +67,11 @@ export function testProject(rootDirectoryPath: string, projectConfig: ProjectCon
   }
 
   for (const testFile of testFiles) {
-    const testFilePath = path.join(projectConfig.testsDirectoryPath, testFile);
+    const testFilePath = datasource.getEntityPath("test", testFile);
 
     console.log(`  => Testing: ${testFilePath.replace(rootDirectoryPath, "")}`);
 
-    const parsed = parseYaml(fs.readFileSync(testFilePath, "utf8")) as Spec;
+    const parsed = datasource.readTest(testFile);
 
     parsed.tests.forEach(function (test) {
       if (test.segments) {
@@ -83,16 +81,16 @@ export function testProject(rootDirectoryPath: string, projectConfig: ProjectCon
 
           console.log(`     => Segment "${segmentKey}":`);
 
-          const segmentPath = path.join(projectConfig.segmentsDirectoryPath, `${segmentKey}.yml`);
+          const segmentExists = datasource.entityExists("segment", segmentKey);
 
-          if (!fs.existsSync(segmentPath)) {
-            console.error(`        => Segment does not exist: ${segmentPath}`);
+          if (!segmentExists) {
+            console.error(`        => Segment does not exist: ${segmentKey}`);
             hasError = true;
 
             return;
           }
 
-          const parsedSegment = parseYaml(fs.readFileSync(segmentPath, "utf8")) as Segment;
+          const parsedSegment = datasource.readSegment(segmentKey);
           const conditions = parsedSegment.conditions as Condition | Condition[];
 
           segment.assertions.forEach(function (assertion, aIndex) {