@firecms/schema_inference 3.0.0-alpha.14 → 3.0.0-alpha.16

package/src/collection_builder.ts

@@ -1,324 +0,0 @@
-import {
-    DataType,
-    EnumValues,
-    mergeDeep,
-    Properties,
-    Property,
-    resolveEnumValues,
-    StringProperty,
-    unslugify
-} from "@firecms/core";
-import {
-    InferencePropertyBuilderProps,
-    TypesCount,
-    TypesCountRecord,
-    ValuesCountEntry,
-    ValuesCountRecord
-} from "./types";
-import { buildStringProperty } from "./builders/string_property_builder";
-import { buildValidation } from "./builders/validation_builder";
-import { buildReferenceProperty } from "./builders/reference_property_builder";
-import { extractEnumFromValues } from "./util";
-
-export type InferenceTypeBuilder = (value: any) => DataType;
-
-export async function buildEntityPropertiesFromData(data: object[], getType: InferenceTypeBuilder): Promise<Properties> {
-    const typesCount: TypesCountRecord = {};
-    const valuesCount: ValuesCountRecord = {};
-    data.forEach((entry) => {
-        if (entry) {
-            Object.entries(entry).forEach(([key, value]) => {
-                increaseMapTypeCount(typesCount, key, value, getType);
-                increaseValuesCount(valuesCount, key, value, getType);
-            })
-        }
-    });
-    // console.log(util.inspect({ typesCount }, { showHidden: false, depth: null, colors: true }));
-    return buildPropertiesFromCount(data.length, typesCount, valuesCount);
-}
-
-export function buildPropertyFromData(data: any[], property: Property, getType: InferenceTypeBuilder): Property {
-    const typesCount = {};
-    const valuesCount: ValuesCountRecord = {};
-    data.forEach((entry) => {
-        increaseTypeCount(property.dataType, typesCount, entry, getType);
-        increaseValuesCount(valuesCount, "inferred_prop", entry, getType);
-    });
-    const enumValues = "enumValues" in property ? resolveEnumValues(property["enumValues"] as EnumValues) : undefined;
-    if (enumValues) {
-        const newEnumValues = extractEnumFromValues(Array.from(valuesCount["inferred_prop"].valuesCount.keys()));
-        console.log("newEnumValues", newEnumValues);
-        return { ...property, enumValues: [...newEnumValues, ...enumValues] } as StringProperty;
-    }
-    const generatedProperty = buildPropertyFromCount("inferred_prop", data.length, property.dataType, typesCount, valuesCount["inferred_prop"]);
-    return mergeDeep(generatedProperty, property);
-}
-
-export function buildPropertiesOrder(properties: Properties<any>): string [] {
-    function propOrder(s: string) {
-        const k = s.toLowerCase();
-        if (k === "title" || k === "name") return 3;
-        if (k.includes("title") || k.includes("name")) return 2;
-        if (k.includes("image") || k.includes("picture")) return 1;
-        return 0;
-    }
-
-    const keys = Object.keys(properties);
-    keys.sort(); // alphabetically
-    keys.sort((a, b) => {
-        return propOrder(b) - propOrder(a);
-    });
-    return keys;
-}
-
-/**
- * @param type
- * @param typesCount
- * @param fieldValue
- * @param getType
- */
-function increaseTypeCount(type: DataType, typesCount: TypesCount, fieldValue: any, getType: InferenceTypeBuilder) {
-    if (type === "map") {
-        if (fieldValue) {
-            let mapTypesCount = typesCount[type];
-            if (!mapTypesCount) {
-                mapTypesCount = {};
-                typesCount[type] = mapTypesCount;
-            }
-            Object.entries(fieldValue).forEach(([key, value]) => {
-                increaseMapTypeCount(mapTypesCount as TypesCountRecord, key, value, getType);
-            })
-        }
-    } else if (type === "array") {
-        let arrayTypesCount = typesCount[type];
-        if (!arrayTypesCount) {
-            arrayTypesCount = {};
-            typesCount[type] = arrayTypesCount;
-        }
-        if (fieldValue && Array.isArray(fieldValue) && fieldValue.length > 0) {
-            const arrayType = getMostProbableTypeInArray(fieldValue, getType); // get type of first element
-            if (!arrayTypesCount[arrayType]) (arrayTypesCount[arrayType] as number) = 1;
-            else (arrayTypesCount[arrayType] as number)++;
-        }
-    } else {
-        if (!typesCount[type]) typesCount[type] = 1;
-        else (typesCount[type] as number)++;
-    }
-}
-
-function increaseMapTypeCount(
-    typesCountRecord: TypesCountRecord,
-    key: string,
-    fieldValue: any,
-    getType: InferenceTypeBuilder
-) {
-    let typesCount: TypesCount = typesCountRecord[key];
-    if (!typesCount) {
-        typesCount = {};
-        typesCountRecord[key] = typesCount;
-    }
-
-    if (fieldValue != null) { // Check that fieldValue is not null or undefined before proceeding
-        const type = getType(fieldValue);
-        increaseTypeCount(type, typesCount, fieldValue, getType);
-    }
-}
-
-function increaseValuesCount(
-    typeValuesRecord: ValuesCountRecord,
-    key: string,
-    fieldValue: any,
-    getType: InferenceTypeBuilder
-) {
-
-    const dataType = getType(fieldValue);
-
-    let valuesRecord: {
-        values: any[];
-        valuesCount: Map<any, number>;
-        map?: ValuesCountRecord;
-    } = typeValuesRecord[key];
-
-    if (!valuesRecord) {
-        valuesRecord = {
-            values: [],
-            valuesCount: new Map()
-        };
-        typeValuesRecord[key] = valuesRecord;
-    }
-
-    if (dataType === "map") {
-        let mapValuesRecord: ValuesCountRecord | undefined = valuesRecord.map;
-        if (!mapValuesRecord) {
-            mapValuesRecord = {};
-            valuesRecord.map = mapValuesRecord;
-        }
-        if (fieldValue)
-            Object.entries(fieldValue).forEach(([key, value]) => increaseValuesCount(mapValuesRecord as ValuesCountRecord, key, value, getType))
-    } else if (dataType === "array") {
-        if (Array.isArray(fieldValue)) {
-            fieldValue.forEach((value) => {
-                valuesRecord.values.push(value);
-                valuesRecord.valuesCount.set(value, (valuesRecord.valuesCount.get(value) ?? 0) + 1);
-            })
-        }
-    } else {
-        if (fieldValue) {
-            valuesRecord.values.push(fieldValue);
-            valuesRecord.valuesCount.set(fieldValue, (valuesRecord.valuesCount.get(fieldValue) ?? 0) + 1);
-        }
-    }
-
-}
-
-function getHighestTypesCount(typesCount: TypesCount): number {
-    let highestCount = 0;
-    Object.entries(typesCount).forEach(([type, count]) => {
-        let countValue = 0;
-        if (type === "map") {
-            countValue = getHighestRecordCount(count as TypesCountRecord);
-        } else if (type === "array") {
-            countValue = getHighestTypesCount(count as TypesCount);
-        } else {
-            countValue = count as number;
-        }
-        if (countValue > highestCount) {
-            highestCount = countValue;
-        }
-    });
-
-    return highestCount;
-}
-
-function getHighestRecordCount(record: TypesCountRecord): number {
-    return Object.entries(record)
-        .map(([key, typesCount]) => getHighestTypesCount(typesCount))
-        .reduce((a, b) => Math.max(a, b), 0);
-}
-
-function getMostProbableType(typesCount: TypesCount): DataType {
-    let highestCount = -1;
-    let probableType: DataType = "string"; //default
-    Object.entries(typesCount).forEach(([type, count]) => {
-        let countValue;
-        if (type === "map") {
-            countValue = getHighestRecordCount(count as TypesCountRecord);
-        } else if (type === "array") {
-            countValue = getHighestTypesCount(count as TypesCount);
-        } else {
-            countValue = count as number;
-        }
-        if (countValue > highestCount) {
-            highestCount = countValue;
-            probableType = type as DataType;
-        }
-    });
-    return probableType;
-}
-
-function buildPropertyFromCount(key: string, totalDocsCount: number, mostProbableType: DataType, typesCount: TypesCount, valuesResult?: ValuesCountEntry): Property {
-    let title: string | undefined;
-
-    if (key) {
-        title = unslugify(key);
-    }
-
-    if (mostProbableType === "map") {
-
-        const highVariability = checkTypesCountHighVariability(typesCount);
-        if (highVariability) {
-            return {
-                dataType: "map",
-                name: title,
-                keyValue: true,
-                properties: {}
-            };
-        }
-        const properties = buildPropertiesFromCount(totalDocsCount, typesCount.map as TypesCountRecord, valuesResult ? valuesResult.mapValues : undefined);
-        return {
-            dataType: "map",
-            name: title,
-            properties
-        };
-    } else if (mostProbableType === "array") {
-        const arrayTypesCount = typesCount.array as TypesCount;
-        const arrayMostProbableType = getMostProbableType(arrayTypesCount);
-        const of = buildPropertyFromCount(key, totalDocsCount, arrayMostProbableType, arrayTypesCount, valuesResult);
-        return {
-            dataType: "array",
-            name: title,
-            of
-        };
-    }
-    let result: Property;
-    const propertyProps: InferencePropertyBuilderProps = {
-        name: key,
-        totalDocsCount,
-        valuesResult
-    };
-    if (mostProbableType === "string") {
-        result = buildStringProperty(propertyProps);
-    } else if (mostProbableType === "reference") {
-        result = buildReferenceProperty(propertyProps);
-    } else {
-        result = {
-            dataType: mostProbableType,
-
-        } as Property;
-    }
-
-    if (title) {
-        result.name = title;
-    }
-
-    const validation = buildValidation(propertyProps);
-    if (validation) {
-        result.validation = validation;
-    }
-
-    return result;
-}
-
-function buildPropertiesFromCount(totalDocsCount: number, typesCountRecord: TypesCountRecord, valuesCountRecord?: ValuesCountRecord): Properties {
-    const res: Properties = {};
-    Object.entries(typesCountRecord).forEach(([key, typesCount]) => {
-        const mostProbableType = getMostProbableType(typesCount);
-        res[key] = buildPropertyFromCount(key, totalDocsCount, mostProbableType, typesCount, valuesCountRecord ? valuesCountRecord[key] : undefined);
-    })
-    return res;
-}
-
-function countMaxDocumentsUnder(typesCount: TypesCount) {
-    let count = 0;
-    Object.entries(typesCount).forEach(([type, value]) => {
-        // console.log(util.inspect({ type, value }, { showHidden: false, depth: null, colors: true }));
-        if (typeof value === "object") {
-            count = Math.max(count, countMaxDocumentsUnder(value as TypesCountRecord));
-        } else {
-            count = Math.max(count, value as number);
-        }
-    });
-    return count;
-}
-
-function getMostProbableTypeInArray(array: any[], getType: InferenceTypeBuilder): DataType {
-    let typesCount: TypesCount = {};
-    array.forEach((value) => {
-        increaseTypeCount(getType(value), typesCount, value, getType);
-    });
-    return getMostProbableType(typesCount);
-}
-
-function checkTypesCountHighVariability(typesCount: TypesCount) {
-    const maxCount = countMaxDocumentsUnder(typesCount);
-    let keysWithFewValues = 0;
-    Object.entries(typesCount.map ?? {})
-        .forEach(([key, value]) => {
-            const count = countMaxDocumentsUnder(value);
-            if (count < maxCount / 3) {
-                keysWithFewValues++;
-            }
-        });
-    return keysWithFewValues / Object.entries(typesCount.map ?? {}).length > 0.5;
-}
-

package/src/strings.ts

@@ -1,47 +0,0 @@
-import { ValuesCountEntry } from "./types";
-import { DocumentReference } from "firebase/firestore";
-
-
-export function findCommonInitialStringInPath(valuesCount?: ValuesCountEntry) {
-
-    if (!valuesCount) return undefined;
-
-    function getPath(value: any) {
-        if (typeof value === "string") return value;
-        else if (value instanceof DocumentReference) return value.path;
-        else return undefined;
-    }
-
-    const strings: string[] = valuesCount.values.map((v) => getPath(v)).filter(v => !!v) as string[];
-    const pathWithSlash = strings.find((s) => s.includes("/"));
-    if (!pathWithSlash)
-        return undefined;
-
-    const searchedPath = pathWithSlash.substr(0, pathWithSlash.lastIndexOf("/"));
-
-    const yep = valuesCount.values
-        .filter((value) => {
-            const path = getPath(value);
-            if (!path) return false;
-            return path.startsWith(searchedPath)
-        }).length > valuesCount.values.length / 3 * 2;
-
-    return yep ? searchedPath : undefined;
-
-}
-
-export function removeInitialAndTrailingSlashes(s: string): string {
-    return removeInitialSlash(removeTrailingSlash(s));
-}
-
-export function removeInitialSlash(s: string) {
-    if (s.startsWith("/"))
-        return s.slice(1);
-    else return s;
-}
-
-export function removeTrailingSlash(s: string) {
-    if (s.endsWith("/"))
-        return s.slice(0, -1);
-    else return s;
-}

package/src/test_schemas/test_schema.ts

@@ -1,28 +0,0 @@
-import { buildEntityPropertiesFromData } from "../collection_builder";
-import { DataType } from "@firecms/core";
-
-import usage from "./usage.json" assert {
-        type: 'json',
-        integrity: 'sha384-ABC123'
-        };
-import * as util from "util";
-
-buildEntityPropertiesFromData(usage, getType)
-    .then((res) => console.log(util.inspect(res, { showHidden: false, depth: null, colors: true })));
-
-
-function getType(value: any): DataType {
-    if (typeof value === "number")
-        return "number";
-    else if (typeof value === "string")
-        return "string";
-    else if (typeof value === "boolean")
-        return "boolean";
-    else if (Array.isArray(value))
-        return "array";
-    else if (value && "_seconds" in value && "_nanoseconds" in value)
-        return "date";
-    else if (value && "id" in value && "path" in value)
-        return "reference";
-    return "map";
-}