@firecms/schema_inference 3.0.0-alpha.4

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 FireCMS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1 @@
+# schema_inference

package/package.json

@@ -0,0 +1,27 @@
+{
+  "version": "3.0.0-alpha.4",
+  "name": "@firecms/schema_inference",
+  "access": "public",
+  "packageManager": "yarn@3.2.3",
+  "type": "module",
+  "main": "./dist/index.umd.js",
+  "module": "./dist/index.es.js",
+  "types": "dist/index.d.ts",
+  "source": "src/index.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.es.js",
+      "require": "./dist/index.umd.js"
+    }
+  },
+  "dependencies": {
+    "@types/node": "^20.5.9",
+    "firecms": "^3.0.0-alpha.4",
+    "typescript": "^5.2.2"
+  },
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build && tsc --emitDeclarationOnly"
+  },
+  "gitHead": "17bba2feea1f6c818c0d9d4b3d6c8e4dfd4e5b4b"
+}

package/src/builders/reference_property_builder.ts

@@ -0,0 +1,16 @@
+import { InferencePropertyBuilderProps } from "../types";
+import { findCommonInitialStringInPath } from "../strings";
+import { Property } from "firecms";
+
+export function buildReferenceProperty({
+                                        totalDocsCount,
+                                        valuesResult
+                                    }: InferencePropertyBuilderProps): Property {
+
+    const property: Property = {
+        dataType: "reference",
+        path: findCommonInitialStringInPath(valuesResult) ?? "!!!FIX_ME!!!"
+    };
+
+    return property;
+}

package/src/builders/string_property_builder.ts

@@ -0,0 +1,107 @@
+import { FileType, Property, StringProperty, unslugify } from "firecms";
+import { InferencePropertyBuilderProps, ValuesCountEntry } from "../types";
+import { findCommonInitialStringInPath } from "../strings";
+import { extractEnumFromValues } from "../util";
+
+const IMAGE_EXTENSIONS = [".jpg", ".png", ".webp", ".gif"];
+const AUDIO_EXTENSIONS = [".mp3", ".ogg", ".opus", ".aac"];
+const VIDEO_EXTENSIONS = [".avi", ".mp4"];
+
+const emailRegEx = /^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/;
+
+
+export function buildStringProperty({
+                                        totalDocsCount,
+                                        valuesResult
+                                    }: InferencePropertyBuilderProps): Property {
+
+    let stringProperty: Property = {
+        dataType: "string",
+
+    };
+
+    if (valuesResult) {
+
+        const totalEntriesCount = valuesResult.values.length;
+        const totalValues = Array.from(valuesResult.valuesCount.keys()).length;
+
+        const config: Partial<StringProperty> = {};
+
+        const probablyAURL = valuesResult.values
+            .filter((value) => typeof value === "string" &&
+                value.toString().startsWith("http")).length > totalDocsCount / 3 * 2;
+        if (probablyAURL) {
+            config.url = true;
+        }
+
+        const probablyAnEmail = valuesResult.values
+            .filter((value) => typeof value === "string" &&
+                emailRegEx.test(value)).length > totalDocsCount / 3 * 2;
+        if (probablyAnEmail) {
+            config.email = true;
+        }
+
+        const probablyUserIds = valuesResult.values
+            .filter((value) => typeof value === "string" && value.length === 28 && !value.includes(" "))
+            .length > totalDocsCount / 3 * 2;
+        if (probablyUserIds)
+            config.readOnly = true;
+
+        if (!probablyAnEmail &&
+            !probablyAURL &&
+            !probablyUserIds &&
+            !probablyAURL &&
+            totalValues < totalEntriesCount / 3
+        ) {
+            const enumValues = extractEnumFromValues(Array.from(valuesResult.valuesCount.keys()));
+
+            if (Object.keys(enumValues).length > 1)
+                config.enumValues = enumValues;
+        }
+
+        // regular string
+        if (!probablyAnEmail &&
+            !probablyAURL &&
+            !probablyUserIds &&
+            !probablyAURL &&
+            !config.enumValues) {
+            const fileType = probableFileType(valuesResult, totalDocsCount);
+            if (fileType) {
+                config.storage = {
+                    acceptedFiles: [fileType as FileType],
+                    storagePath: findCommonInitialStringInPath(valuesResult) ?? "/"
+                };
+            }
+        }
+
+        if (Object.keys(config).length > 0)
+            stringProperty = {
+                ...stringProperty,
+                ...config
+            };
+    }
+
+    return stringProperty;
+}
+
+// TODO: support returning multiple types
+function probableFileType(valuesCount: ValuesCountEntry, totalDocsCount: number): boolean | FileType {
+    const probablyAnImage = valuesCount.values
+        .filter((value) => typeof value === "string" &&
+            IMAGE_EXTENSIONS.some((extension) => value.toString().endsWith(extension))).length > totalDocsCount / 3 * 2;
+
+    const probablyAudio = valuesCount.values
+        .filter((value) => typeof value === "string" &&
+            AUDIO_EXTENSIONS.some((extension) => value.toString().endsWith(extension))).length > totalDocsCount / 3 * 2;
+
+    const probablyVideo = valuesCount.values
+        .filter((value) => typeof value === "string" &&
+            VIDEO_EXTENSIONS.some((extension) => value.toString().endsWith(extension))).length > totalDocsCount / 3 * 2;
+
+    const fileType: boolean | FileType = probablyAnImage
+        ? "image/*"
+        : probablyAudio
+            ? "audio/*"
+            : probablyVideo ? "video/*" : false;
+    return fileType;
+}

package/src/builders/validation_builder.ts

@@ -0,0 +1,18 @@
+import { PropertyValidationSchema } from "firecms";
+import { InferencePropertyBuilderProps } from "../types";
+
+export function buildValidation({
+                                    totalDocsCount,
+                                    valuesResult
+                                }: InferencePropertyBuilderProps): PropertyValidationSchema | undefined {
+
+    if (valuesResult) {
+        const totalEntriesCount = valuesResult.values.length;
+        if (totalDocsCount === totalEntriesCount)
+            return {
+                required: true
+            }
+    }
+
+    return undefined;
+}

package/src/collection_builder.ts

@@ -0,0 +1,324 @@
+import {
+    DataType,
+    EnumValues,
+    mergeDeep,
+    Properties,
+    Property,
+    resolveEnumValues,
+    StringProperty,
+    unslugify
+} from "firecms";
+import {
+    InferencePropertyBuilderProps,
+    TypesCount,
+    TypesCountRecord,
+    ValuesCountEntry,
+    ValuesCountRecord
+} from "./types";
+import { buildStringProperty } from "./builders/string_property_builder";
+import { buildValidation } from "./builders/validation_builder";
+import { buildReferenceProperty } from "./builders/reference_property_builder";
+import { extractEnumFromValues } from "./util";
+
+export type InferenceTypeBuilder = (value: any) => DataType;
+
+export async function buildEntityPropertiesFromData(data: object[], getType: InferenceTypeBuilder): Promise<Properties> {
+    const typesCount: TypesCountRecord = {};
+    const valuesCount: ValuesCountRecord = {};
+    data.forEach((entry) => {
+        if (entry) {
+            Object.entries(entry).forEach(([key, value]) => {
+                increaseMapTypeCount(typesCount, key, value, getType);
+                increaseValuesCount(valuesCount, key, value, getType);
+            })
+        }
+    });
+    // console.log(util.inspect({ typesCount }, { showHidden: false, depth: null, colors: true }));
+    return buildPropertiesFromCount(data.length, typesCount, valuesCount);
+}
+
+export function buildPropertyFromData(data: any[], property: Property, getType: InferenceTypeBuilder): Property {
+    const typesCount = {};
+    const valuesCount: ValuesCountRecord = {};
+    data.forEach((entry) => {
+        increaseTypeCount(property.dataType, typesCount, entry, getType);
+        increaseValuesCount(valuesCount, "inferred_prop", entry, getType);
+    });
+    const enumValues = "enumValues" in property ? resolveEnumValues(property["enumValues"] as EnumValues) : undefined;
+    if (enumValues) {
+        const newEnumValues = extractEnumFromValues(Array.from(valuesCount["inferred_prop"].valuesCount.keys()));
+        console.log("newEnumValues", newEnumValues);
+        return { ...property, enumValues: [...newEnumValues, ...enumValues] } as StringProperty;
+    }
+    const generatedProperty = buildPropertyFromCount("inferred_prop", data.length, property.dataType, typesCount, valuesCount["inferred_prop"]);
+    return mergeDeep(generatedProperty, property);
+}
+
+export function buildPropertiesOrder(properties: Properties<any>): string [] {
+    function propOrder(s: string) {
+        const k = s.toLowerCase();
+        if (k === "title" || k === "name") return 3;
+        if (k.includes("title") || k.includes("name")) return 2;
+        if (k.includes("image") || k.includes("picture")) return 1;
+        return 0;
+    }
+
+    const keys = Object.keys(properties);
+    keys.sort(); // alphabetically
+    keys.sort((a, b) => {
+        return propOrder(b) - propOrder(a);
+    });
+    return keys;
+}
+
+/**
+ * @param type
+ * @param typesCount
+ * @param fieldValue
+ * @param getType
+ */
+function increaseTypeCount(type: DataType, typesCount: TypesCount, fieldValue: any, getType: InferenceTypeBuilder) {
+    if (type === "map") {
+        if (fieldValue) {
+            let mapTypesCount = typesCount[type];
+            if (!mapTypesCount) {
+                mapTypesCount = {};
+                typesCount[type] = mapTypesCount;
+            }
+            Object.entries(fieldValue).forEach(([key, value]) => {
+                increaseMapTypeCount(mapTypesCount as TypesCountRecord, key, value, getType);
+            })
+        }
+    } else if (type === "array") {
+        let arrayTypesCount = typesCount[type];
+        if (!arrayTypesCount) {
+            arrayTypesCount = {};
+            typesCount[type] = arrayTypesCount;
+        }
+        if (fieldValue && Array.isArray(fieldValue) && fieldValue.length > 0) {
+            const arrayType = getMostProbableTypeInArray(fieldValue, getType); // get type of first element
+            if (!arrayTypesCount[arrayType]) (arrayTypesCount[arrayType] as number) = 1;
+            else (arrayTypesCount[arrayType] as number)++;
+        }
+    } else {
+        if (!typesCount[type]) typesCount[type] = 1;
+        else (typesCount[type] as number)++;
+    }
+}
+
+function increaseMapTypeCount(
+    typesCountRecord: TypesCountRecord,
+    key: string,
+    fieldValue: any,
+    getType: InferenceTypeBuilder
+) {
+    let typesCount: TypesCount = typesCountRecord[key];
+    if (!typesCount) {
+        typesCount = {};
+        typesCountRecord[key] = typesCount;
+    }
+
+    if (fieldValue != null) { // Check that fieldValue is not null or undefined before proceeding
+        const type = getType(fieldValue);
+        increaseTypeCount(type, typesCount, fieldValue, getType);
+    }
+}
+
+function increaseValuesCount(
+    typeValuesRecord: ValuesCountRecord,
+    key: string,
+    fieldValue: any,
+    getType: InferenceTypeBuilder
+) {
+
+    const dataType = getType(fieldValue);
+
+    let valuesRecord: {
+        values: any[];
+        valuesCount: Map<any, number>;
+        map?: ValuesCountRecord;
+    } = typeValuesRecord[key];
+
+    if (!valuesRecord) {
+        valuesRecord = {
+            values: [],
+            valuesCount: new Map()
+        };
+        typeValuesRecord[key] = valuesRecord;
+    }
+
+    if (dataType === "map") {
+        let mapValuesRecord: ValuesCountRecord | undefined = valuesRecord.map;
+        if (!mapValuesRecord) {
+            mapValuesRecord = {};
+            valuesRecord.map = mapValuesRecord;
+        }
+        if (fieldValue)
+            Object.entries(fieldValue).forEach(([key, value]) => increaseValuesCount(mapValuesRecord as ValuesCountRecord, key, value, getType))
+    } else if (dataType === "array") {
+        if (Array.isArray(fieldValue)) {
+            fieldValue.forEach((value) => {
+                valuesRecord.values.push(value);
+                valuesRecord.valuesCount.set(value, (valuesRecord.valuesCount.get(value) ?? 0) + 1);
+            })
+        }
+    } else {
+        if (fieldValue) {
+            valuesRecord.values.push(fieldValue);
+            valuesRecord.valuesCount.set(fieldValue, (valuesRecord.valuesCount.get(fieldValue) ?? 0) + 1);
+        }
+    }
+
+}
+
+function getHighestTypesCount(typesCount: TypesCount): number {
+    let highestCount = 0;
+    Object.entries(typesCount).forEach(([type, count]) => {
+        let countValue = 0;
+        if (type === "map") {
+            countValue = getHighestRecordCount(count as TypesCountRecord);
+        } else if (type === "array") {
+            countValue = getHighestTypesCount(count as TypesCount);
+        } else {
+            countValue = count as number;
+        }
+        if (countValue > highestCount) {
+            highestCount = countValue;
+        }
+    });
+
+    return highestCount;
+}
+
+function getHighestRecordCount(record: TypesCountRecord): number {
+    return Object.entries(record)
+        .map(([key, typesCount]) => getHighestTypesCount(typesCount))
+        .reduce((a, b) => Math.max(a, b), 0);
+}
+
+function getMostProbableType(typesCount: TypesCount): DataType {
+    let highestCount = -1;
+    let probableType: DataType = "string"; //default
+    Object.entries(typesCount).forEach(([type, count]) => {
+        let countValue;
+        if (type === "map") {
+            countValue = getHighestRecordCount(count as TypesCountRecord);
+        } else if (type === "array") {
+            countValue = getHighestTypesCount(count as TypesCount);
+        } else {
+            countValue = count as number;
+        }
+        if (countValue > highestCount) {
+            highestCount = countValue;
+            probableType = type as DataType;
+        }
+    });
+    return probableType;
+}
+
+function buildPropertyFromCount(key: string, totalDocsCount: number, mostProbableType: DataType, typesCount: TypesCount, valuesResult?: ValuesCountEntry): Property {
+    let title: string | undefined;
+
+    if (key) {
+        title = unslugify(key);
+    }
+
+    if (mostProbableType === "map") {
+
+        const highVariability = checkTypesCountHighVariability(typesCount);
+        if (highVariability) {
+            return {
+                dataType: "map",
+                name: title,
+                keyValue: true,
+                properties: {}
+            };
+        }
+        const properties = buildPropertiesFromCount(totalDocsCount, typesCount.map as TypesCountRecord, valuesResult ? valuesResult.mapValues : undefined);
+        return {
+            dataType: "map",
+            name: title,
+            properties
+        };
+    } else if (mostProbableType === "array") {
+        const arrayTypesCount = typesCount.array as TypesCount;
+        const arrayMostProbableType = getMostProbableType(arrayTypesCount);
+        const of = buildPropertyFromCount(key, totalDocsCount, arrayMostProbableType, arrayTypesCount, valuesResult);
+        return {
+            dataType: "array",
+            name: title,
+            of
+        };
+    }
+    let result: Property;
+    const propertyProps: InferencePropertyBuilderProps = {
+        name: key,
+        totalDocsCount,
+        valuesResult
+    };
+    if (mostProbableType === "string") {
+        result = buildStringProperty(propertyProps);
+    } else if (mostProbableType === "reference") {
+        result = buildReferenceProperty(propertyProps);
+    } else {
+        result = {
+            dataType: mostProbableType,
+
+        } as Property;
+    }
+
+    if (title) {
+        result.name = title;
+    }
+
+    const validation = buildValidation(propertyProps);
+    if (validation) {
+        result.validation = validation;
+    }
+
+    return result;
+}
+
+function buildPropertiesFromCount(totalDocsCount: number, typesCountRecord: TypesCountRecord, valuesCountRecord?: ValuesCountRecord): Properties {
+    const res: Properties = {};
+    Object.entries(typesCountRecord).forEach(([key, typesCount]) => {
+        const mostProbableType = getMostProbableType(typesCount);
+        res[key] = buildPropertyFromCount(key, totalDocsCount, mostProbableType, typesCount, valuesCountRecord ? valuesCountRecord[key] : undefined);
+    })
+    return res;
+}
+
+function countMaxDocumentsUnder(typesCount: TypesCount) {
+    let count = 0;
+    Object.entries(typesCount).forEach(([type, value]) => {
+        // console.log(util.inspect({ type, value }, { showHidden: false, depth: null, colors: true }));
+        if (typeof value === "object") {
+            count = Math.max(count, countMaxDocumentsUnder(value as TypesCountRecord));
+        } else {
+            count = Math.max(count, value as number);
+        }
+    });
+    return count;
+}
+
+function getMostProbableTypeInArray(array: any[], getType: InferenceTypeBuilder): DataType {
+    let typesCount: TypesCount = {};
+    array.forEach((value) => {
+        increaseTypeCount(getType(value), typesCount, value, getType);
+    });
+    return getMostProbableType(typesCount);
+}
+
+function checkTypesCountHighVariability(typesCount: TypesCount) {
+    const maxCount = countMaxDocumentsUnder(typesCount);
+    let keysWithFewValues = 0;
+    Object.entries(typesCount.map ?? {})
+        .forEach(([key, value]) => {
+            const count = countMaxDocumentsUnder(value);
+            if (count < maxCount / 3) {
+                keysWithFewValues++;
+            }
+        });
+    return keysWithFewValues / Object.entries(typesCount.map ?? {}).length > 0.5;
+}
+

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./collection_builder";
+export * from "./util";

package/src/strings.ts

@@ -0,0 +1,47 @@
+import { ValuesCountEntry } from "./types";
+import { DocumentReference } from "firebase/firestore";
+
+
+export function findCommonInitialStringInPath(valuesCount?: ValuesCountEntry) {
+
+    if (!valuesCount) return undefined;
+
+    function getPath(value: any) {
+        if (typeof value === "string") return value;
+        else if (value instanceof DocumentReference) return value.path;
+        else return undefined;
+    }
+
+    const strings: string[] = valuesCount.values.map((v) => getPath(v)).filter(v => !!v) as string[];
+    const pathWithSlash = strings.find((s) => s.includes("/"));
+    if (!pathWithSlash)
+        return undefined;
+
+    const searchedPath = pathWithSlash.substr(0, pathWithSlash.lastIndexOf("/"));
+
+    const yep = valuesCount.values
+        .filter((value) => {
+            const path = getPath(value);
+            if (!path) return false;
+            return path.startsWith(searchedPath)
+        }).length > valuesCount.values.length / 3 * 2;
+
+    return yep ? searchedPath : undefined;
+
+}
+
+export function removeInitialAndTrailingSlashes(s: string): string {
+    return removeInitialSlash(removeTrailingSlash(s));
+}
+
+export function removeInitialSlash(s: string) {
+    if (s.startsWith("/"))
+        return s.slice(1);
+    else return s;
+}
+
+export function removeTrailingSlash(s: string) {
+    if (s.endsWith("/"))
+        return s.slice(0, -1);
+    else return s;
+}

package/src/test_schemas/test_schema.ts

@@ -0,0 +1,28 @@
+import { buildEntityPropertiesFromData } from "../collection_builder";
+import { DataType } from "firecms";
+
+import usage from "./usage.json" assert {
+        type: 'json',
+        integrity: 'sha384-ABC123'
+        };
+import * as util from "util";
+
+buildEntityPropertiesFromData(usage, getType)
+    .then((res) => console.log(util.inspect(res, { showHidden: false, depth: null, colors: true })));
+
+
+function getType(value: any): DataType {
+    if (typeof value === "number")
+        return "number";
+    else if (typeof value === "string")
+        return "string";
+    else if (typeof value === "boolean")
+        return "boolean";
+    else if (Array.isArray(value))
+        return "array";
+    else if (value && "_seconds" in value && "_nanoseconds" in value)
+        return "date";
+    else if (value && "id" in value && "path" in value)
+        return "reference";
+    return "map";
+}