@rebasepro/schema-inference 0.0.1-canary.4d4fb3e

package/dist/strings.d.ts

@@ -0,0 +1,19 @@
+import { ValuesCountEntry } from "./types";
+/**
+ * Parse a reference string value which can be in the format:
+ * - Simple: "path/entityId"
+ * - With database: "database_name:::path/entityId"
+ * Returns the path and database (undefined if not specified or if "(default)")
+ */
+export declare function parseReferenceString(value: string): {
+    path: string;
+    database?: string;
+} | null;
+/**
+ * Check if a string value looks like a reference
+ */
+export declare function looksLikeReference(value: any): boolean;
+export declare function findCommonInitialStringInPath(valuesCount?: ValuesCountEntry): string | undefined;
+export declare function removeInitialAndTrailingSlashes(s: string): string;
+export declare function removeInitialSlash(s: string): string;
+export declare function removeTrailingSlash(s: string): string;

package/dist/test_schemas/test_schema.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types.d.ts

@@ -0,0 +1,35 @@
+import { DataType } from "@rebasepro/types";
+export type TypesCount = {
+    number?: number;
+    string?: number;
+    boolean?: number;
+    map?: TypesCountRecord;
+    array?: TypesCount;
+    date?: number;
+    geopoint?: number;
+    reference?: number;
+    relation?: number;
+};
+export type TypesCountRecord<K extends keyof DataType = any> = {
+    [P in K]: TypesCount;
+};
+export type ValuesCountEntry = {
+    values: any[];
+    valuesCount: Map<any, number>;
+    mapValues?: ValuesCountRecord;
+};
+export type ValuesCountRecord = Record<string, ValuesCountEntry>;
+export type InferencePropertyBuilderProps = {
+    /**
+     * Name of the property
+     */
+    name: string;
+    /**
+     * Total documents this props are built from
+     */
+    totalDocsCount: number;
+    /**
+     * How many times does each value show up
+     */
+    valuesResult?: ValuesCountEntry;
+};

package/dist/util.d.ts

@@ -0,0 +1,11 @@
+import { EnumValueConfig, EnumValues } from "@rebasepro/types";
+export declare function extractEnumFromValues(values: unknown[]): {
+    id: string;
+    label: string;
+}[];
+export declare function prettifyIdentifier(input: string): string;
+export declare function unslugify(slug?: string): string;
+export declare function resolveEnumValues(input: EnumValues): EnumValueConfig[] | undefined;
+export declare function mergeDeep<T extends Record<any, any>, U extends Record<any, any>>(target: T, source: U, ignoreUndefined?: boolean): T & U;
+export declare function isObject(item: any): any;
+export declare function isPlainObject(obj: any): boolean;

package/package.json

@@ -0,0 +1,37 @@
+{
+    "name": "@rebasepro/schema-inference",
+    "version": "0.0.1-canary.4d4fb3e",
+    "type": "module",
+    "publishConfig": {
+        "access": "public"
+    },
+    "main": "./dist/index.umd.cjs",
+    "module": "./dist/index.es.js",
+    "types": "dist/index.d.ts",
+    "source": "src/index.ts",
+    "devDependencies": {
+        "@rebasepro/types": "0.0.1-canary.4d4fb3e",
+        "@types/node": "^20.17.14",
+        "typescript": "^5.9.3",
+        "vite": "^7.2.4"
+    },
+    "exports": {
+        ".": {
+            "types": "./dist/index.d.ts",
+            "development": "./src/index.ts",
+            "import": "./dist/index.es.js",
+            "require": "./dist/index.umd.cjs"
+        },
+        "./package.json": "./package.json"
+    },
+    "files": [
+        "dist",
+        "src"
+    ],
+    "scripts": {
+        "dev": "vite",
+        "build": "vite build && tsc --emitDeclarationOnly -p tsconfig.prod.json",
+        "clean": "rm -rf dist && find ./src -name '*.js' -type f | xargs rm -f"
+    },
+    "gitHead": "71bcef3c51a458cd054f7924cc18efbbe515dcc8"
+}

package/src/builders/reference_property_builder.ts

@@ -0,0 +1,18 @@
+import { findCommonInitialStringInPath } from "../strings";
+import { InferencePropertyBuilderProps } from "../types";
+import { Property } from "@rebasepro/types";
+
+export function buildReferenceProperty({
+    name,
+    totalDocsCount,
+    valuesResult
+}: InferencePropertyBuilderProps): Property {
+
+    const property: Property = {
+        name: name ?? "",
+        type: "reference",
+        path: findCommonInitialStringInPath(valuesResult) ?? "!!!FIX_ME!!!"
+    };
+
+    return property;
+}

package/src/builders/string_property_builder.ts

@@ -0,0 +1,115 @@
+import { findCommonInitialStringInPath } from "../strings";
+import { extractEnumFromValues } from "../util";
+import { FileType, Property, StringProperty } from "@rebasepro/types";
+import { InferencePropertyBuilderProps, ValuesCountEntry } from "../types";
+
+const IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif"];
+const AUDIO_EXTENSIONS = [".mp3", ".ogg", ".opus", ".aac"];
+const VIDEO_EXTENSIONS = [".avi", ".mp4"];
+
+const emailRegEx = /^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/;
+
+export function buildStringProperty({
+    name,
+    totalDocsCount,
+    valuesResult
+}: InferencePropertyBuilderProps): Property {
+
+    let stringProperty: Property = {
+        name: name ?? "",
+        type: "string",
+
+    };
+
+    if (valuesResult) {
+
+        const totalEntriesCount = valuesResult.values.length;
+        const totalValues = Array.from(valuesResult.valuesCount.keys()).length;
+
+        const config: Partial<StringProperty> = {};
+
+        const probablyAURL = valuesResult.values
+            .filter((value) => typeof value === "string" &&
+                value.toString().startsWith("http")).length > totalDocsCount / 3 * 2;
+        if (probablyAURL) {
+            config.url = true;
+        }
+
+        const probablyAnEmail = valuesResult.values
+            .filter((value) => typeof value === "string" &&
+                emailRegEx.test(value)).length > totalDocsCount / 3 * 2;
+        if (probablyAnEmail) {
+            config.email = true;
+        }
+
+        const probablyUserIds = valuesResult.values
+            .filter((value) => typeof value === "string" && value.length === 28 && !value.includes(" "))
+            .length > totalDocsCount / 3 * 2;
+        if (probablyUserIds)
+            config.readOnly = true;
+
+        if (!probablyAnEmail &&
+            !probablyAURL &&
+            !probablyUserIds &&
+            !probablyAURL &&
+            totalValues < totalEntriesCount / 3
+        ) {
+            const enumValues = extractEnumFromValues(Array.from(valuesResult.valuesCount.keys()));
+
+            if (Object.keys(enumValues).length > 1)
+                config.enum = enumValues;
+        }
+
+        // regular string
+        if (!probablyAnEmail &&
+            !probablyAURL &&
+            !probablyUserIds &&
+            !probablyAURL &&
+            !config.enum) {
+            const fileType = probableFileType(valuesResult, totalDocsCount);
+            if (fileType) {
+                config.storage = {
+                    acceptedFiles: fileType as FileType[],
+                    storagePath: findCommonInitialStringInPath(valuesResult) ?? "/"
+                };
+            }
+        }
+
+        if (Object.keys(config).length > 0)
+            stringProperty = {
+                ...stringProperty,
+                ...config
+            } as StringProperty;
+    }
+
+    return stringProperty;
+}
+
+function probableFileType(valuesCount: ValuesCountEntry, totalDocsCount: number): false | FileType[] {
+    const isImage = (value: string) => IMAGE_EXTENSIONS.some((extension) => value.toString().endsWith(extension));
+    const isAudio = (value: string) => AUDIO_EXTENSIONS.some((extension) => value.toString().endsWith(extension));
+    const isVideo = (value: string) => VIDEO_EXTENSIONS.some((extension) => value.toString().endsWith(extension));
+
+    const stringValues = valuesCount.values.filter((v): v is string => typeof v === "string");
+    
+    let imageCount = 0;
+    let audioCount = 0;
+    let videoCount = 0;
+
+    for (const value of stringValues) {
+        if (isImage(value)) imageCount++;
+        else if (isAudio(value)) audioCount++;
+        else if (isVideo(value)) videoCount++;
+    }
+
+    const totalMediaCount = imageCount + audioCount + videoCount;
+    if (totalMediaCount > (totalDocsCount * 2) / 3) {
+        const fileTypes: FileType[] = [];
+        if (imageCount > 0) fileTypes.push("image/*");
+        if (audioCount > 0) fileTypes.push("audio/*");
+        if (videoCount > 0) fileTypes.push("video/*");
+        return fileTypes.length > 0 ? fileTypes : false;
+    }
+
+    return false;
+}

package/src/builders/validation_builder.ts

@@ -0,0 +1,18 @@
+import { PropertyValidationSchema } from "@rebasepro/types";
+import { InferencePropertyBuilderProps } from "../types";
+
+export function buildValidation({
+    totalDocsCount,
+    valuesResult
+}: InferencePropertyBuilderProps): PropertyValidationSchema | undefined {
+
+    if (valuesResult) {
+        const totalEntriesCount = valuesResult.values.length;
+        if (totalDocsCount === totalEntriesCount)
+            return {
+                required: true
+            }
+    }
+
+    return undefined;
+}

package/src/collection_builder.ts

@@ -0,0 +1,404 @@
+import {
+    InferencePropertyBuilderProps,
+    TypesCount,
+    TypesCountRecord,
+    ValuesCountEntry,
+    ValuesCountRecord
+} from "./types";
+import { buildStringProperty } from "./builders/string_property_builder";
+import { buildValidation } from "./builders/validation_builder";
+import { buildReferenceProperty } from "./builders/reference_property_builder";
+import { extractEnumFromValues, mergeDeep, prettifyIdentifier, resolveEnumValues } from "./util";
+import { DataType, EnumValues, Properties, Property, StringProperty } from "@rebasepro/types";
+
+export type InferenceTypeBuilder = (value: any) => DataType;
+
+export async function buildEntityPropertiesFromData(
+    data: object[],
+    getType: InferenceTypeBuilder
+): Promise<Properties> {
+    const typesCount: TypesCountRecord = {};
+    const valuesCount: ValuesCountRecord = {};
+    if (data) {
+        data.forEach((entry) => {
+            if (entry) {
+                Object.entries(entry).forEach(([key, value]) => {
+                    if (key.startsWith("_")) return; // Ignore properties starting with _
+                    increaseMapTypeCount(typesCount, key, value, getType);
+                    increaseValuesCount(valuesCount, key, value, getType);
+                });
+            }
+        });
+    }
+    return buildPropertiesFromCount(data.length, typesCount, valuesCount);
+}
+
+export function buildPropertyFromData(
+    data: any[],
+    property: Property,
+    getType: InferenceTypeBuilder
+): Property {
+    const typesCount = {};
+    const valuesCount: ValuesCountRecord = {};
+    if (data) {
+        data.forEach((entry) => {
+            increaseTypeCount(property.type, typesCount, entry, getType);
+            increaseValuesCount(valuesCount, "inferred_prop", entry, getType);
+        });
+    }
+    const enumValues = "enum" in property ? resolveEnumValues(property["enum"] as EnumValues) : undefined;
+    if (enumValues) {
+        const newEnumValues = extractEnumFromValues(Array.from(valuesCount["inferred_prop"].valuesCount.keys()));
+        return {
+            ...property,
+            enum: [...newEnumValues, ...enumValues]
+        } as StringProperty;
+    }
+    const generatedProperty = buildPropertyFromCount(
+        "inferred_prop",
+        data.length,
+        property.type,
+        typesCount,
+        valuesCount["inferred_prop"]
+    );
+    return mergeDeep(generatedProperty, property);
+}
+
+export function buildPropertiesOrder(
+    properties: Properties,
+    propertiesOrder?: string[],
+    priorityKeys?: string[]
+): string[] {
+    const lowerCasePriorityKeys = (priorityKeys ?? []).map((key) => key.toLowerCase());
+
+    function propOrder(s: string) {
+        const k = s.toLowerCase();
+        if (lowerCasePriorityKeys.includes(k)) return 4;
+        if (k === "title" || k === "name") return 3;
+        if (k.includes("title") || k.includes("name")) return 2;
+        if (k.includes("image") || k.includes("picture")) return 1;
+        return 0;
+    }
+
+    const keys = propertiesOrder ?? Object.keys(properties);
+    keys.sort(); // alphabetically
+    keys.sort((a, b) => {
+        return propOrder(b) - propOrder(a);
+    });
+    return keys;
+}
+
+/**
+ * @param type
+ * @param typesCount
+ * @param fieldValue
+ * @param getType
+ */
+function increaseTypeCount(
+    type: DataType,
+    typesCount: TypesCount,
+    fieldValue: any,
+    getType: InferenceTypeBuilder
+) {
+    if (type === "map") {
+        if (fieldValue) {
+            let mapTypesCount = typesCount[type];
+            if (!mapTypesCount) {
+                mapTypesCount = {};
+                typesCount[type] = mapTypesCount;
+            }
+            Object.entries(fieldValue).forEach(([key, value]) => {
+                increaseMapTypeCount(mapTypesCount as TypesCountRecord, key, value, getType);
+            });
+        }
+    } else if (type === "array") {
+        let arrayTypesCount = typesCount[type];
+        if (!arrayTypesCount) {
+            arrayTypesCount = {};
+            typesCount[type] = arrayTypesCount;
+        }
+        if (fieldValue && Array.isArray(fieldValue) && fieldValue.length > 0) {
+            const arrayType = getMostProbableTypeInArray(fieldValue, getType);
+            if (arrayType === "map") {
+                let mapTypesCount = arrayTypesCount[arrayType];
+                if (!mapTypesCount) {
+                    mapTypesCount = {};
+                }
+                fieldValue.forEach((value) => {
+                    if (value && typeof value === "object" && !Array.isArray(value)) { // Ensure value is an object for Object.entries
+                        Object.entries(value).forEach(([key, v]) =>
+                            increaseMapTypeCount(mapTypesCount, key, v, getType)
+                        );
+                    }
+                });
+                arrayTypesCount[arrayType] = mapTypesCount;
+            } else {
+                if (!arrayTypesCount[arrayType]) arrayTypesCount[arrayType] = 1;
+                else arrayTypesCount[arrayType] = Number(arrayTypesCount[arrayType]) + 1;
+            }
+        }
+    } else {
+        if (!typesCount[type]) typesCount[type] = 1;
+        else typesCount[type] = Number(typesCount[type]) + 1;
+    }
+}
+
+function increaseMapTypeCount(
+    typesCountRecord: TypesCountRecord,
+    key: string,
+    fieldValue: any,
+    getType: InferenceTypeBuilder
+) {
+    if (key.startsWith("_")) return; // Ignore properties starting with _
+
+    let typesCount: TypesCount = typesCountRecord[key];
+    if (!typesCount) {
+        typesCount = {};
+        typesCountRecord[key] = typesCount;
+    }
+
+    if (fieldValue != null) {
+        // Check that fieldValue is not null or undefined before proceeding
+        const type = getType(fieldValue);
+        increaseTypeCount(type, typesCount, fieldValue, getType);
+    }
+}
+
+function increaseValuesCount(
+    typeValuesRecord: ValuesCountRecord,
+    key: string,
+    fieldValue: any,
+    getType: InferenceTypeBuilder
+) {
+    if (key.startsWith("_")) return; // Ignore properties starting with _
+
+    const type = getType(fieldValue);
+
+    let valuesRecord: {
+        values: any[];
+        valuesCount: Map<any, number>;
+        map?: ValuesCountRecord;
+    } = typeValuesRecord[key];
+
+    if (!valuesRecord) {
+        valuesRecord = {
+            values: [],
+            valuesCount: new Map()
+        };
+        typeValuesRecord[key] = valuesRecord;
+    }
+
+    if (type === "map") {
+        let mapValuesRecord: ValuesCountRecord | undefined = valuesRecord.map;
+        if (!mapValuesRecord) {
+            mapValuesRecord = {};
+            valuesRecord.map = mapValuesRecord;
+        }
+        if (fieldValue)
+            Object.entries(fieldValue).forEach(([subKey, value]) =>
+                increaseValuesCount(mapValuesRecord as ValuesCountRecord, subKey, value, getType)
+            );
+    } else if (type === "array") {
+        if (Array.isArray(fieldValue)) {
+            fieldValue.forEach((value) => {
+                valuesRecord.values.push(value);
+                valuesRecord.valuesCount.set(value, (valuesRecord.valuesCount.get(value) ?? 0) + 1);
+            });
+        }
+    } else {
+        if (fieldValue !== null && fieldValue !== undefined) {
+            valuesRecord.values.push(fieldValue);
+            valuesRecord.valuesCount.set(fieldValue, (valuesRecord.valuesCount.get(fieldValue) ?? 0) + 1);
+        }
+    }
+}
+
+function getHighestTypesCount(typesCount: TypesCount): number {
+    let highestCount = 0;
+    Object.entries(typesCount).forEach(([type, count]) => {
+        let countValue = 0;
+        if (type === "map") {
+            countValue = getHighestRecordCount(count as TypesCountRecord);
+        } else if (type === "array") {
+            countValue = getHighestTypesCount(count as TypesCount);
+        } else {
+            countValue = Number(count);
+        }
+        if (countValue > highestCount) {
+            highestCount = countValue;
+        }
+    });
+
+    return highestCount;
+}
+
+function getHighestRecordCount(record: TypesCountRecord): number {
+    return Object.entries(record)
+        .map(([key, typesCount]) => getHighestTypesCount(typesCount))
+        .reduce((a, b) => Math.max(a, b), 0);
+}
+
+function getMostProbableType(typesCount: TypesCount): DataType {
+    let highestCount = -1;
+    let probableType: DataType = "string"; // default
+    Object.entries(typesCount).forEach(([type, count]) => {
+        let countValue;
+        if (type === "map") {
+            countValue = getHighestRecordCount(count as TypesCountRecord);
+        } else if (type === "array") {
+            countValue = getHighestTypesCount(count as TypesCount);
+        } else {
+            countValue = Number(count);
+        }
+        if (countValue > highestCount) {
+            highestCount = countValue;
+            probableType = type as DataType;
+        }
+    });
+    return probableType;
+}
+
+function buildPropertyFromCount(
+    key: string,
+    totalDocsCount: number,
+    mostProbableType: DataType,
+    typesCount: TypesCount,
+    valuesResult?: ValuesCountEntry
+): Property {
+    let title: string | undefined;
+
+    if (key) {
+        title = prettifyIdentifier(key);
+    }
+
+    let result: Property | undefined = undefined;
+    if (mostProbableType === "map") {
+        const highVariability = checkTypesCountHighVariability(typesCount);
+        if (highVariability) {
+            result = {
+                type: "map",
+                name: title ?? key ?? "",
+                keyValue: true,
+                properties: {}
+            };
+        }
+        const properties = buildPropertiesFromCount(
+            totalDocsCount,
+            typesCount.map as TypesCountRecord,
+            valuesResult ? valuesResult.mapValues : undefined
+        );
+        result = {
+            type: "map",
+            name: title ?? key ?? "",
+            properties
+        };
+    } else if (mostProbableType === "array") {
+        const arrayTypesCount = typesCount.array as TypesCount;
+        const arrayMostProbableType = getMostProbableType(arrayTypesCount);
+        const of = buildPropertyFromCount(
+            key,
+            totalDocsCount,
+            arrayMostProbableType,
+            arrayTypesCount,
+            valuesResult
+        );
+        result = {
+            type: "array",
+            name: title ?? key ?? "",
+            of
+        };
+    }
+
+    if (!result) {
+        const propertyProps: InferencePropertyBuilderProps = {
+            name: key,
+            totalDocsCount,
+            valuesResult
+        };
+        if (mostProbableType === "string") {
+            result = buildStringProperty(propertyProps);
+        } else if (mostProbableType === "reference") {
+            result = buildReferenceProperty(propertyProps);
+        } else {
+            result = {
+                type: mostProbableType
+            } as Property;
+        }
+
+        if (title) {
+            result.name = title;
+        }
+
+        const validation = buildValidation(propertyProps);
+        if (validation) {
+            result.validation = validation;
+        }
+    }
+
+    return result;
+}
+
+function buildPropertiesFromCount(
+    totalDocsCount: number,
+    typesCountRecord: TypesCountRecord,
+    valuesCountRecord?: ValuesCountRecord
+): Properties {
+    const res: Properties = {};
+    Object.entries(typesCountRecord).forEach(([key, typesCount]) => {
+        const mostProbableType = getMostProbableType(typesCount);
+        res[key] = buildPropertyFromCount(
+            key,
+            totalDocsCount,
+            mostProbableType,
+            typesCount,
+            valuesCountRecord ? valuesCountRecord[key] : undefined
+        );
+    });
+    return res;
+}
+
+function countMaxDocumentsUnder(typesCount: TypesCount) {
+    let count = 0;
+    Object.entries(typesCount).forEach(([type, value]) => {
+        if (typeof value === "object") {
+            count = Math.max(count, countMaxDocumentsUnder(value as TypesCountRecord));
+        } else {
+            count = Math.max(count, Number(value));
+        }
+    });
+    return count;
+}
+
+function getMostProbableTypeInArray(
+    array: any[],
+    getType: InferenceTypeBuilder
+): DataType {
+    const typesCount: TypesCount = {};
+    array.forEach((value) => {
+        increaseTypeCount(getType(value), typesCount, value, getType);
+    });
+    return getMostProbableType(typesCount);
+}
+
+function checkTypesCountHighVariability(typesCount: TypesCount) {
+    const maxCount = countMaxDocumentsUnder(typesCount);
+    let keysWithFewValues = 0;
+    Object.entries(typesCount.map ?? {}).forEach(([key, value]) => {
+        const count = countMaxDocumentsUnder(value);
+        if (count < maxCount / 3) {
+            keysWithFewValues++;
+        }
+    });
+    return keysWithFewValues / Object.entries(typesCount.map ?? {}).length > 0.5;
+}
+
+
+export function inferTypeFromValue(value: any): DataType {
+    if (value === null || value === undefined) return "string";
+    if (typeof value === "string") return "string";
+    if (typeof value === "number") return "number";
+    if (typeof value === "boolean") return "boolean";
+    if (Array.isArray(value)) return "array";
+    if (typeof value === "object") return "map";
+    return "string";
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./collection_builder";
+export * from "./util";
+export * from "./strings";