@rebasepro/inference 0.17.3 → 0.18.0

package/LICENSE

@@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
-

package/README.md

@@ -8,6 +8,10 @@ Automatically infer Rebase collection property schemas from sample data — anal
 pnpm add @rebasepro/inference
 ```
 
+ESM-only: `"type": "module"` with no CommonJS build, so it is loaded with
+`import`. `require()` of it resolves only on Node 22.12+, which supports
+`require(esm)`.
+
 ## What This Package Does
 
 `@rebasepro/inference` examines arrays of data objects and produces `Properties` definitions compatible with the Rebase collection schema. It uses statistical analysis to determine the most probable type for each field, detect enum values, identify reference patterns, and build nested map/array structures. Used by the Rebase introspection pipeline and data import tools.

package/package.json

@@ -1,7 +1,26 @@
 {
   "name": "@rebasepro/inference",
-  "version": "0.17.3",
+  "version": "0.18.0",
+  "description": "Infer a Rebase collection schema from existing data.",
+  "keywords": [
+    "rebase",
+    "schema",
+    "inference",
+    "codegen"
+  ],
+  "homepage": "https://rebase.pro",
+  "bugs": {
+    "url": "https://github.com/rebasepro/rebase/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/rebasepro/rebase.git",
+    "directory": "packages/inference"
+  },
   "license": "MIT",
+  "engines": {
+    "node": ">=22.22.0"
+  },
   "type": "module",
   "publishConfig": {
     "access": "public"
@@ -9,10 +28,9 @@
   "main": "./dist/index.es.js",
   "module": "./dist/index.es.js",
   "types": "dist/index.d.ts",
-  "source": "src/index.ts",
   "dependencies": {
-    "@rebasepro/types": "0.17.3",
-    "@rebasepro/utils": "0.17.3"
+    "@rebasepro/types": "0.18.0",
+    "@rebasepro/utils": "0.18.0"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",
@@ -26,24 +44,20 @@
     ".": {
       "types": "./dist/index.d.ts",
       "development": "./dist/index.es.js",
-      "import": "./dist/index.es.js"
+      "import": "./dist/index.es.js",
+      "default": "./dist/index.es.js"
     },
     "./package.json": "./package.json"
   },
   "files": [
-    "dist",
-    "src"
+    "dist"
   ],
   "gitHead": "d935eefa5aa8d1009a2398cfac2c1e4ee9aeb6b6",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/rebasepro/rebase.git",
-    "directory": "packages/inference"
-  },
   "scripts": {
     "dev": "vite",
     "build": "vite build && tsc --emitDeclarationOnly -p tsconfig.prod.json && node ../../tooling/scripts/add-dts-extensions.mjs dist && node ../../tooling/scripts/assert-build-output.mjs",
     "test": "jest --passWithNoTests",
+    "test:watch": "jest --watch",
     "clean": "rm -rf dist && find ./src -name '*.js' -type f | xargs rm -f"
   }
 }

package/src/builders/reference_property_builder.ts

@@ -1,18 +0,0 @@
-import { findCommonInitialStringInPath } from "../strings";
-import { InferencePropertyBuilderProps } from "../types";
-import { Property } from "@rebasepro/types";
-
-export function buildReferenceProperty({
-    name,
-    totalDocsCount,
-    valuesResult
-}: InferencePropertyBuilderProps): Property {
-
-    const property: Property = {
-        name: name ?? "",
-        type: "reference",
-        path: findCommonInitialStringInPath(valuesResult) ?? "!!!FIX_ME!!!"
-    };
-
-    return property;
-}

package/src/builders/string_property_builder.ts

@@ -1,118 +0,0 @@
-import { findCommonInitialStringInPath } from "../strings";
-import { extractEnumFromValues } from "../util";
-import { FileType, Property, StringProperty } from "@rebasepro/types";
-import { InferencePropertyBuilderProps, ValuesCountEntry } from "../types";
-
-const IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif"];
-const AUDIO_EXTENSIONS = [".mp3", ".ogg", ".opus", ".aac"];
-const VIDEO_EXTENSIONS = [".avi", ".mp4"];
-
-const emailRegEx = /^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/;
-
-export function buildStringProperty({
-    name,
-    totalDocsCount,
-    valuesResult
-}: InferencePropertyBuilderProps): Property {
-
-    let stringProperty: Property = {
-        name: name ?? "",
-        type: "string"
-
-    };
-
-    if (valuesResult) {
-
-        const totalEntriesCount = valuesResult.values.length;
-        const totalValues = Array.from(valuesResult.valuesCount.keys()).length;
-
-        const config: Partial<StringProperty> = {};
-
-        const probablyAURL = valuesResult.values
-            .filter((value) => typeof value === "string" &&
-                value.toString().startsWith("http")).length > totalDocsCount / 3 * 2;
-        if (probablyAURL) {
-            config.url = true;
-        }
-
-        const probablyAnEmail = valuesResult.values
-            .filter((value) => typeof value === "string" &&
-                emailRegEx.test(value)).length > totalDocsCount / 3 * 2;
-        if (probablyAnEmail) {
-            config.email = true;
-        }
-
-        const probablyUserIds = valuesResult.values
-            .filter((value) => typeof value === "string" && value.length === 28 && !value.includes(" "))
-            .length > totalDocsCount / 3 * 2;
-        // No `admin.readOnly` here. This package is core — the field does not exist in
-        // `@rebasepro/types`, because `@rebasepro/cms-types` declares it — and a
-        // guess about whether a form should be editable is presentation, not schema.
-        // (It also would not have enforced anything: nothing on the server reads
-        // `readOnly`.) The inferred `isId` below is the part that carries meaning.
-
-        if (!probablyAnEmail &&
-            !probablyAURL &&
-            !probablyUserIds &&
-            !probablyAURL &&
-            totalValues < totalEntriesCount / 3
-        ) {
-            const enumValues = extractEnumFromValues(Array.from(valuesResult.valuesCount.keys()));
-
-            if (Object.keys(enumValues).length > 1)
-                config.enum = enumValues;
-        }
-
-        // regular string
-        if (!probablyAnEmail &&
-            !probablyAURL &&
-            !probablyUserIds &&
-            !probablyAURL &&
-            !config.enum) {
-            const fileType = probableFileType(valuesResult, totalDocsCount);
-            if (fileType) {
-                config.storage = {
-                    acceptedFiles: fileType as FileType[],
-                    storagePath: findCommonInitialStringInPath(valuesResult) ?? "/"
-                };
-            }
-        }
-
-        if (Object.keys(config).length > 0)
-            stringProperty = {
-                ...stringProperty,
-                ...config
-            } as StringProperty;
-    }
-
-    return stringProperty;
-}
-
-function probableFileType(valuesCount: ValuesCountEntry, totalDocsCount: number): false | FileType[] {
-    const isImage = (value: string) => IMAGE_EXTENSIONS.some((extension) => value.toString().endsWith(extension));
-    const isAudio = (value: string) => AUDIO_EXTENSIONS.some((extension) => value.toString().endsWith(extension));
-    const isVideo = (value: string) => VIDEO_EXTENSIONS.some((extension) => value.toString().endsWith(extension));
-
-    const stringValues = valuesCount.values.filter((v): v is string => typeof v === "string");
-
-    let imageCount = 0;
-    let audioCount = 0;
-    let videoCount = 0;
-
-    for (const value of stringValues) {
-        if (isImage(value)) imageCount++;
-        else if (isAudio(value)) audioCount++;
-        else if (isVideo(value)) videoCount++;
-    }
-
-    const totalMediaCount = imageCount + audioCount + videoCount;
-    if (totalMediaCount > (totalDocsCount * 2) / 3) {
-        const fileTypes: FileType[] = [];
-        if (imageCount > 0) fileTypes.push("image/*");
-        if (audioCount > 0) fileTypes.push("audio/*");
-        if (videoCount > 0) fileTypes.push("video/*");
-        return fileTypes.length > 0 ? fileTypes : false;
-    }
-
-    return false;
-}

package/src/builders/validation_builder.ts

@@ -1,18 +0,0 @@
-import { PropertyValidationSchema } from "@rebasepro/types";
-import { InferencePropertyBuilderProps } from "../types";
-
-export function buildValidation({
-    totalDocsCount,
-    valuesResult
-}: InferencePropertyBuilderProps): PropertyValidationSchema | undefined {
-
-    if (valuesResult) {
-        const totalEntriesCount = valuesResult.values.length;
-        if (totalDocsCount === totalEntriesCount)
-            return {
-                required: true
-            }
-    }
-
-    return undefined;
-}

package/src/collection_builder.ts

@@ -1,424 +0,0 @@
-import {
-    InferencePropertyBuilderProps,
-    TypesCount,
-    TypesCountRecord,
-    ValuesCountEntry,
-    ValuesCountRecord
-} from "./types";
-import { buildStringProperty } from "./builders/string_property_builder";
-import { buildValidation } from "./builders/validation_builder";
-import { buildReferenceProperty } from "./builders/reference_property_builder";
-import { extractEnumFromValues, mergeDeep, prettifyIdentifier, resolveEnumValues } from "./util";
-import { DataType, EnumValues, Properties, Property, StringProperty, Vector } from "@rebasepro/types";
-
-export type InferenceTypeBuilder = (value: unknown) => DataType;
-
-export async function buildEntityPropertiesFromData(
-    data: object[],
-    getType: InferenceTypeBuilder
-): Promise<Properties> {
-    const typesCount: TypesCountRecord = {};
-    const valuesCount: ValuesCountRecord = {};
-    if (data) {
-        data.forEach((entry) => {
-            if (entry) {
-                Object.entries(entry).forEach(([key, value]) => {
-                    if (key.startsWith("_")) return; // Ignore properties starting with _
-                    increaseMapTypeCount(typesCount, key, value, getType);
-                    increaseValuesCount(valuesCount, key, value, getType);
-                });
-            }
-        });
-    }
-    return buildPropertiesFromCount(data.length, typesCount, valuesCount);
-}
-
-export function buildPropertyFromData(
-    data: unknown[],
-    property: Property,
-    getType: InferenceTypeBuilder
-): Property {
-    const typesCount = {};
-    const valuesCount: ValuesCountRecord = {};
-    if (data) {
-        data.forEach((entry) => {
-            increaseTypeCount(property.type, typesCount, entry, getType);
-            increaseValuesCount(valuesCount, "inferred_prop", entry, getType);
-        });
-    }
-    const enumValues = "enum" in property ? resolveEnumValues(property["enum"] as EnumValues) : undefined;
-    if (enumValues) {
-        const newEnumValues = extractEnumFromValues(Array.from(valuesCount["inferred_prop"].valuesCount.keys()));
-        return {
-            ...property,
-            enum: [...newEnumValues, ...enumValues]
-        } as StringProperty;
-    }
-    const generatedProperty = buildPropertyFromCount(
-        "inferred_prop",
-        data.length,
-        property.type,
-        typesCount,
-        valuesCount["inferred_prop"]
-    );
-    return mergeDeep(generatedProperty, property);
-}
-
-export function buildPropertiesOrder(
-    properties: Properties,
-    propertiesOrder?: string[],
-    priorityKeys?: string[]
-): string[] {
-    const lowerCasePriorityKeys = (priorityKeys ?? []).map((key) => key.toLowerCase());
-
-    function propOrder(s: string) {
-        const k = s.toLowerCase();
-        if (lowerCasePriorityKeys.includes(k)) return 4;
-        if (k === "title" || k === "name") return 3;
-        if (k.includes("title") || k.includes("name")) return 2;
-        if (k.includes("image") || k.includes("picture")) return 1;
-        return 0;
-    }
-
-    function deriveOrder(keys: string[]): string[] {
-        return [...keys]
-            .sort() // alphabetically
-            .sort((a, b) => propOrder(b) - propOrder(a));
-    }
-
-    // A caller-supplied order is an answer, not an input. It used to be handed
-    // to the same alphabetical + priority sort as a derived order, which threw
-    // it away entirely — and because `Array.prototype.sort` sorts in place, the
-    // caller's own array was silently reordered as a side effect.
-    if (propertiesOrder) {
-        const placed = new Set(propertiesOrder);
-        // Properties the caller did not place — a newly inferred column, say —
-        // still have to appear, so they follow in derived order.
-        const rest = Object.keys(properties).filter(key => !placed.has(key));
-        return [...propertiesOrder, ...deriveOrder(rest)];
-    }
-
-    return deriveOrder(Object.keys(properties));
-}
-
-/**
- * @param type
- * @param typesCount
- * @param fieldValue
- * @param getType
- */
-function increaseTypeCount(
-    type: DataType,
-    typesCount: TypesCount,
-    fieldValue: unknown,
-    getType: InferenceTypeBuilder
-) {
-    if (type === "map") {
-        if (fieldValue) {
-            let mapTypesCount = typesCount[type];
-            if (!mapTypesCount) {
-                mapTypesCount = {};
-                typesCount[type] = mapTypesCount;
-            }
-            Object.entries(fieldValue as Record<string, unknown>).forEach(([key, value]) => {
-                increaseMapTypeCount(mapTypesCount as TypesCountRecord, key, value, getType);
-            });
-        }
-    } else if (type === "array") {
-        let arrayTypesCount = typesCount[type];
-        if (!arrayTypesCount) {
-            arrayTypesCount = {};
-            typesCount[type] = arrayTypesCount;
-        }
-        if (fieldValue && Array.isArray(fieldValue) && fieldValue.length > 0) {
-            const arrayType = getMostProbableTypeInArray(fieldValue, getType);
-            if (arrayType === "map") {
-                let mapTypesCount = arrayTypesCount[arrayType];
-                if (!mapTypesCount) {
-                    mapTypesCount = {};
-                }
-                fieldValue.forEach((value) => {
-                    if (value && typeof value === "object" && !Array.isArray(value)) { // Ensure value is an object for Object.entries
-                        Object.entries(value).forEach(([key, v]) =>
-                            increaseMapTypeCount(mapTypesCount, key, v, getType)
-                        );
-                    }
-                });
-                arrayTypesCount[arrayType] = mapTypesCount;
-            } else {
-                if (!arrayTypesCount[arrayType]) arrayTypesCount[arrayType] = 1;
-                else arrayTypesCount[arrayType] = Number(arrayTypesCount[arrayType]) + 1;
-            }
-        }
-    } else {
-        if (!typesCount[type]) typesCount[type] = 1;
-        else typesCount[type] = Number(typesCount[type]) + 1;
-    }
-}
-
-function increaseMapTypeCount(
-    typesCountRecord: TypesCountRecord,
-    key: string,
-    fieldValue: unknown,
-    getType: InferenceTypeBuilder
-) {
-    if (key.startsWith("_")) return; // Ignore properties starting with _
-
-    let typesCount: TypesCount = typesCountRecord[key];
-    if (!typesCount) {
-        typesCount = {};
-        typesCountRecord[key] = typesCount;
-    }
-
-    if (fieldValue != null) {
-        // Check that fieldValue is not null or undefined before proceeding
-        const type = getType(fieldValue);
-        increaseTypeCount(type, typesCount, fieldValue, getType);
-    }
-}
-
-function increaseValuesCount(
-    typeValuesRecord: ValuesCountRecord,
-    key: string,
-    fieldValue: unknown,
-    getType: InferenceTypeBuilder
-) {
-    if (key.startsWith("_")) return; // Ignore properties starting with _
-
-    const type = getType(fieldValue);
-
-    let valuesRecord: {
-        values: unknown[];
-        valuesCount: Map<unknown, number>;
-        map?: ValuesCountRecord;
-    } = typeValuesRecord[key];
-
-    if (!valuesRecord) {
-        valuesRecord = {
-            values: [],
-            valuesCount: new Map()
-        };
-        typeValuesRecord[key] = valuesRecord;
-    }
-
-    if (type === "map") {
-        let mapValuesRecord: ValuesCountRecord | undefined = valuesRecord.map;
-        if (!mapValuesRecord) {
-            mapValuesRecord = {};
-            valuesRecord.map = mapValuesRecord;
-        }
-        if (fieldValue)
-            Object.entries(fieldValue as Record<string, unknown>).forEach(([subKey, value]) =>
-                increaseValuesCount(mapValuesRecord as ValuesCountRecord, subKey, value, getType)
-            );
-    } else if (type === "array") {
-        if (Array.isArray(fieldValue)) {
-            fieldValue.forEach((value) => {
-                valuesRecord.values.push(value);
-                valuesRecord.valuesCount.set(value, (valuesRecord.valuesCount.get(value) ?? 0) + 1);
-            });
-        }
-    } else {
-        if (fieldValue !== null && fieldValue !== undefined) {
-            valuesRecord.values.push(fieldValue);
-            valuesRecord.valuesCount.set(fieldValue, (valuesRecord.valuesCount.get(fieldValue) ?? 0) + 1);
-        }
-    }
-}
-
-function getHighestTypesCount(typesCount: TypesCount): number {
-    let highestCount = 0;
-    Object.entries(typesCount).forEach(([type, count]) => {
-        let countValue = 0;
-        if (type === "map") {
-            countValue = getHighestRecordCount(count as TypesCountRecord);
-        } else if (type === "array") {
-            countValue = getHighestTypesCount(count as TypesCount);
-        } else {
-            countValue = Number(count);
-        }
-        if (countValue > highestCount) {
-            highestCount = countValue;
-        }
-    });
-
-    return highestCount;
-}
-
-function getHighestRecordCount(record: TypesCountRecord): number {
-    return Object.entries(record)
-        .map(([key, typesCount]) => getHighestTypesCount(typesCount))
-        .reduce((a, b) => Math.max(a, b), 0);
-}
-
-function getMostProbableType(typesCount: TypesCount): DataType {
-    let highestCount = -1;
-    let probableType: DataType = "string"; // default
-    Object.entries(typesCount).forEach(([type, count]) => {
-        let countValue;
-        if (type === "map") {
-            countValue = getHighestRecordCount(count as TypesCountRecord);
-        } else if (type === "array") {
-            countValue = getHighestTypesCount(count as TypesCount);
-        } else {
-            countValue = Number(count);
-        }
-        if (countValue > highestCount) {
-            highestCount = countValue;
-            probableType = type as DataType;
-        }
-    });
-    return probableType;
-}
-
-function buildPropertyFromCount(
-    key: string,
-    totalDocsCount: number,
-    mostProbableType: DataType,
-    typesCount: TypesCount,
-    valuesResult?: ValuesCountEntry
-): Property {
-    let title: string | undefined;
-
-    if (key) {
-        title = prettifyIdentifier(key);
-    }
-
-    let result: Property | undefined = undefined;
-    if (mostProbableType === "map") {
-        const highVariability = checkTypesCountHighVariability(typesCount);
-        if (highVariability) {
-            result = {
-                type: "map",
-                name: title ?? key ?? "",
-                keyValue: true,
-                properties: {}
-            };
-        }
-        const properties = buildPropertiesFromCount(
-            totalDocsCount,
-            typesCount.map as TypesCountRecord,
-            valuesResult ? valuesResult.mapValues : undefined
-        );
-        result = {
-            type: "map",
-            name: title ?? key ?? "",
-            properties
-        };
-    } else if (mostProbableType === "array") {
-        const arrayTypesCount = typesCount.array as TypesCount;
-        const arrayMostProbableType = getMostProbableType(arrayTypesCount);
-        const of = buildPropertyFromCount(
-            key,
-            totalDocsCount,
-            arrayMostProbableType,
-            arrayTypesCount,
-            valuesResult
-        );
-        result = {
-            type: "array",
-            name: title ?? key ?? "",
-            of
-        };
-    }
-
-    if (!result) {
-        const propertyProps: InferencePropertyBuilderProps = {
-            name: key,
-            totalDocsCount,
-            valuesResult
-        };
-        if (mostProbableType === "string") {
-            result = buildStringProperty(propertyProps);
-        } else if (mostProbableType === "reference") {
-            result = buildReferenceProperty(propertyProps);
-        } else {
-            result = {
-                type: mostProbableType
-            } as Property;
-        }
-
-        if (title) {
-            result.name = title;
-        }
-
-        const validation = buildValidation(propertyProps);
-        if (validation) {
-            result.validation = validation;
-        }
-    }
-
-    return result;
-}
-
-function buildPropertiesFromCount(
-    totalDocsCount: number,
-    typesCountRecord: TypesCountRecord,
-    valuesCountRecord?: ValuesCountRecord
-): Properties {
-    const res: Properties = {};
-    Object.entries(typesCountRecord).forEach(([key, typesCount]) => {
-        const mostProbableType = getMostProbableType(typesCount);
-        res[key] = buildPropertyFromCount(
-            key,
-            totalDocsCount,
-            mostProbableType,
-            typesCount,
-            valuesCountRecord ? valuesCountRecord[key] : undefined
-        );
-    });
-    return res;
-}
-
-function countMaxDocumentsUnder(typesCount: TypesCount) {
-    let count = 0;
-    Object.entries(typesCount).forEach(([type, value]) => {
-        if (typeof value === "object") {
-            count = Math.max(count, countMaxDocumentsUnder(value as TypesCountRecord));
-        } else {
-            count = Math.max(count, Number(value));
-        }
-    });
-    return count;
-}
-
-function getMostProbableTypeInArray(
-    array: unknown[],
-    getType: InferenceTypeBuilder
-): DataType {
-    const typesCount: TypesCount = {};
-    array.forEach((value) => {
-        // A hole in the array says nothing about what the array holds, so it
-        // gets no vote. This used to fall out of `getType` answering "map" for
-        // null and `increaseTypeCount` then skipping falsy map values; now that
-        // null infers as "string" the skip has to be explicit, or every array
-        // with a gap in it would drift towards "string".
-        if (value === null || value === undefined) return;
-        increaseTypeCount(getType(value), typesCount, value, getType);
-    });
-    return getMostProbableType(typesCount);
-}
-
-function checkTypesCountHighVariability(typesCount: TypesCount) {
-    const maxCount = countMaxDocumentsUnder(typesCount);
-    let keysWithFewValues = 0;
-    Object.entries(typesCount.map ?? {}).forEach(([key, value]) => {
-        const count = countMaxDocumentsUnder(value);
-        if (count < maxCount / 3) {
-            keysWithFewValues++;
-        }
-    });
-    return keysWithFewValues / Object.entries(typesCount.map ?? {}).length > 0.5;
-}
-
-
-export function inferTypeFromValue(value: unknown): DataType {
-    if (value === null || value === undefined) return "string";
-    if (value instanceof Vector || (value && typeof value === "object" && "__type" in value && (value as Record<string, unknown>).__type === "Vector")) return "vector";
-    if (typeof value === "string") return "string";
-    if (typeof value === "number") return "number";
-    if (typeof value === "boolean") return "boolean";
-    if (Array.isArray(value)) return "array";
-    if (typeof value === "object") return "map";
-    return "string";
-}

package/src/index.ts

@@ -1,3 +0,0 @@
-export * from "./collection_builder";
-export * from "./util";
-export * from "./strings";