@lancedb/lancedb 0.37.1 → 0.38.0-beta.12

package/dist/schema.js

@@ -0,0 +1,387 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inferSchema = inferSchema;
+const apache_arrow_1 = require("apache-arrow");
+const arrow_type_1 = require("./arrow_type");
+const sanitize_1 = require("./sanitize");
+/**
+ * Infer the Arrow schema represented by a set of records.
+ *
+ * This is the intentionally small interface to schema inference. The stateful
+ * details of combining partial type evidence are encapsulated below so callers
+ * only need to provide records, an optional schema, and inference options.
+ */
+function inferSchema(data, schema, options) {
+    return new SchemaInferrer(schema, options).infer(data);
+}
+class SchemaInferrer {
+    providedSchema;
+    options;
+    fields = new FieldTree();
+    constructor(providedSchema, options) {
+        this.providedSchema = providedSchema;
+        this.options = options;
+    }
+    infer(data) {
+        for (const [row, record] of data.entries()) {
+            for (const [path, value] of recordPathsAndValues(record)) {
+                this.observe(path, value, row);
+            }
+        }
+        return this.providedSchema === undefined
+            ? new apache_arrow_1.Schema(fieldsFromTree(this.fields))
+            : new apache_arrow_1.Schema(matchingFields(this.providedSchema.fields, this.fields));
+    }
+    observe(path, value, row) {
+        const current = this.fields.get(path);
+        if (current === undefined) {
+            this.addField(path, value, row);
+        }
+        else if (this.providedSchema === undefined) {
+            this.updateInferredField(path, value, row, current);
+        }
+    }
+    addField(path, value, row) {
+        if (this.providedSchema !== undefined) {
+            this.addSchemaField(this.providedSchema, path, row);
+            return;
+        }
+        const evidence = this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
+        if (evidence === undefined) {
+            throw typeInferenceError(path, row);
+        }
+        const conflict = this.fields.set(path, evidence, (existing) => existing instanceof DeferredTypeEvidence && existing.isOnlyNulls());
+        if (conflict !== undefined) {
+            throw branchConflictError(conflict, row, "Struct");
+        }
+    }
+    addSchemaField(schema, path, row) {
+        const field = fieldAtPath(schema, path);
+        if (field === undefined) {
+            throw new Error(`Found field not in schema: ${path.join(".")} at row ${row}`);
+        }
+        const conflict = this.fields.set(path, field.type);
+        if (conflict !== undefined) {
+            throw branchConflictError(conflict, row, "Struct");
+        }
+    }
+    updateInferredField(path, value, row, current) {
+        const newType = this.inferType(value, path);
+        const deferred = DeferredTypeEvidence.from(value, row);
+        if (current instanceof FieldTree) {
+            if (deferred?.isOnlyNulls()) {
+                return;
+            }
+            throw schemaInferenceError(path, row, "Struct", describeEvidence(newType ?? deferred));
+        }
+        if (current instanceof DeferredTypeEvidence) {
+            this.resolveDeferredField(path, row, current, newType, deferred);
+            return;
+        }
+        if (newType !== undefined) {
+            if (!inferredTypesEqual(current, newType)) {
+                throw schemaInferenceError(path, row, describeEvidence(current), describeEvidence(newType));
+            }
+            return;
+        }
+        if (deferred === undefined || !deferred.matches(current)) {
+            throw schemaInferenceError(path, row, describeEvidence(current), describeEvidence(deferred));
+        }
+    }
+    resolveDeferredField(path, row, current, newType, deferred) {
+        if (newType !== undefined) {
+            if (!current.matches(newType)) {
+                throw schemaInferenceError(path, row, current.describe(), describeEvidence(newType));
+            }
+            this.fields.set(path, newType);
+            return;
+        }
+        if (deferred !== undefined) {
+            this.fields.set(path, current.merge(deferred));
+            return;
+        }
+        throw schemaInferenceError(path, row, current.describe(), describeEvidence(newType));
+    }
+    inferType(value, path) {
+        if (typeof value === "bigint") {
+            return new apache_arrow_1.Int64();
+        }
+        if (typeof value === "number") {
+            return new apache_arrow_1.Float64();
+        }
+        if (typeof value === "string") {
+            return this.options.dictionaryEncodeStrings
+                ? new apache_arrow_1.Dictionary(new apache_arrow_1.Utf8(), new apache_arrow_1.Int32())
+                : new apache_arrow_1.Utf8();
+        }
+        if (typeof value === "boolean") {
+            return new apache_arrow_1.Bool();
+        }
+        if (value instanceof Buffer) {
+            return new apache_arrow_1.Binary();
+        }
+        if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
+            const typedArray = (0, arrow_type_1.typedArrayToArrowType)(value);
+            return typedArray === undefined
+                ? undefined
+                : new apache_arrow_1.FixedSizeList(typedArray.length, new apache_arrow_1.Field("item", typedArray.elementType, true));
+        }
+        if (!Array.isArray(value) || value.length === 0) {
+            return undefined;
+        }
+        const configuredVector = path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
+        if (configuredVector !== undefined) {
+            return new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", (0, sanitize_1.sanitizeType)(configuredVector.type), true));
+        }
+        const itemType = this.inferArrayItemType(value, path);
+        if (itemType === undefined) {
+            return undefined;
+        }
+        return nameSuggestsVectorColumn(path[path.length - 1])
+            ? new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", new apache_arrow_1.Float32(), true))
+            : new apache_arrow_1.List(new apache_arrow_1.Field("item", itemType, true));
+    }
+    inferArrayItemType(values, path) {
+        let itemType;
+        const deferredItems = [];
+        for (const value of values) {
+            const candidate = this.inferType(value, path);
+            if (candidate === undefined) {
+                if (!isDeferredValue(value)) {
+                    return undefined;
+                }
+                deferredItems.push(value);
+            }
+            else if (itemType === undefined) {
+                itemType = candidate;
+            }
+            else if (!inferredTypesEqual(itemType, candidate)) {
+                return undefined;
+            }
+        }
+        if (itemType === undefined) {
+            return undefined;
+        }
+        return deferredItems.every((value) => deferredValueMatchesType(value, itemType))
+            ? itemType
+            : undefined;
+    }
+}
+/** Nulls and empty/all-null lists that do not determine a type by themselves. */
+class DeferredTypeEvidence {
+    values;
+    constructor(values) {
+        this.values = values;
+    }
+    static from(value, row) {
+        return isDeferredValue(value)
+            ? new DeferredTypeEvidence([{ value, row }])
+            : undefined;
+    }
+    isOnlyNulls() {
+        return this.values.every(({ value }) => value == null);
+    }
+    matches(type) {
+        return this.values.every(({ value }) => deferredValueMatchesType(value, type));
+    }
+    merge(other) {
+        return new DeferredTypeEvidence([...this.values, ...other.values]);
+    }
+    describe() {
+        const list = this.values.find(({ value }) => Array.isArray(value));
+        return list === undefined
+            ? "null"
+            : `List[${list.value.length}]`;
+    }
+    firstRow() {
+        return this.values[0].row;
+    }
+}
+/** Nested field state, kept separate from Arrow's eventual Struct types. */
+class FieldTree {
+    children = new Map();
+    get(path) {
+        let current = this;
+        for (const part of path) {
+            if (!(current instanceof FieldTree)) {
+                return undefined;
+            }
+            const child = current.children.get(part);
+            if (child === undefined) {
+                return undefined;
+            }
+            current = child;
+        }
+        return current;
+    }
+    set(path, value, canReplaceLeaf = () => false) {
+        let branch = this;
+        for (const [index, part] of path.slice(0, -1).entries()) {
+            const child = branch.children.get(part);
+            if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
+                const nextBranch = new FieldTree();
+                branch.children.set(part, nextBranch);
+                branch = nextBranch;
+            }
+            else if (child instanceof FieldTree) {
+                branch = child;
+            }
+            else {
+                return { path: path.slice(0, index + 1), value: child };
+            }
+        }
+        const name = path[path.length - 1];
+        const current = branch.children.get(name);
+        if (current instanceof FieldTree) {
+            return { path, value: current };
+        }
+        branch.children.set(name, value);
+        return undefined;
+    }
+    entries() {
+        return this.children.entries();
+    }
+    has(name) {
+        return this.children.has(name);
+    }
+}
+function isLeaf(value) {
+    return !(value instanceof FieldTree);
+}
+function fieldsFromTree(tree, path = []) {
+    const fields = [];
+    for (const [name, value] of tree.entries()) {
+        if (value instanceof FieldTree) {
+            fields.push(new apache_arrow_1.Field(name, new apache_arrow_1.Struct(fieldsFromTree(value, [...path, name])), true));
+        }
+        else if (value instanceof DeferredTypeEvidence) {
+            throw typeInferenceError([...path, name], value.firstRow());
+        }
+        else {
+            fields.push(new apache_arrow_1.Field(name, value, true));
+        }
+    }
+    return fields;
+}
+function matchingFields(fields, tree) {
+    const matches = [];
+    for (const field of fields) {
+        if (!tree.has(field.name)) {
+            continue;
+        }
+        const value = tree.get([field.name]);
+        if (value instanceof FieldTree) {
+            const struct = field.type;
+            matches.push(new apache_arrow_1.Field(field.name, new apache_arrow_1.Struct(matchingFields(struct.children, value)), field.nullable, field.metadata));
+        }
+        else {
+            matches.push(field);
+        }
+    }
+    return matches;
+}
+function* recordPathsAndValues(record, path = []) {
+    for (const [name, value] of Object.entries(record)) {
+        if (isRecord(value)) {
+            yield* recordPathsAndValues(value, [...path, name]);
+        }
+        else if (value !== undefined) {
+            yield [[...path, name], value];
+        }
+    }
+}
+function isRecord(value) {
+    return (typeof value === "object" &&
+        value !== null &&
+        !Array.isArray(value) &&
+        !(value instanceof RegExp) &&
+        !(value instanceof Date) &&
+        !(value instanceof Set) &&
+        !(value instanceof Map) &&
+        !(value instanceof Buffer) &&
+        !ArrayBuffer.isView(value));
+}
+function fieldAtPath(schema, path) {
+    let fields = schema.fields;
+    let field;
+    for (const [index, name] of path.entries()) {
+        field = fields.find((candidate) => candidate.name === name);
+        if (field === undefined || index === path.length - 1) {
+            return field;
+        }
+        if (!apache_arrow_1.DataType.isStruct(field.type)) {
+            return undefined;
+        }
+        fields = field.type.children;
+    }
+    return field;
+}
+function isDeferredValue(value) {
+    return (value == null || (Array.isArray(value) && value.every(isDeferredValue)));
+}
+function deferredValueMatchesType(value, type) {
+    if (value == null) {
+        return true;
+    }
+    if (!Array.isArray(value)) {
+        return false;
+    }
+    if (apache_arrow_1.DataType.isList(type)) {
+        return value.every((item) => deferredValueMatchesType(item, type.valueType));
+    }
+    if (apache_arrow_1.DataType.isFixedSizeList(type)) {
+        return (value.length === type.listSize &&
+            value.every((item) => deferredValueMatchesType(item, type.valueType)));
+    }
+    return false;
+}
+function inferredTypesEqual(current, candidate) {
+    if (apache_arrow_1.DataType.isDictionary(current)) {
+        return (apache_arrow_1.DataType.isDictionary(candidate) &&
+            current.isOrdered === candidate.isOrdered &&
+            inferredTypesEqual(current.indices, candidate.indices) &&
+            inferredTypesEqual(current.dictionary, candidate.dictionary));
+    }
+    if (apache_arrow_1.DataType.isList(current)) {
+        return (apache_arrow_1.DataType.isList(candidate) &&
+            current.valueField.name === candidate.valueField.name &&
+            current.valueField.nullable === candidate.valueField.nullable &&
+            inferredTypesEqual(current.valueType, candidate.valueType));
+    }
+    if (apache_arrow_1.DataType.isFixedSizeList(current)) {
+        return (apache_arrow_1.DataType.isFixedSizeList(candidate) &&
+            current.listSize === candidate.listSize &&
+            current.valueField.name === candidate.valueField.name &&
+            current.valueField.nullable === candidate.valueField.nullable &&
+            inferredTypesEqual(current.valueType, candidate.valueType));
+    }
+    return apache_arrow_1.util.compareTypes(current, candidate);
+}
+function describeEvidence(evidence) {
+    if (evidence === undefined) {
+        return "an unsupported value";
+    }
+    return evidence instanceof DeferredTypeEvidence
+        ? evidence.describe()
+        : evidence.toString();
+}
+function branchConflictError(conflict, row, candidate) {
+    return schemaInferenceError(conflict.path, row, conflict.value instanceof FieldTree
+        ? "Struct"
+        : describeEvidence(conflict.value), candidate);
+}
+function schemaInferenceError(path, row, currentType, newType) {
+    return new Error(`Failed to infer schema for data. Previously inferred type ${currentType} ` +
+        `but found ${newType} for field ${path.join(".")} at row ${row}. ` +
+        "Consider providing an explicit schema.");
+}
+function typeInferenceError(path, row) {
+    return new Error(`Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
+        "Consider providing an explicit schema.");
+}
+function nameSuggestsVectorColumn(name) {
+    const normalized = name.toLowerCase();
+    return normalized.includes("vector") || normalized.includes("embedding");
+}