@based/schema 5.0.2 → 5.0.4-alpha.1

package/dist/dbSchema.d.ts

@@ -0,0 +1,8 @@
+import { StrictSchema } from "./types.js";
+export type DbSchema = StrictSchema & {
+    lastId: number;
+    hash: number;
+};
+export type SchemaChecksum = number;
+export declare const strictSchemaToDbSchema: (schema: StrictSchema) => DbSchema;
+//# sourceMappingURL=dbSchema.d.ts.map

package/dist/dbSchema.js

@@ -0,0 +1,59 @@
+import { hash } from '@based/hash';
+import { getPropType } from "./parse/utils.js";
+import { deepCopy } from '@based/utils';
+export const strictSchemaToDbSchema = (schema) => {
+    // @ts-ignore
+    let dbSchema = deepCopy(schema);
+    // reserve 1 for root (even if you dont have it)
+    dbSchema.lastId = 1;
+    // Make the _root type
+    if (dbSchema.props) {
+        for (const key in dbSchema.props) {
+            const prop = dbSchema.props[key];
+            const propType = getPropType(prop);
+            let refProp;
+            if (propType === 'reference') {
+                refProp = prop;
+            }
+            else if (propType === 'references') {
+                refProp = prop.items;
+                prop.items = refProp;
+            }
+            if (refProp) {
+                const type = dbSchema.types[refProp.ref];
+                const inverseKey = '_' + key;
+                dbSchema.types[refProp.ref] = {
+                    ...type,
+                    props: {
+                        ...type.props,
+                        [inverseKey]: {
+                            items: {
+                                ref: '_root',
+                                prop: key,
+                            },
+                        },
+                    },
+                };
+                refProp.prop = inverseKey;
+            }
+        }
+        dbSchema.types ??= {};
+        // @ts-ignore This creates an internal type to use for root props
+        dbSchema.types._root = {
+            id: 1,
+            props: dbSchema.props,
+        };
+        delete dbSchema.props;
+    }
+    // Assign typeIds
+    for (const typeName in dbSchema.types) {
+        if (!('id' in dbSchema.types[typeName])) {
+            dbSchema.lastId++;
+            dbSchema.types[typeName].id = dbSchema.lastId;
+        }
+    }
+    delete dbSchema.hash;
+    dbSchema.hash = hash(dbSchema);
+    return dbSchema;
+};
+//# sourceMappingURL=dbSchema.js.map

package/dist/def/addEdges.js

@@ -2,7 +2,7 @@ import { getPropType } from '../index.js';
 import { DEFAULT_MAP } from './defaultMap.js';
 import { fillEmptyMain } from './fillEmptyMain.js';
 import { TYPE_INDEX_MAP, REFERENCES, REFERENCE, ENUM, NUMBER, } from './types.js';
-import { getPropLen, isSeparate, parseMinMaxStep } from './utils.js';
+import { getPropLen, isSeparate, parseMinMaxStep, sortMainProps, } from './utils.js';
 import { defaultValidation, VALIDATION_MAP } from './validation.js';
 export const addEdges = (prop, refProp) => {
     const mainEdges = [];
@@ -38,7 +38,7 @@ export const addEdges = (prop, refProp) => {
                 separate,
                 path: [...prop.path, key],
                 default: edgeProp.default ?? DEFAULT_MAP[typeIndex],
-                start: prop.edgeMainLen,
+                // start: prop.edgeMainLen,
             };
             if (!separate) {
                 mainEdges.push(edge);
@@ -55,7 +55,6 @@ export const addEdges = (prop, refProp) => {
             if (edge.typeIndex !== NUMBER && edge.step === undefined) {
                 edge.step = 1;
             }
-            prop.edgeMainLen += edge.len;
             if (edge.typeIndex === ENUM) {
                 edge.enum = Array.isArray(refProp[key])
                     ? refProp[key]
@@ -75,11 +74,14 @@ export const addEdges = (prop, refProp) => {
             if (separate) {
                 prop.reverseSeperateEdges[edge.prop] = edge;
             }
-            else {
-                prop.reverseMainEdges[edge.start] = edge;
-            }
         }
     }
+    mainEdges.sort(sortMainProps);
+    for (const edge of mainEdges) {
+        edge.start = prop.edgeMainLen;
+        prop.edgeMainLen += edge.len;
+        prop.reverseMainEdges[edge.start] = edge;
+    }
     prop.edgeMainEmpty = fillEmptyMain(mainEdges, prop.edgeMainLen);
 };
 //# sourceMappingURL=addEdges.js.map

package/dist/def/typeDef.d.ts

@@ -1,5 +1,4 @@
-import { SchemaObject, StrictSchemaType, SchemaLocales } from '../index.js';
-import { SchemaTypeDef, SchemaTypesParsed } from './types.js';
+import { SchemaTypeDef } from './types.js';
 import { StrictSchema } from '../types.js';
 export declare const updateTypeDefs: (schema: StrictSchema) => {
     schemaTypesParsed: {
@@ -9,5 +8,4 @@ export declare const updateTypeDefs: (schema: StrictSchema) => {
         [id: number]: SchemaTypeDef;
     };
 };
-export declare const createSchemaTypeDef: (typeName: string, type: StrictSchemaType | SchemaObject, parsed: SchemaTypesParsed, locales: Partial<SchemaLocales>, result?: Partial<SchemaTypeDef>, path?: string[], top?: boolean) => SchemaTypeDef;
 //# sourceMappingURL=typeDef.d.ts.map

package/dist/def/typeDef.js

@@ -1,10 +1,10 @@
 import { isPropType, getPropType, } from '../index.js';
 import { setByPath } from '@based/utils';
-import { TYPE_INDEX_MAP, REFERENCES, REFERENCE, NUMBER, BLOCK_CAPACITY_MAX, BLOCK_CAPACITY_DEFAULT, BLOCK_CAPACITY_MIN, ALIAS, ALIASES, VECTOR, COLVEC, } from './types.js';
+import { TYPE_INDEX_MAP, REFERENCES, REFERENCE, NUMBER, BLOCK_CAPACITY_MAX, BLOCK_CAPACITY_DEFAULT, BLOCK_CAPACITY_MIN, VECTOR, COLVEC, CARDINALITY, } from './types.js';
 import { DEFAULT_MAP } from './defaultMap.js';
 import { makeSeparateTextSort } from './makeSeparateTextSort.js';
 import { makeSeparateSort } from './makeSeparateSort.js';
-import { getPropLen, isSeparate, parseMinMaxStep, schemaVectorBaseTypeToEnum, } from './utils.js';
+import { getPropLen, isSeparate, parseMinMaxStep, reorderProps, schemaVectorBaseTypeToEnum, sortMainProps, cardinalityModeToEnum, } from './utils.js';
 import { addEdges } from './addEdges.js';
 import { createEmptyDef } from './createEmptyDef.js';
 import { fillEmptyMain, isZeroes } from './fillEmptyMain.js';
@@ -12,14 +12,6 @@ import { defaultValidation, VALIDATION_MAP } from './validation.js';
 export const updateTypeDefs = (schema) => {
     const schemaTypesParsed = {};
     const schemaTypesParsedById = {};
-    for (const typeName in schemaTypesParsed) {
-        if (typeName in schema.types) {
-            continue;
-        }
-        const id = schemaTypesParsed[typeName].id;
-        delete schemaTypesParsed[typeName];
-        delete schemaTypesParsedById[id];
-    }
     for (const typeName in schema.types) {
         const type = schema.types[typeName];
         if (!type.id) {
@@ -45,33 +37,7 @@ export const updateTypeDefs = (schema) => {
     }
     return { schemaTypesParsed, schemaTypesParsedById };
 };
-function propIndexOffset(prop) {
-    if (!prop.separate) {
-        return 0;
-    }
-    switch (prop.typeIndex) {
-        case REFERENCES:
-        case REFERENCE:
-            return -300;
-        case ALIAS:
-        case ALIASES:
-        case COLVEC:
-            return 300;
-        default:
-            return 0;
-    }
-}
-function reorderProps(props) {
-    props.sort((a, b) => a.prop + propIndexOffset(a) - (b.prop + propIndexOffset(b)));
-    // Reassign prop indices
-    let lastProp = 0;
-    for (const p of props) {
-        if (p.separate) {
-            p.prop = ++lastProp;
-        }
-    }
-}
-export const createSchemaTypeDef = (typeName, type, parsed, locales, result = createEmptyDef(typeName, type, locales), path = [], top = true) => {
+const createSchemaTypeDef = (typeName, type, parsed, locales, result = createEmptyDef(typeName, type, locales), path = [], top = true) => {
     if (top) {
         if (result.id == 0) {
             if ('id' in type) {
@@ -171,6 +137,11 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = cr
             if (prop.typeIndex === VECTOR || prop.typeIndex === COLVEC) {
                 prop.vectorBaseType = schemaVectorBaseTypeToEnum(schemaProp.baseType ?? 'number');
             }
+            if (prop.typeIndex === CARDINALITY) {
+                prop.cardinalityMode ??= cardinalityModeToEnum((schemaProp.mode ??= 'sparse'));
+                const prec = typeName == '_root' ? 14 : 8;
+                prop.cardinalityPrecision ??= schemaProp.precision ??= prec;
+            }
             if (isPropType('enum', schemaProp)) {
                 prop.enum = Array.isArray(schemaProp) ? schemaProp : schemaProp.enum;
                 prop.reverseEnum = {};
@@ -247,15 +218,16 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = cr
                     }
                 }
             }
-            else {
-                if (!result.mainLen) {
-                    len += 2;
-                }
-                len += 1;
-                f.start = result.mainLen;
-                result.mainLen += f.len;
-                setByPath(result.tree, f.path, f);
+        }
+        const mainProps = vals.filter((v) => !v.separate).sort(sortMainProps);
+        for (const f of mainProps) {
+            if (!result.mainLen) {
+                len += 2;
             }
+            len += 1;
+            f.start = result.mainLen;
+            result.mainLen += f.len;
+            setByPath(result.tree, f.path, f);
         }
         if (result.hasSeperateDefaults) {
             result.separateDefaults.bufferTmp = new Uint8Array(biggestSeperatePropDefault + 1);

package/dist/def/types.d.ts

@@ -33,6 +33,8 @@ export type PropDef = {
     default: any;
     vectorBaseType?: VectorBaseType;
     vectorSize?: number;
+    cardinalityMode?: number;
+    cardinalityPrecision?: number;
     edgeMainLen?: 0;
     hasDefaultEdges?: boolean;
     reverseEnum?: {

package/dist/def/utils.d.ts

@@ -1,9 +1,13 @@
 import { PropDef, PropDefEdge, VectorBaseType } from './types.js';
-import { SchemaProp, SchemaVectorBaseType } from '../types.js';
+import { SchemaProp, SchemaVectorBaseType, HLLRegisterRepresentation } from '../types.js';
 export declare function isSeparate(schemaProp: SchemaProp, len: number): boolean;
 export declare const propIsSigned: (prop: PropDef | PropDefEdge) => boolean;
 export declare const propIsNumerical: (prop: PropDef | PropDefEdge) => boolean;
 export declare const schemaVectorBaseTypeToEnum: (vector: SchemaVectorBaseType) => VectorBaseType;
+export declare const cardinalityModeToEnum: (mode: HLLRegisterRepresentation) => number;
 export declare function getPropLen(schemaProp: SchemaProp): number;
 export declare const parseMinMaxStep: (val: any) => string | number;
+export declare const sortMainProps: (a: PropDef | PropDefEdge, b: PropDef | PropDefEdge) => 0 | 1 | -1;
+export declare const propIndexOffset: (prop: PropDef) => 0 | 300 | -300;
+export declare const reorderProps: (props: PropDef[]) => void;
 //# sourceMappingURL=utils.d.ts.map

package/dist/def/utils.js

@@ -1,5 +1,5 @@
-import { INT16, INT32, INT8, UINT16, UINT32, UINT8, NUMBER, TIMESTAMP, SIZE_MAP, VECTOR_BASE_TYPE_SIZE_MAP, VectorBaseType, } from './types.js';
-import { isPropType } from '../types.js';
+import { INT16, INT32, INT8, UINT16, UINT32, UINT8, NUMBER, TIMESTAMP, SIZE_MAP, VECTOR_BASE_TYPE_SIZE_MAP, VectorBaseType, REVERSE_SIZE_MAP, REFERENCES, REFERENCE, ALIAS, ALIASES, COLVEC, } from './types.js';
+import { isPropType, } from '../types.js';
 import { getPropType } from '../parse/utils.js';
 import { convertToTimestamp } from '@based/utils';
 export function isSeparate(schemaProp, len) {
@@ -50,6 +50,12 @@ export const schemaVectorBaseTypeToEnum = (vector) => {
             return VectorBaseType.Float64;
     }
 };
+export const cardinalityModeToEnum = (mode) => {
+    if (mode === 'dense')
+        return 1;
+    else
+        0;
+};
 export function getPropLen(schemaProp) {
     let len = SIZE_MAP[getPropType(schemaProp)];
     if (isPropType('string', schemaProp) ||
@@ -86,4 +92,44 @@ export const parseMinMaxStep = (val) => {
         return val;
     }
 };
+export const sortMainProps = (a, b) => {
+    const sizeA = REVERSE_SIZE_MAP[a.typeIndex];
+    const sizeB = REVERSE_SIZE_MAP[b.typeIndex];
+    if (sizeA === 8) {
+        return -1;
+    }
+    if (sizeA === 4 && sizeB !== 8) {
+        return -1;
+    }
+    if (sizeA === sizeB) {
+        return 0;
+    }
+    return 1;
+};
+export const propIndexOffset = (prop) => {
+    if (!prop.separate) {
+        return 0;
+    }
+    switch (prop.typeIndex) {
+        case REFERENCES:
+        case REFERENCE:
+            return -300;
+        case ALIAS:
+        case ALIASES:
+        case COLVEC:
+            return 300;
+        default:
+            return 0;
+    }
+};
+export const reorderProps = (props) => {
+    props.sort((a, b) => a.prop + propIndexOffset(a) - (b.prop + propIndexOffset(b)));
+    // Reassign prop indices
+    let lastProp = 0;
+    for (const p of props) {
+        if (p.separate) {
+            p.prop = ++lastProp;
+        }
+    }
+};
 //# sourceMappingURL=utils.js.map

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 export * from './types.js';
+export * from './dbSchema.js';
 export * from './parse/index.js';
 export * from './lang.js';
 export * from './def/validation.js';

package/dist/index.js

@@ -1,4 +1,5 @@
 export * from './types.js';
+export * from './dbSchema.js';
 export * from './parse/index.js';
 export * from './lang.js';
 export * from './def/validation.js';

package/dist/infer.d.ts

@@ -1,14 +1,24 @@
 import { Schema } from './types.js';
-type InferString = string;
-type InferNumber = number;
-type InferBoolean = boolean;
-type InferText = string;
-type InferJson = any;
-type InferTimestamp = number;
-type InferBinary = Uint8Array;
-type InferCardinality = number;
-type InferVector = Float32Array;
-type InferColvec = Float32Array;
+type TypedArray = Uint8Array | Float32Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array;
+type TypeMap = {
+    string: string;
+    number: number;
+    int8: number;
+    uint8: number;
+    int16: number;
+    uint16: number;
+    int32: number;
+    uint32: number;
+    boolean: boolean;
+    text: string;
+    json: any;
+    timestamp: number | string | Date;
+    binary: Uint8Array;
+    cardinality: number;
+    vector: TypedArray;
+    colvec: TypedArray;
+    alias: string;
+};
 type InferEnum<T extends readonly (string | number | boolean)[]> = T[number];
 type InferReference<RefName extends string, Types> = RefName extends keyof Types ? {
     id: number;
@@ -25,38 +35,20 @@ type InferObject<T extends Record<string, any>, Types> = {
 };
 type InferSet<T, Types> = InferProp<T, Types>[];
 type InferProp<T, Types> = T extends {
-    type: 'string';
-} ? InferString : T extends {
-    type: 'number' | 'int8' | 'uint8' | 'int16' | 'uint16' | 'int32' | 'uint32';
-} ? InferNumber : T extends {
-    type: 'boolean';
-} ? InferBoolean : T extends {
-    type: 'text';
-} ? InferText : T extends {
-    type: 'json';
-} ? InferJson : T extends {
-    type: 'timestamp';
-} ? InferTimestamp : T extends {
-    type: 'binary';
-} ? InferBinary : T extends {
-    type: 'cardinality';
-} ? InferCardinality : T extends {
-    type: 'vector';
-} ? InferVector : T extends {
-    type: 'colvec';
-} ? InferColvec : T extends {
-    enum: infer E;
-} ? E extends readonly (string | number | boolean)[] ? InferEnum<E> : never : T extends {
-    ref: infer R extends string;
-} ? InferReference<R, Types> : T extends {
+    props: infer P;
+} ? InferObject<P, Types> : T extends {
     items: {
         ref: infer R extends string;
     };
 } ? InferReferences<R, Types> : T extends {
     items: infer I;
 } ? InferSet<I, Types> : T extends {
-    props: infer P;
-} ? InferObject<P, Types> : T extends string ? T extends 'string' ? InferString : T extends 'number' | 'int8' | 'uint8' | 'int16' | 'uint16' | 'int32' | 'uint32' ? InferNumber : T extends 'boolean' ? InferBoolean : T extends 'text' ? InferText : T extends 'json' ? InferJson : T extends 'timestamp' ? InferTimestamp : T extends 'binary' ? InferBinary : T extends 'cardinality' ? InferCardinality : T extends 'vector' ? InferVector : T extends 'colvec' ? InferColvec : never : never;
+    ref: infer R extends string;
+} ? InferReference<R, Types> : T extends {
+    enum: infer E;
+} ? E extends readonly (string | number | boolean)[] ? InferEnum<E> : never : T extends {
+    type: infer U;
+} ? U extends keyof TypeMap ? TypeMap[U] : never : T extends keyof TypeMap ? TypeMap[T] : never;
 type InferSchemaType<T, Types> = T extends {
     props: infer Props;
 } ? {

package/dist/parse/props.js

@@ -504,6 +504,18 @@ p.cardinality = propParser(STUB, {
     default(val, prop, ctx) {
         return isDefault(val, prop, ctx);
     },
+    mode(val, prop, ctx) {
+        if (!['dense', 'sparse'].includes(val)) {
+            throw Error(INVALID_VALUE);
+        }
+        p.mode = val;
+    },
+    precision(val, prop, ctx) {
+        if (val < 2 || val > 16) {
+            throw Error(INVALID_VALUE);
+        }
+        p.precision = val;
+    },
 }, 0);
 p.json = propParser(STUB, {
     default(val, prop, ctx) {

package/dist/types.d.ts

@@ -2,7 +2,7 @@ import type { LangName } from './lang.js';
 import type { Validation } from './def/validation.js';
 type Role = 'title' | 'source' | 'media' | string;
 export declare const numberDisplays: readonly ["short", "human", "ratio", "bytes", "euro", "dollar", "pound", "meter"];
-export declare const dateDisplays: readonly ["date", "date-time", "date-time-text", "date-time-human", "time", "time-precise"];
+export declare const dateDisplays: readonly ["date", "date-time", "date-time-text", "date-time-human", "date-time-human-short", "time", "time-precise"];
 export declare const stringFormats: readonly ["alpha", "alphaLocales", "alphanumeric", "alphanumericLocales", "ascii", "base32", "base58", "base64", "BIC", "btcAddress", "clike", "code", "creditCard", "css", "currency", "dataURI", "EAN", "email", "ethereumAddress", "FQDN", "hexadecimal", "hexColor", "HSL", "html", "IBAN", "identityCard", "IMEI", "IP", "IPRange", "ISBN", "ISIN", "ISO31661Alpha2", "ISO31661Alpha3", "ISO4217", "ISO6391", "ISO8601", "ISRC", "ISSN", "javascript", "json", "JWT", "latLong", "licensePlate", "lowercase", "luhnNumber", "MACAddress", "magnetURI", "markdown", "MD5", "mimeType", "mobilePhone", "mobilePhoneLocales", "octal", "password", "passportNumber", "port", "postalCode", "postalCodeLocales", "python", "RFC3339", "rgbColor", "rust", "semVer", "slug", "surrogatePair", "taxID", "typescript", "uppercase", "URL", "UUID", "VAT", "multiline"];
 type DateDisplay = (typeof dateDisplays)[number];
 type NumberDisplay = (typeof numberDisplays)[number] | `round-${number}`;
@@ -85,11 +85,12 @@ export type SchemaBoolean = Prop<{
     type: 'boolean';
     default?: boolean;
 }>;
+export type HLLRegisterRepresentation = 'sparse' | 'dense';
 export type SchemaCardinality = Prop<{
     type: 'cardinality';
     maxBytes?: number;
-    mime?: Mime;
-    format?: NumberDisplay;
+    precision?: number;
+    mode?: HLLRegisterRepresentation;
 }>;
 type VectorDefaultType = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array;
 export type SchemaVectorBaseType = NumberType | 'float32' | 'float64';

package/dist/types.js

@@ -14,6 +14,7 @@ export const dateDisplays = [
     'date-time',
     'date-time-text',
     'date-time-human',
+    'date-time-human-short',
     'time',
     'time-precise',
 ];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/schema",
-  "version": "5.0.2",
+  "version": "5.0.4-alpha.1",
   "files": [
     "dist",
     "README.md",