@based/db 0.0.15 → 0.0.17

package/dist/src/server/index.js

@@ -1,12 +1,12 @@
 import native from '../native.js';
 import { rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
-import { getPropType } from '@based/schema';
+import { getPropType, langCodesMap, } from '@based/schema';
 import { genRootId } from './schema/utils.js';
 import { updateTypeDefs } from './schema/typeDef.js';
 import { schemaToSelvaBuffer } from './schema/selvaBuffer.js';
 import { start } from './start.js';
-import { foreachDirtyBlock, makeCsmtKey } from './tree.js';
+import { foreachDirtyBlock, makeCsmtKey, makeCsmtKeyFromNodeId, } from './tree.js';
 import { save } from './save.js';
 import { Worker, MessageChannel } from 'node:worker_threads';
 import { fileURLToPath } from 'node:url';
@@ -112,14 +112,16 @@ export class DbServer {
                 for (const type in this.sortIndexes) {
                     for (const field in this.sortIndexes[type]) {
                         for (const start in this.sortIndexes[type][field]) {
-                            const sortIndex = this.sortIndexes[type][field][start];
-                            sortIndex.cnt /= 2;
-                            if (sortIndex.cnt < 1) {
-                                native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
-                                delete this.sortIndexes[type][field][start];
-                            }
-                            else {
-                                remaining = true;
+                            for (const lang in this.sortIndexes[type][field][start]) {
+                                const sortIndex = this.sortIndexes[type][field][start][lang];
+                                sortIndex.cnt /= 2;
+                                if (sortIndex.cnt < 1) {
+                                    native.destroySortIndex(sortIndex.buf, this.dbCtxExternal);
+                                    delete this.sortIndexes[type][field][start][lang];
+                                }
+                                else {
+                                    remaining = true;
+                                }
                             }
                         }
                     }
@@ -130,33 +132,41 @@ export class DbServer {
             }, 60e3);
         }
     }
-    createSortIndex(type, field) {
+    createSortIndex(type, field, lang = 'none') {
         const t = this.schemaTypesParsed[type];
         const prop = t.props[field];
+        const langCode = langCodesMap.get(lang ?? Object.keys(this.schema?.locales ?? 'en')[0]) ??
+            0;
         let types = this.sortIndexes[t.id];
         if (!types) {
             types = this.sortIndexes[t.id] = {};
         }
-        let fields = types[prop.prop];
+        let f = types[prop.prop];
+        if (!f) {
+            f = types[prop.prop] = {};
+        }
+        let fields = f[prop.start];
         if (!fields) {
-            fields = types[prop.prop] = {};
+            fields = f[prop.start] = {};
         }
-        let sortIndex = fields[prop.start];
+        let sortIndex = fields[langCode];
         if (sortIndex) {
             return sortIndex;
         }
         const buf = Buffer.allocUnsafe(8);
-        // size [2 type] [1 field] [2 start] [2 len]
+        // size [2 type] [1 field]  [2 start] [2 len] [propIndex] [lang]
+        // call createSortBuf here
         buf.writeUint16LE(t.id, 0);
         buf[2] = prop.prop;
         buf.writeUint16LE(prop.start, 3);
         buf.writeUint16LE(prop.len, 5);
         buf[7] = prop.typeIndex;
+        buf[8] = langCode;
         sortIndex = new SortIndex(buf, this.dbCtxExternal);
-        fields[prop.start] = sortIndex;
+        fields[langCode] = sortIndex;
         return sortIndex;
     }
-    destroySortIndex(type, field) {
+    destroySortIndex(type, field, lang = 'none') {
         const t = this.schemaTypesParsed[type];
         const prop = t.props[field];
         let types = this.sortIndexes[t.id];
@@ -169,30 +179,37 @@ export class DbServer {
         }
         let sortIndex = fields[prop.start];
         if (sortIndex) {
-            const buf = Buffer.allocUnsafe(5);
+            // [2 type] [1 field] [2 start] [1 lang]
+            const buf = Buffer.allocUnsafe(6);
             buf.writeUint16LE(t.id, 0);
             buf[2] = prop.prop;
             buf.writeUint16LE(prop.start, 3);
+            buf[5] =
+                langCodesMap.get(lang ?? Object.keys(this.schema?.locales ?? 'en')[0]) ?? 0;
             native.destroySortIndex(buf, this.dbCtxExternal);
             delete fields[prop.start];
         }
     }
-    getSortIndex(typeId, field, start) {
+    getSortIndex(typeId, field, start, lang) {
         let types = this.sortIndexes[typeId];
         if (!types) {
             types = this.sortIndexes[typeId] = {};
         }
-        let fields = types[field];
+        let f = types[field];
+        if (!f) {
+            f = types[field] = {};
+        }
+        let fields = f[start];
         if (!fields) {
-            fields = types[field] = {};
+            fields = f[start] = {};
         }
-        return fields[start];
+        return fields[lang];
     }
     migrateSchema(schema, transform) {
         return migrate(this, schema, transform);
     }
-    createSortIndexBuffer(typeId, field, start) {
-        const buf = Buffer.allocUnsafe(8);
+    createSortIndexBuffer(typeId, field, start, lang) {
+        const buf = Buffer.allocUnsafe(9);
         buf.writeUint16LE(typeId, 0);
         buf[2] = field;
         buf.writeUint16LE(start, 3);
@@ -219,11 +236,13 @@ export class DbServer {
         }
         buf.writeUint16LE(prop.len, 5);
         buf[7] = prop.typeIndex;
+        buf[8] = lang;
         // put in modify stuff
-        const sortIndex = new SortIndex(buf, this.dbCtxExternal);
+        const sortIndex = this.getSortIndex(typeId, prop.prop, prop.start, lang) ??
+            new SortIndex(buf, this.dbCtxExternal);
         const types = this.sortIndexes[typeId];
         const fields = types[field];
-        fields[start] = sortIndex;
+        fields[start][lang] = sortIndex;
         return sortIndex;
     }
     updateMerkleTree() {
@@ -360,11 +379,18 @@ export class DbServer {
     #modify(buf) {
         const end = buf.length - 4;
         const dataLen = buf.readUint32LE(end);
-        const typesSize = buf.readUint16LE(dataLen);
+        let typesSize = buf.readUint16LE(dataLen);
         const typesLen = typesSize * 10;
         const types = buf.subarray(dataLen + 2, dataLen + typesLen + 2);
         const data = buf.subarray(0, dataLen);
-        let i = dataLen + 2 + typesLen;
+        let i = dataLen + 2;
+        while (typesSize--) {
+            const typeId = buf.readUint16LE(i);
+            const def = this.schemaTypesParsedById[typeId];
+            const key = makeCsmtKeyFromNodeId(def.id, def.blockCapacity, def.lastId);
+            this.dirtyRanges.add(key);
+            i += 10;
+        }
         while (i < end) {
             const key = buf.readDoubleLE(i);
             this.dirtyRanges.add(key);
@@ -388,14 +414,14 @@ export class DbServer {
                     const sort = buf.slice(s + 2, s + 2 + sortLen);
                     const field = sort[1];
                     const start = sort.readUint16LE(2 + 1);
-                    let sortIndex = this.getSortIndex(typeId, field, start);
+                    let sortIndex = this.getSortIndex(typeId, field, start, 0);
                     if (!sortIndex) {
                         if (this.processingQueries) {
                             return new Promise((resolve) => {
                                 this.queryQueue.set(resolve, buf);
                             });
                         }
-                        sortIndex = this.createSortIndexBuffer(typeId, field, start);
+                        sortIndex = this.createSortIndexBuffer(typeId, field, start, sort[sort.byteLength - 1]);
                     }
                     // increment
                     sortIndex.cnt++;

package/dist/src/server/migrate/index.js

@@ -1,7 +1,7 @@
 import { BasedDb } from '../../index.js';
 import { dirname, join } from 'path';
 import { tmpdir } from 'os';
-import { Worker, MessageChannel } from 'node:worker_threads';
+import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
 import native from '../../native.js';
 import './worker.js';
 import { foreachDirtyBlock } from '../tree.js';
@@ -99,7 +99,15 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
         }
     }
     if (!abort()) {
-        fromDbServer.putSchema(toSchema, true);
+        let msg;
+        let schema;
+        let schemaTypesParsed;
+        while ((msg = receiveMessageOnPort(port1))) {
+            ;
+            [schema, schemaTypesParsed] = msg.message;
+        }
+        fromDbServer.schema = schema;
+        fromDbServer.schemaTypesParsed = schemaTypesParsed;
         fromDbServer.dbCtxExternal = toCtx;
         toDb.server.dbCtxExternal = fromCtx;
     }

package/dist/src/server/migrate/worker.js

@@ -71,7 +71,7 @@ else {
             }
         }
         await toDb.drain();
-        // await setTimeout()
+        channel.postMessage([toDb.server.schema, toDb.server.schemaTypesParsed]);
         // put it to sleep
         atomics[0] = 0;
         Atomics.notify(atomics, 0);

package/dist/src/server/schema/typeDef.js

@@ -1,7 +1,7 @@
 import { isPropType, getPropType, } from '@based/schema';
 import { setByPath } from '@saulx/utils';
 import { hashObjectIgnoreKeyOrder } from '@saulx/hash';
-import { SIZE_MAP, TYPE_INDEX_MAP, } from './types.js';
+import { SIZE_MAP, TYPE_INDEX_MAP, STRING, ALIAS, CARDINALITY, } from './types.js';
 import { genId } from './utils.js';
 import { DEFAULT_BLOCK_CAPACITY } from '../start.js';
 const addEdges = (prop, refProp) => {
@@ -95,9 +95,21 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = {
     tree: {},
     total: 0,
     lastId: 0,
-    stringPropsSize: 0,
-    stringPropsLoop: [],
     main: {},
+    hasSeperateSort: false,
+    seperateSort: {
+        size: 0,
+        props: [],
+        buffer: Buffer.allocUnsafe(0),
+        bufferTmp: Buffer.allocUnsafe(0),
+    },
+    hasSeperateTextSort: false,
+    seperateTextSort: {
+        size: 0,
+        props: [],
+        buffer: Buffer.allocUnsafe(0),
+        bufferTmp: Buffer.allocUnsafe(0),
+    },
 }, path = [], top = true) => {
     if (result.id == 0 && top) {
         if ('id' in type) {
@@ -112,7 +124,8 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = {
     result.idUint8[1] = result.id >> 8;
     const encoder = new TextEncoder();
     const target = type.props;
-    let stringFields = 0;
+    let separateSortProps = 0;
+    let separateSortText = 0;
     for (const key in target) {
         const schemaProp = target[key];
         const propPath = [...path, key];
@@ -124,9 +137,7 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = {
             let len = SIZE_MAP[propType];
             if (isPropType('string', schemaProp) ||
                 isPropType('alias', schemaProp) ||
-                isPropType('binary', schemaProp) ||
-                isPropType('cardinality', schemaProp) // TODO: maieutica
-            ) {
+                isPropType('cardinality', schemaProp)) {
                 if (typeof schemaProp === 'object') {
                     if (schemaProp.maxBytes < 61) {
                         len = schemaProp.maxBytes + 1;
@@ -135,18 +146,13 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = {
                         len = schemaProp.max * 2 + 1;
                     }
                     else {
-                        stringFields++;
+                        separateSortProps++;
                     }
                 }
                 else {
-                    stringFields++;
+                    separateSortProps++;
                 }
             }
-            else if (isPropType('text', schemaProp) ||
-                isPropType('cardinality', schemaProp)) {
-                // TODO: maieutica
-                stringFields++;
-            }
             else if (isPropType('vector', schemaProp)) {
                 len = 4 * schemaProp.size;
             }
@@ -290,27 +296,31 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = {
             result.propNames.set(f, lastWritten + 1);
             lastWritten += f.byteLength + 1;
         }
-        // result.responseCtx = new BasedNode(result as SchemaTypeDef, parsed)
-        if (stringFields > 0) {
-            result.hasStringProp = true;
+        if (separateSortProps > 0) {
+            result.hasSeperateSort = true;
             let max = 0;
             for (const f of result.separate) {
-                if (f.typeIndex === 11) {
+                // Does not only need to be on string can also be HLL
+                if (f.typeIndex === STRING ||
+                    f.typeIndex === ALIAS ||
+                    f.typeIndex === CARDINALITY) {
                     if (f.prop > max) {
                         max = f.prop;
                     }
                 }
             }
-            result.stringProps = Buffer.allocUnsafe(max + 1);
+            result.seperateSort.buffer = Buffer.allocUnsafe(max + 1);
             for (const f of result.separate) {
-                if (f.typeIndex === 11) {
-                    result.stringProps[f.prop] = 1;
-                    result.stringPropsLoop.push(f);
-                    result.stringPropsSize++;
+                if (f.typeIndex === STRING ||
+                    f.typeIndex === ALIAS ||
+                    f.typeIndex === CARDINALITY) {
+                    result.seperateSort.buffer[f.prop] = 1;
+                    result.seperateSort.props.push(f);
+                    result.seperateSort.size++;
                 }
             }
-            result.stringPropsCurrent = Buffer.allocUnsafe(max + 1);
-            result.stringProps.copy(result.stringPropsCurrent);
+            result.seperateSort.bufferTmp = Buffer.allocUnsafe(max + 1);
+            result.seperateSort.buffer.copy(result.seperateSort.bufferTmp);
         }
         for (const p in result.props) {
             const x = result.props[p];

package/dist/src/server/schema/types.d.ts

@@ -94,6 +94,12 @@ export type PropDefEdge = Partial<PropDef> & {
 export type SchemaPropTree = {
     [key: string]: SchemaPropTree | PropDef;
 };
+export type SchemaSortUndefinedHandler = {
+    size: number;
+    buffer: Buffer;
+    bufferTmp: Buffer;
+    props: PropDef[];
+};
 export type SchemaTypeDef = {
     cnt: number;
     checksum: number;
@@ -117,11 +123,10 @@ export type SchemaTypeDef = {
         [start: string]: PropDef;
     };
     tree: SchemaPropTree;
-    hasStringProp: boolean;
-    stringPropsSize: number;
-    stringProps: Buffer;
-    stringPropsCurrent: Buffer;
-    stringPropsLoop: PropDef[];
+    hasSeperateSort: boolean;
+    seperateSort: SchemaSortUndefinedHandler;
+    hasSeperateTextSort: boolean;
+    seperateTextSort: SchemaSortUndefinedHandler;
     createTs?: PropDef[];
     updateTs?: PropDef[];
     locales: Partial<SchemaLocales>;

package/dist/src/server/schema/types.js

@@ -4,7 +4,7 @@ export const TIMESTAMP = 1;
 export const CREATED = 2;
 export const UPDATED = 3;
 export const NUMBER = 4;
-export const CARDINALITY = 5; //se colocar 11 ele faz o console de Ao menos entra aqui?
+export const CARDINALITY = 5;
 export const INT8 = 20;
 export const UINT8 = 6;
 export const INT16 = 21;

package/dist/src/server/schema/utils.d.ts

@@ -2,5 +2,6 @@ import { DbClient } from '../../client/index.js';
 import { DbServer } from '../index.js';
 import { PropDef, PropDefEdge } from './types.js';
 export declare const propIsSigned: (prop: PropDef | PropDefEdge) => boolean;
+export declare const propIsNumerical: (prop: PropDef | PropDefEdge) => boolean;
 export declare const genRootId: () => number;
 export declare const genId: (db: DbServer | DbClient) => number;

package/dist/src/server/schema/utils.js

@@ -1,9 +1,22 @@
-import { INT16, INT32, INT64, INT8 } from './types.js';
+import { INT16, INT32, INT64, INT8, UINT16, UINT32, UINT8, NUMBER, TIMESTAMP, } from './types.js';
 export const propIsSigned = (prop) => {
-    if (prop.typeIndex === INT16 ||
-        prop.typeIndex === INT32 ||
-        prop.typeIndex === INT64 ||
-        prop.typeIndex === INT8) {
+    const t = prop.typeIndex;
+    if (t === INT16 || t === INT32 || t === INT64 || t === INT8) {
+        return true;
+    }
+    return false;
+};
+export const propIsNumerical = (prop) => {
+    const t = prop.typeIndex;
+    if (t === INT16 ||
+        t === INT32 ||
+        t === INT64 ||
+        t === INT8 ||
+        t === UINT8 ||
+        t === UINT16 ||
+        t === UINT32 ||
+        t === NUMBER ||
+        t === TIMESTAMP) {
         return true;
     }
     return false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/db",
-  "version": "0.0.15",
+  "version": "0.0.17",
   "license": "MIT",
   "type": "module",
   "main": "./dist/src/index.js",
@@ -17,7 +17,7 @@
     "test": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale lldb -b -o run --one-line-on-crash bt -o \"script import os; os._exit(0)\" node ./scripts/test.js",
     "test-gdb": "npm run build && LOCPATH=../locale/locale-x86_64-gnu/locale gdb -ex run --args node ./scripts/test.js",
     "test-fast": "LOCPATH=../locale/locale-x86_64-gnu/locale node ./scripts/test.js",
-    "test-zig": "npm run build-zig && tsc && npm run test-fast",
+    "test-zig": "npm run build-zig -- debug && tsc && npm run test-fast",
     "test-ts": "tsc && node ./scripts/test.js",
     "perf": "npm run build && node benchmarks/references.js && node benchmarks/transfermarkt/transfermarkt-based.js"
   },
@@ -35,12 +35,14 @@
     "@saulx/hash": "^3.0.0",
     "@saulx/utils": "^4.3.2",
     "exit-hook": "^4.0.0",
-    "picocolors": "^1.1.0"
+    "picocolors": "^1.1.0",
+    "@based/crc32c": "^1.0.0"
   },
   "optionalDependencies": {
     "@based/locale-x86-64-gnu": "*"
   },
   "devDependencies": {
+    "@based/crc32c": "^1.0.0",
     "@types/node": "^22.5.3",
     "axios": "^1.7.9",
     "rimraf": "^3.0.2",