@based/db 0.0.66 → 0.0.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/lib/darwin_aarch64/include/selva/colvec.h +71 -0
- package/dist/lib/darwin_aarch64/include/selva/db.h +3 -0
- package/dist/lib/darwin_aarch64/include/selva/fields.h +1 -1
- package/dist/lib/darwin_aarch64/include/selva/types.h +7 -0
- package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
- package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
- package/dist/lib/darwin_aarch64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
- package/dist/lib/linux_aarch64/include/selva/colvec.h +71 -0
- package/dist/lib/linux_aarch64/include/selva/db.h +3 -0
- package/dist/lib/linux_aarch64/include/selva/fields.h +1 -1
- package/dist/lib/linux_aarch64/include/selva/types.h +7 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/selva/colvec.h +71 -0
- package/dist/lib/linux_x86_64/include/selva/db.h +3 -0
- package/dist/lib/linux_x86_64/include/selva/fields.h +1 -1
- package/dist/lib/linux_x86_64/include/selva/types.h +7 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/flushModify.d.ts +2 -1
- package/dist/src/client/flushModify.js +12 -4
- package/dist/src/client/modify/create.js +11 -0
- package/dist/src/client/modify/delete.js +3 -0
- package/dist/src/client/modify/modify.js +2 -2
- package/dist/src/client/modify/setCursor.d.ts +2 -1
- package/dist/src/client/query/BasedDbQuery.d.ts +7 -2
- package/dist/src/client/query/BasedDbQuery.js +111 -14
- package/dist/src/client/query/aggregates/aggregation.js +24 -11
- package/dist/src/client/query/aggregates/types.d.ts +21 -2
- package/dist/src/client/query/aggregates/types.js +34 -1
- package/dist/src/client/query/display.js +8 -2
- package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -3
- package/dist/src/client/query/filter/createVariableFilterBuffer.js +20 -7
- package/dist/src/client/query/filter/filter.js +13 -3
- package/dist/src/client/query/filter/primitiveFilter.d.ts +1 -2
- package/dist/src/client/query/include/props.js +18 -2
- package/dist/src/client/query/include/toBuffer.js +11 -3
- package/dist/src/client/query/include/walk.js +5 -1
- package/dist/src/client/query/queryDef.js +4 -1
- package/dist/src/client/query/read/read.js +50 -23
- package/dist/src/client/query/registerQuery.js +1 -0
- package/dist/src/client/query/search/index.d.ts +1 -1
- package/dist/src/client/query/search/index.js +21 -7
- package/dist/src/client/query/sort.d.ts +1 -1
- package/dist/src/client/query/toByteCode/default.d.ts +1 -1
- package/dist/src/client/query/toByteCode/default.js +0 -2
- package/dist/src/client/query/toByteCode/toBuffer.js +0 -7
- package/dist/src/client/query/types.d.ts +16 -5
- package/dist/src/client/query/validation.js +25 -2
- package/dist/src/client/xxHash64.d.ts +1 -1
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/dist/src/native.d.ts +2 -2
- package/dist/src/native.js +7 -8
- package/dist/src/server/IoWorker.d.ts +8 -0
- package/dist/src/server/IoWorker.js +39 -0
- package/dist/src/server/QueryWorker.d.ts +8 -0
- package/dist/src/server/QueryWorker.js +26 -0
- package/dist/src/server/blocks.d.ts +24 -0
- package/dist/src/server/blocks.js +112 -0
- package/dist/src/server/dbHash.d.ts +1 -1
- package/dist/src/server/index.d.ts +9 -16
- package/dist/src/server/index.js +37 -15
- package/dist/src/server/migrate/index.d.ts +5 -0
- package/dist/src/server/migrate/index.js +10 -7
- package/dist/src/server/save.d.ts +8 -6
- package/dist/src/server/save.js +34 -78
- package/dist/src/server/schema.js +6 -5
- package/dist/src/server/start.js +57 -60
- package/dist/src/server/tree.d.ts +24 -13
- package/dist/src/server/tree.js +95 -66
- package/dist/src/server/workers/DbWorker.d.ts +17 -0
- package/dist/src/server/{DbWorker.js → workers/DbWorker.js} +15 -17
- package/dist/src/server/workers/io_worker.js +39 -0
- package/dist/src/server/workers/io_worker_types.d.ts +12 -0
- package/dist/src/server/workers/io_worker_types.js +2 -0
- package/dist/src/server/workers/query_worker.d.ts +1 -0
- package/dist/src/server/workers/query_worker.js +4 -0
- package/dist/src/server/workers/worker.d.ts +1 -0
- package/dist/src/server/workers/worker.js +41 -0
- package/dist/src/shared/Emitter.d.ts +1 -0
- package/dist/src/types.d.ts +1 -1
- package/dist/src/types.js +1 -1
- package/package.json +3 -3
- package/dist/lib/darwin_aarch64/include/selva/history.h +0 -64
- package/dist/lib/linux_aarch64/include/selva/history.h +0 -64
- package/dist/lib/linux_x86_64/include/selva/history.h +0 -64
- package/dist/src/client/query/serialize.d.ts +0 -4
- package/dist/src/client/query/serialize.js +0 -26
- package/dist/src/server/DbWorker.d.ts +0 -13
- package/dist/src/server/csmt/draw-dot.d.ts +0 -4
- package/dist/src/server/csmt/draw-dot.js +0 -38
- package/dist/src/server/csmt/index.d.ts +0 -4
- package/dist/src/server/csmt/index.js +0 -5
- package/dist/src/server/csmt/match.d.ts +0 -7
- package/dist/src/server/csmt/match.js +0 -10
- package/dist/src/server/csmt/memebership-proof.d.ts +0 -7
- package/dist/src/server/csmt/memebership-proof.js +0 -122
- package/dist/src/server/csmt/tree-utils.d.ts +0 -6
- package/dist/src/server/csmt/tree-utils.js +0 -33
- package/dist/src/server/csmt/tree.d.ts +0 -3
- package/dist/src/server/csmt/tree.js +0 -270
- package/dist/src/server/csmt/types.d.ts +0 -46
- package/dist/src/server/csmt/types.js +0 -2
- package/dist/src/server/worker.js +0 -33
- /package/dist/src/server/{worker.d.ts → workers/io_worker.d.ts} +0 -0
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { ALIAS, ALIASES, BINARY, BOOLEAN, ENUM, INT16, INT32, INT8, NUMBER, STRING, TEXT, TIMESTAMP, UINT16, UINT32, UINT8, VECTOR, JSON, CARDINALITY, } from '@based/schema/def';
|
|
1
|
+
import { ALIAS, ALIASES, BINARY, BOOLEAN, ENUM, INT16, INT32, INT8, NUMBER, STRING, TEXT, TIMESTAMP, UINT16, UINT32, UINT8, VECTOR, JSON, CARDINALITY, COLVEC, } from '@based/schema/def';
|
|
2
2
|
import { QueryDefType } from '../types.js';
|
|
3
3
|
import { read, readUtf8 } from '../../string.js';
|
|
4
4
|
import { DECODER, readDoubleLE, readFloatLE, readInt16, readInt32, readUint16, readUint32, setByPath, } from '@saulx/utils';
|
|
5
5
|
import { inverseLangMap } from '@based/schema';
|
|
6
6
|
import { READ_EDGE, READ_ID, READ_REFERENCE, READ_REFERENCES, READ_AGGREGATION, } from '../types.js';
|
|
7
|
+
import { AggregateType } from '../aggregates/types.js';
|
|
7
8
|
const readAggregate = (q, result, offset, len) => {
|
|
8
9
|
const results = {};
|
|
9
10
|
if (q.aggregate.groupBy) {
|
|
@@ -21,24 +22,47 @@ const readAggregate = (q, result, offset, len) => {
|
|
|
21
22
|
i += 2;
|
|
22
23
|
}
|
|
23
24
|
else {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
if (q.aggregate.groupBy.typeIndex == ENUM) {
|
|
26
|
+
i += 2;
|
|
27
|
+
key = q.aggregate.groupBy.enum[result[i] - 1];
|
|
28
|
+
i++;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
keyLen = readUint16(result, i);
|
|
32
|
+
i += 2;
|
|
33
|
+
key = DECODER.decode(result.subarray(i, i + keyLen));
|
|
34
|
+
i += keyLen;
|
|
35
|
+
}
|
|
28
36
|
}
|
|
29
37
|
const resultKey = (results[key] = {});
|
|
30
38
|
for (const aggregatesArray of q.aggregate.aggregates.values()) {
|
|
31
39
|
for (const agg of aggregatesArray) {
|
|
32
|
-
|
|
40
|
+
var val = undefined;
|
|
41
|
+
if (agg.type === AggregateType.CARDINALITY ||
|
|
42
|
+
agg.type === AggregateType.COUNT) {
|
|
43
|
+
val = readUint32(result, agg.resultPos + i);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
val = readDoubleLE(result, agg.resultPos + i);
|
|
47
|
+
}
|
|
48
|
+
setByPath(resultKey, agg.propDef.path, val);
|
|
33
49
|
}
|
|
34
50
|
}
|
|
35
|
-
i += q.aggregate.
|
|
51
|
+
i += q.aggregate.totalResultsSize;
|
|
36
52
|
}
|
|
37
53
|
}
|
|
38
54
|
else {
|
|
39
55
|
for (const aggregatesArray of q.aggregate.aggregates.values()) {
|
|
40
56
|
for (const agg of aggregatesArray) {
|
|
41
|
-
|
|
57
|
+
var val = undefined;
|
|
58
|
+
if (agg.type === AggregateType.CARDINALITY ||
|
|
59
|
+
agg.type === AggregateType.COUNT) {
|
|
60
|
+
val = readUint32(result, agg.resultPos + offset);
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
val = readDoubleLE(result, agg.resultPos + offset);
|
|
64
|
+
}
|
|
65
|
+
setByPath(results, agg.propDef.path, val);
|
|
42
66
|
}
|
|
43
67
|
}
|
|
44
68
|
}
|
|
@@ -172,13 +196,13 @@ const handleUndefinedProps = (id, q, item) => {
|
|
|
172
196
|
if (prop.typeIndex === CARDINALITY) {
|
|
173
197
|
addField(prop, 0, item);
|
|
174
198
|
}
|
|
175
|
-
else if (prop.typeIndex === TEXT && q.lang == 0) {
|
|
199
|
+
else if (prop.typeIndex === TEXT && q.lang.lang == 0) {
|
|
176
200
|
const lan = getEmptyField(prop, item);
|
|
177
201
|
const lang = q.include.langTextFields.get(prop.prop).codes;
|
|
178
202
|
if (lang.has(0)) {
|
|
179
203
|
for (const locale in q.schema.locales) {
|
|
180
204
|
if (lan[locale] == undefined) {
|
|
181
|
-
lan[locale] =
|
|
205
|
+
lan[locale] = '';
|
|
182
206
|
}
|
|
183
207
|
}
|
|
184
208
|
}
|
|
@@ -186,23 +210,26 @@ const handleUndefinedProps = (id, q, item) => {
|
|
|
186
210
|
for (const code of lang) {
|
|
187
211
|
const locale = inverseLangMap.get(code);
|
|
188
212
|
if (!lan[locale]) {
|
|
189
|
-
lan[locale] =
|
|
213
|
+
lan[locale] = '';
|
|
190
214
|
}
|
|
191
215
|
}
|
|
192
216
|
}
|
|
193
217
|
}
|
|
194
|
-
else if (prop.typeIndex === BINARY) {
|
|
195
|
-
|
|
196
|
-
}
|
|
197
|
-
else if (prop.typeIndex === TEXT
|
|
218
|
+
// else if (prop.typeIndex === BINARY) {
|
|
219
|
+
// addField(prop, prop.default, item)
|
|
220
|
+
// }
|
|
221
|
+
else if (prop.typeIndex === TEXT ||
|
|
222
|
+
prop.typeIndex === STRING ||
|
|
223
|
+
prop.typeIndex === ALIAS) {
|
|
198
224
|
addField(prop, '', item);
|
|
199
225
|
}
|
|
200
|
-
else {
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
}
|
|
226
|
+
// else if (prop.typeIndex === JSON) {
|
|
227
|
+
// // addField(prop, null, item)
|
|
228
|
+
// } else {
|
|
229
|
+
// // if (prop.default !== undefined) {
|
|
230
|
+
// // addField(prop, '', item)
|
|
231
|
+
// // }
|
|
232
|
+
// }
|
|
206
233
|
}
|
|
207
234
|
}
|
|
208
235
|
};
|
|
@@ -380,7 +407,7 @@ export const readAllFields = (q, result, offset, end, item, id) => {
|
|
|
380
407
|
// do nothing
|
|
381
408
|
}
|
|
382
409
|
else {
|
|
383
|
-
if (q.lang != 0) {
|
|
410
|
+
if (q.lang.lang != 0) {
|
|
384
411
|
q.include.propsRead[index] = id;
|
|
385
412
|
addField(prop, read(result, i + 4, size, true), item);
|
|
386
413
|
}
|
|
@@ -403,7 +430,7 @@ export const readAllFields = (q, result, offset, end, item, id) => {
|
|
|
403
430
|
addField(prop, string, item);
|
|
404
431
|
}
|
|
405
432
|
}
|
|
406
|
-
else if (prop.typeIndex == VECTOR) {
|
|
433
|
+
else if (prop.typeIndex == VECTOR || prop.typeIndex == COLVEC) {
|
|
407
434
|
q.include.propsRead[index] = id;
|
|
408
435
|
const size = readUint32(result, i);
|
|
409
436
|
const arr = new Float32Array(size / 4);
|
|
@@ -5,4 +5,4 @@ export type Search = string[] | {
|
|
|
5
5
|
} | string;
|
|
6
6
|
export declare const vectorSearch: (def: QueryDef, q: ArrayBufferView, field: string, opts: Omit<FilterOpts, "lowerCase">) => void;
|
|
7
7
|
export declare const search: (def: QueryDef, q: string, s?: Search) => void;
|
|
8
|
-
export declare const searchToBuffer: (search: QueryDefSearch) => Uint8Array
|
|
8
|
+
export declare const searchToBuffer: (search: QueryDefSearch) => Uint8Array<ArrayBuffer>;
|
|
@@ -74,14 +74,16 @@ export const search = (def, q, s) => {
|
|
|
74
74
|
}
|
|
75
75
|
for (const key in s) {
|
|
76
76
|
let prop = def.props[key];
|
|
77
|
-
let lang = def.lang;
|
|
77
|
+
let lang = def.lang.lang;
|
|
78
|
+
let fallback = def.lang.fallback;
|
|
78
79
|
if (!prop) {
|
|
79
80
|
if (key.includes('.')) {
|
|
80
81
|
const k = key.split('.');
|
|
81
82
|
prop = def.props[k.slice(0, -1).join('.')];
|
|
82
83
|
if (prop && prop.typeIndex === TEXT) {
|
|
83
84
|
lang = langCodesMap.get(k[k.length - 1]);
|
|
84
|
-
|
|
85
|
+
fallback = [];
|
|
86
|
+
// handle incorrect LANG
|
|
85
87
|
}
|
|
86
88
|
else {
|
|
87
89
|
prop = searchDoesNotExist(def, key, false);
|
|
@@ -94,11 +96,16 @@ export const search = (def, q, s) => {
|
|
|
94
96
|
if (prop.typeIndex !== STRING && prop.typeIndex !== TEXT) {
|
|
95
97
|
searchIncorrectType(def, prop);
|
|
96
98
|
}
|
|
97
|
-
def.search.size +=
|
|
99
|
+
def.search.size += 10;
|
|
100
|
+
if (fallback.length > 3) {
|
|
101
|
+
// Max 4 lang fallback size (tmp)
|
|
102
|
+
console.warn('Search has a max of 4 lang fallbacks!');
|
|
103
|
+
fallback = fallback.slice(0, 3);
|
|
104
|
+
}
|
|
98
105
|
def.search.fields.push({
|
|
99
106
|
typeIndex: prop.typeIndex,
|
|
100
107
|
weight: s[key],
|
|
101
|
-
lang,
|
|
108
|
+
lang: { lang, fallback },
|
|
102
109
|
field: prop.prop,
|
|
103
110
|
start: prop.start ?? 0, // also need lang ofc if you have start
|
|
104
111
|
});
|
|
@@ -147,6 +154,8 @@ export const searchToBuffer = (search) => {
|
|
|
147
154
|
| 2 | weight | 1 | Field weight value |
|
|
148
155
|
| 3 | start | 2 | Start position in the query (u16) |
|
|
149
156
|
| 5 | lang | 1 | Language identifier |
|
|
157
|
+
| 6 | fallbackL | 1 | Language fallback size |
|
|
158
|
+
| 7 | fallback | 3 | Language fallback |
|
|
150
159
|
|
|
151
160
|
### Notes:
|
|
152
161
|
- The number of field entries is inferred from the total packet size.
|
|
@@ -162,15 +171,20 @@ export const searchToBuffer = (search) => {
|
|
|
162
171
|
return a.weight - b.weight;
|
|
163
172
|
});
|
|
164
173
|
// @ts-ignore
|
|
165
|
-
for (let i = 0; i < search.fields.length *
|
|
174
|
+
for (let i = 0; i < search.fields.length * 10; i += 10) {
|
|
166
175
|
// @ts-ignore
|
|
167
|
-
const f = search.fields[Math.floor(i /
|
|
176
|
+
const f = search.fields[Math.floor(i / 10)];
|
|
168
177
|
result[i + offset] = f.field;
|
|
169
178
|
result[i + offset + 1] = f.typeIndex;
|
|
170
179
|
result[i + offset + 2] = f.weight;
|
|
171
180
|
result[i + offset + 3] = f.start;
|
|
172
181
|
result[i + offset + 4] = f.start >>> 8;
|
|
173
|
-
|
|
182
|
+
// fallback
|
|
183
|
+
result[i + offset + 5] = f.lang.lang;
|
|
184
|
+
result[i + offset + 6] = f.lang.fallback.length;
|
|
185
|
+
for (let j = 0; j < f.lang.fallback.length; j++) {
|
|
186
|
+
result[i + j + offset + 7] = f.lang.fallback[j];
|
|
187
|
+
}
|
|
174
188
|
}
|
|
175
189
|
return result;
|
|
176
190
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { QueryDef, QueryDefSort } from './types.js';
|
|
2
|
-
export declare const createSortBuffer: (sort: QueryDefSort) => Uint8Array
|
|
2
|
+
export declare const createSortBuffer: (sort: QueryDefSort) => Uint8Array<ArrayBuffer>;
|
|
3
3
|
export declare const sort: (def: QueryDef, field: string, order?: "asc" | "desc") => void;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
import { QueryDef } from '../types.js';
|
|
2
|
-
export declare const defaultQuery: (def: QueryDef, filterSize: number, sortSize: number, searchSize: number, sort: Uint8Array, search: Uint8Array) => Uint8Array
|
|
2
|
+
export declare const defaultQuery: (def: QueryDef, filterSize: number, sortSize: number, searchSize: number, sort: Uint8Array, search: Uint8Array) => Uint8Array<ArrayBuffer>;
|
|
@@ -17,8 +17,6 @@ export const defaultQuery = (def, filterSize, sortSize, searchSize, sort, search
|
|
|
17
17
|
buf[index++] = filterSize;
|
|
18
18
|
buf[index++] = filterSize >>> 8;
|
|
19
19
|
buf[index++] = filterSize && isSimpleMainFilter(def.filter) ? 1 : 0;
|
|
20
|
-
// if (filterSize && isSimpleMainFilter(def.filter)) {
|
|
21
|
-
// console.log('SIMPLE FILTER!')
|
|
22
20
|
if (filterSize) {
|
|
23
21
|
buf.set(filterToBuffer(def.filter), index);
|
|
24
22
|
index += filterSize;
|
|
@@ -14,7 +14,6 @@ const byteSize = (arr) => {
|
|
|
14
14
|
export function defToBuffer(db, def) {
|
|
15
15
|
const result = [];
|
|
16
16
|
const include = includeToBuffer(db, def);
|
|
17
|
-
// ---------------------------------------
|
|
18
17
|
def.references.forEach((ref) => {
|
|
19
18
|
include.push(...defToBuffer(db, ref));
|
|
20
19
|
if (ref.errors) {
|
|
@@ -46,7 +45,6 @@ export function defToBuffer(db, def) {
|
|
|
46
45
|
buf[0] = 251 /* includeOp.REFERENCES_AGGREGATION */;
|
|
47
46
|
buf[1] = sz;
|
|
48
47
|
buf[2] = sz >>> 8;
|
|
49
|
-
// ---
|
|
50
48
|
buf[3] = filterSize;
|
|
51
49
|
buf[4] = filterSize >>> 8;
|
|
52
50
|
buf[5] = def.range.offset;
|
|
@@ -62,8 +60,6 @@ export function defToBuffer(db, def) {
|
|
|
62
60
|
buf[9 + 2 + filterSize] = def.target.propDef.prop; // refField
|
|
63
61
|
const aggregateBuffer = aggregateToBuffer(def.aggregate);
|
|
64
62
|
buf.set(aggregateBuffer, 9 + 3 + filterSize);
|
|
65
|
-
// buf[12 + filterSize] = aggregateSize
|
|
66
|
-
// buf[12 + 1 + filterSize] = aggregateSize >>> 8
|
|
67
63
|
result.push(buf);
|
|
68
64
|
}
|
|
69
65
|
else {
|
|
@@ -92,8 +88,6 @@ export function defToBuffer(db, def) {
|
|
|
92
88
|
buf.set(aggregateBuffer, 16 + filterSize);
|
|
93
89
|
result.push(buf);
|
|
94
90
|
}
|
|
95
|
-
// ignore this for now...
|
|
96
|
-
// result.push(...include)
|
|
97
91
|
if (def.type === QueryDefType.Root) {
|
|
98
92
|
const checksum = new Uint8Array(8);
|
|
99
93
|
writeUint64(checksum, def.schemaChecksum ?? 0, 0);
|
|
@@ -175,7 +169,6 @@ export function defToBuffer(db, def) {
|
|
|
175
169
|
buf[idsSize + 13] = def.range.limit >>> 16;
|
|
176
170
|
buf[idsSize + 14] = def.range.limit >>> 24;
|
|
177
171
|
// if (filterSize && isSimpleMainFilter(def.filter)) {
|
|
178
|
-
// console.log('SIMPLE FILTER!')
|
|
179
172
|
buf[idsSize + 15] = filterSize;
|
|
180
173
|
buf[idsSize + 16] = filterSize >>> 8;
|
|
181
174
|
if (filterSize) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { LangCode } from '@based/schema';
|
|
1
|
+
import { LangCode, LangName } from '@based/schema';
|
|
2
2
|
import { PropDef, PropDefEdge, SchemaTypeDef } from '@based/schema/def';
|
|
3
3
|
import { FilterOpts } from './filter/types.js';
|
|
4
4
|
import { QueryError } from './validation.js';
|
|
@@ -59,7 +59,10 @@ export type QueryDefSearch = {
|
|
|
59
59
|
weight: number;
|
|
60
60
|
field: number;
|
|
61
61
|
start: number;
|
|
62
|
-
lang:
|
|
62
|
+
lang: {
|
|
63
|
+
lang: LangCode;
|
|
64
|
+
fallback: LangCode[];
|
|
65
|
+
};
|
|
63
66
|
typeIndex: number;
|
|
64
67
|
}[];
|
|
65
68
|
} | {
|
|
@@ -78,17 +81,23 @@ export type Aggregation = {
|
|
|
78
81
|
type: AggregateType;
|
|
79
82
|
propDef: PropDef;
|
|
80
83
|
resultPos: number;
|
|
84
|
+
accumulatorPos: number;
|
|
81
85
|
};
|
|
82
86
|
export type QueryDefAggregation = {
|
|
83
87
|
size: number;
|
|
84
88
|
groupBy?: PropDef;
|
|
85
89
|
aggregates: Map<number, Aggregation[]>;
|
|
86
|
-
|
|
90
|
+
totalResultsSize: number;
|
|
91
|
+
totalAccumulatorSize: number;
|
|
87
92
|
};
|
|
93
|
+
export type LangFallback = LangName | false;
|
|
88
94
|
export type QueryDefShared = {
|
|
89
95
|
schemaChecksum?: number;
|
|
90
96
|
errors: QueryError[];
|
|
91
|
-
lang:
|
|
97
|
+
lang: {
|
|
98
|
+
lang: LangCode;
|
|
99
|
+
fallback: LangCode[];
|
|
100
|
+
};
|
|
92
101
|
filter: QueryDefFilter;
|
|
93
102
|
aggregate: null | QueryDefAggregation;
|
|
94
103
|
search: null | QueryDefSearch;
|
|
@@ -99,9 +108,11 @@ export type QueryDefShared = {
|
|
|
99
108
|
limit: number;
|
|
100
109
|
};
|
|
101
110
|
include: {
|
|
102
|
-
langTextFields: Map<number,
|
|
111
|
+
langTextFields: Map<number, // prop name
|
|
112
|
+
{
|
|
103
113
|
def: PropDef | PropDefEdge;
|
|
104
114
|
codes: Set<LangCode>;
|
|
115
|
+
fallBacks: LangCode[];
|
|
105
116
|
}>;
|
|
106
117
|
stringFields: Set<string>;
|
|
107
118
|
props: Map<number, PropDef | PropDefEdge>;
|
|
@@ -139,6 +139,28 @@ export const validateFilter = (def, prop, f) => {
|
|
|
139
139
|
});
|
|
140
140
|
return true;
|
|
141
141
|
}
|
|
142
|
+
// map { id: } format for filter
|
|
143
|
+
const values = f[2];
|
|
144
|
+
if (Array.isArray(values)) {
|
|
145
|
+
let hasObject = false;
|
|
146
|
+
for (const v of values) {
|
|
147
|
+
if (typeof v === 'object' && 'id' in v) {
|
|
148
|
+
hasObject = true;
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (hasObject) {
|
|
153
|
+
f[2] = values.map((v) => {
|
|
154
|
+
if (typeof v === 'object' && 'id' in v) {
|
|
155
|
+
return v.id;
|
|
156
|
+
}
|
|
157
|
+
return v;
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
else if (typeof values === 'object' && 'id' in values) {
|
|
162
|
+
f[2] = values.id;
|
|
163
|
+
}
|
|
142
164
|
if (validateVal(def, f, prop.validation)) {
|
|
143
165
|
return true;
|
|
144
166
|
}
|
|
@@ -274,7 +296,7 @@ export const validateSort = (def, field, orderInput) => {
|
|
|
274
296
|
return {
|
|
275
297
|
prop: EMPTY_ALIAS_PROP_DEF,
|
|
276
298
|
order,
|
|
277
|
-
lang: def.lang,
|
|
299
|
+
lang: def.lang?.lang,
|
|
278
300
|
};
|
|
279
301
|
}
|
|
280
302
|
}
|
|
@@ -287,7 +309,7 @@ export const validateSort = (def, field, orderInput) => {
|
|
|
287
309
|
}
|
|
288
310
|
else if (type === TEXT) {
|
|
289
311
|
if (lang === 0) {
|
|
290
|
-
lang = def.lang ?? 0;
|
|
312
|
+
lang = def.lang?.lang ?? 0;
|
|
291
313
|
if (lang === 0) {
|
|
292
314
|
def.errors.push({
|
|
293
315
|
code: ERR_SORT_LANG,
|
|
@@ -445,6 +467,7 @@ export const EMPTY_SCHEMA_DEF = {
|
|
|
445
467
|
propNames: new Uint8Array([]),
|
|
446
468
|
idUint8: new Uint8Array([0, 0]),
|
|
447
469
|
mainEmptyAllZeroes: true,
|
|
470
|
+
hasSeperateDefaults: false,
|
|
448
471
|
};
|
|
449
472
|
export const aggregationFieldDoesNotExist = (def, field) => {
|
|
450
473
|
def.errors.push({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const xxHash64: (buf: Uint8Array, target?: Uint8Array, index?: number) => Uint8Array
|
|
1
|
+
export declare const xxHash64: (buf: Uint8Array, target?: Uint8Array, index?: number) => Uint8Array<ArrayBufferLike>;
|
package/dist/src/index.d.ts
CHANGED
|
@@ -12,7 +12,6 @@ export { DbClient, DbServer };
|
|
|
12
12
|
export { xxHash64 } from './client/xxHash64.js';
|
|
13
13
|
export { crc32 } from './client/crc32.js';
|
|
14
14
|
export { default as createHash } from './server/dbHash.js';
|
|
15
|
-
export * from './client/query/serialize.js';
|
|
16
15
|
export * from './utils.js';
|
|
17
16
|
export * from './client/query/query.js';
|
|
18
17
|
export * from './client/query/BasedDbQuery.js';
|
package/dist/src/index.js
CHANGED
|
@@ -13,7 +13,6 @@ export { DbClient, DbServer };
|
|
|
13
13
|
export { xxHash64 } from './client/xxHash64.js';
|
|
14
14
|
export { crc32 } from './client/crc32.js';
|
|
15
15
|
export { default as createHash } from './server/dbHash.js';
|
|
16
|
-
export * from './client/query/serialize.js';
|
|
17
16
|
export * from './utils.js';
|
|
18
17
|
export * from './client/query/query.js';
|
|
19
18
|
export * from './client/query/BasedDbQuery.js';
|
package/dist/src/native.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
declare const native: {
|
|
2
2
|
threadCtx: any;
|
|
3
|
-
historyAppend(history: any, typeId: number, nodeId: number, dbCtx: any): any;
|
|
4
|
-
historyCreate(pathname: string, mainLen: number): any;
|
|
5
3
|
workerCtxInit: () => void;
|
|
6
4
|
externalFromInt(address: BigInt): any;
|
|
7
5
|
intFromExternal(external: any): BigInt;
|
|
@@ -13,6 +11,7 @@ declare const native: {
|
|
|
13
11
|
saveBlock: (path: string, typeCode: number, start: number, dbCtx: any, hashOut: Uint8Array) => number;
|
|
14
12
|
loadCommon: (path: string, dbCtx: any) => void;
|
|
15
13
|
loadBlock: (path: string, dbCtx: any) => void;
|
|
14
|
+
delBlock: (dbCtx: any, typeId: number, block: number) => void;
|
|
16
15
|
updateSchemaType: (prefix: number, buf: Uint8Array, dbCtx: any) => any;
|
|
17
16
|
getTypeInfo: (typeId: number, dbCtx: any) => any;
|
|
18
17
|
getNodeRangeHash: (typeId: number, start: number, end: number, bufOut: Uint8Array, dbCtx: any) => any;
|
|
@@ -25,5 +24,6 @@ declare const native: {
|
|
|
25
24
|
equals: (a: Uint8Array, b: Uint8Array) => boolean;
|
|
26
25
|
membarSyncRead: () => void;
|
|
27
26
|
membarSyncWrite: () => void;
|
|
27
|
+
colvecTest: (dbCtx: any, typeId: number, field: number, nodeId: number, len: number) => any;
|
|
28
28
|
};
|
|
29
29
|
export default native;
|
package/dist/src/native.js
CHANGED
|
@@ -17,13 +17,6 @@ function SelvaIoErrlogToString(buf) {
|
|
|
17
17
|
// then add ThreadCtx to modify ctx and query ctx
|
|
18
18
|
const native = {
|
|
19
19
|
threadCtx: null, // add compressors here as well!
|
|
20
|
-
historyAppend(history, typeId, nodeId, dbCtx) {
|
|
21
|
-
return db.historyAppend(history, typeId, nodeId, dbCtx);
|
|
22
|
-
},
|
|
23
|
-
historyCreate(pathname, mainLen) {
|
|
24
|
-
const pathBuf = ENCODER.encode(pathname + '\0');
|
|
25
|
-
return db.historyCreate(pathBuf, mainLen + 16 - (mainLen % 16));
|
|
26
|
-
},
|
|
27
20
|
workerCtxInit: () => {
|
|
28
21
|
return db.workerCtxInit();
|
|
29
22
|
},
|
|
@@ -65,9 +58,12 @@ const native = {
|
|
|
65
58
|
const pathBuf = ENCODER.encode(path + '\0');
|
|
66
59
|
const err = db.loadBlock(pathBuf, dbCtx, selvaIoErrlog);
|
|
67
60
|
if (err) {
|
|
68
|
-
throw new Error(`Failed to load a range. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
|
|
61
|
+
throw new Error(`Failed to load a range "${path}". selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
|
|
69
62
|
}
|
|
70
63
|
},
|
|
64
|
+
delBlock: (dbCtx, typeId, block) => {
|
|
65
|
+
db.delBlock(dbCtx, typeId, block);
|
|
66
|
+
},
|
|
71
67
|
updateSchemaType: (prefix, buf, dbCtx) => {
|
|
72
68
|
return db.updateSchema(prefix, buf, dbCtx);
|
|
73
69
|
},
|
|
@@ -104,6 +100,9 @@ const native = {
|
|
|
104
100
|
membarSyncWrite: () => {
|
|
105
101
|
db.membarSyncWrite();
|
|
106
102
|
},
|
|
103
|
+
colvecTest: (dbCtx, typeId, field, nodeId, len) => {
|
|
104
|
+
return db.colvecTest(dbCtx, typeId, field, nodeId, len);
|
|
105
|
+
}
|
|
107
106
|
};
|
|
108
107
|
global.__basedDb__native__ = native;
|
|
109
108
|
export default native;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { DbWorker } from './workers/DbWorker.js';
|
|
2
|
+
import { DbServer } from './index.js';
|
|
3
|
+
export declare class IoWorker extends DbWorker {
|
|
4
|
+
constructor(address: BigInt, db: DbServer);
|
|
5
|
+
handleMsg(_buf: any): void;
|
|
6
|
+
loadBlock(filepath: string): Promise<void>;
|
|
7
|
+
unloadBlock(filepath: string, typeId: number, start: number): Promise<Uint8Array>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { DbWorker } from './workers/DbWorker.js';
|
|
2
|
+
import { DECODER, readInt32 } from '@saulx/utils';
|
|
3
|
+
export class IoWorker extends DbWorker {
|
|
4
|
+
constructor(address, db) {
|
|
5
|
+
const onExit = (_code) => {
|
|
6
|
+
this.db.ioWorker = new IoWorker(address, db);
|
|
7
|
+
};
|
|
8
|
+
super(address, db, onExit, 'io_worker.js');
|
|
9
|
+
}
|
|
10
|
+
handleMsg(_buf) {
|
|
11
|
+
}
|
|
12
|
+
async loadBlock(filepath) {
|
|
13
|
+
const job = {
|
|
14
|
+
type: 'load',
|
|
15
|
+
filepath
|
|
16
|
+
};
|
|
17
|
+
const resBuf = await this.call(job);
|
|
18
|
+
if (resBuf.length) {
|
|
19
|
+
throw new Error(DECODER.decode(resBuf));
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
async unloadBlock(filepath, typeId, start) {
|
|
23
|
+
const job = {
|
|
24
|
+
type: 'unload',
|
|
25
|
+
filepath,
|
|
26
|
+
typeId,
|
|
27
|
+
start
|
|
28
|
+
};
|
|
29
|
+
const resBuf = await this.call(job);
|
|
30
|
+
const err = readInt32(resBuf, 0);
|
|
31
|
+
if (err) {
|
|
32
|
+
throw new Error(`selva error: ${err}`);
|
|
33
|
+
}
|
|
34
|
+
// Note that this shares the original buffer which may not be 100% optimal,
|
|
35
|
+
// as the first 4 bytes are no longer needed.
|
|
36
|
+
return new Uint8Array(resBuf.buffer, 4);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=IoWorker.js.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { DbWorker } from './workers/DbWorker.js';
|
|
2
|
+
import { DbServer } from './index.js';
|
|
3
|
+
export declare class QueryWorker extends DbWorker {
|
|
4
|
+
constructor(address: BigInt, db: DbServer, workerIndex: number);
|
|
5
|
+
handleMsg(_buf: any): void;
|
|
6
|
+
protected callback: (resolve: (x: any) => any) => void;
|
|
7
|
+
getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { DbWorker } from './workers/DbWorker.js';
|
|
2
|
+
import { readUint64 } from '@saulx/utils';
|
|
3
|
+
export class QueryWorker extends DbWorker {
|
|
4
|
+
constructor(address, db, workerIndex) {
|
|
5
|
+
const onExit = (_code) => {
|
|
6
|
+
this.db.workers[workerIndex] = new QueryWorker(address, db, workerIndex);
|
|
7
|
+
};
|
|
8
|
+
super(address, db, onExit, 'query_worker.js');
|
|
9
|
+
}
|
|
10
|
+
handleMsg(_buf) {
|
|
11
|
+
this.db.processingQueries--;
|
|
12
|
+
this.db.onQueryEnd();
|
|
13
|
+
}
|
|
14
|
+
callback = (resolve) => {
|
|
15
|
+
this.db.processingQueries++;
|
|
16
|
+
this.resolvers.push(resolve);
|
|
17
|
+
};
|
|
18
|
+
getQueryBuf(buf) {
|
|
19
|
+
const schemaChecksum = readUint64(buf, buf.byteLength - 8);
|
|
20
|
+
if (schemaChecksum !== this.db.schema?.hash) {
|
|
21
|
+
return Promise.resolve(new Uint8Array(1));
|
|
22
|
+
}
|
|
23
|
+
return this.call(buf);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=QueryWorker.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { SchemaTypeDef } from '@based/schema/def';
|
|
2
|
+
import { DbServer } from './index.js';
|
|
3
|
+
/**
|
|
4
|
+
* Save a block.
|
|
5
|
+
*/
|
|
6
|
+
export declare function saveBlock(db: DbServer, typeId: number, start: number, end: number): void;
|
|
7
|
+
/**
|
|
8
|
+
* Load a block (typically of a partial type) back to memory.
|
|
9
|
+
*/
|
|
10
|
+
export declare function loadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Save a block and remove it from memory.
|
|
13
|
+
*/
|
|
14
|
+
export declare function unloadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Execute cb() for each block in memory.
|
|
17
|
+
*/
|
|
18
|
+
export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void, includeEmptyBlocks?: boolean): void;
|
|
19
|
+
/**
|
|
20
|
+
* Execute cb() for each dirty block.
|
|
21
|
+
* A dirty block is one that is changed in memory but not yet persisted in the
|
|
22
|
+
* file system.
|
|
23
|
+
*/
|
|
24
|
+
export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): void;
|