@based/db 0.0.21 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/dist/lib/darwin_aarch64/include/selva/types.h +9 -18
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/linux_aarch64/include/selva/types.h +9 -18
- package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/selva/types.h +9 -18
- package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/bitWise.d.ts +0 -1
- package/dist/src/client/bitWise.js +0 -10
- package/dist/src/client/{operations.d.ts → flushModify.d.ts} +4 -4
- package/dist/src/client/{operations.js → flushModify.js} +39 -15
- package/dist/src/client/index.d.ts +10 -6
- package/dist/src/client/index.js +12 -4
- package/dist/src/client/modify/ModifyRes.d.ts +2 -8
- package/dist/src/client/modify/ModifyRes.js +23 -29
- package/dist/src/client/modify/alias.d.ts +1 -1
- package/dist/src/client/modify/alias.js +5 -2
- package/dist/src/client/modify/binary.d.ts +1 -1
- package/dist/src/client/modify/binary.js +1 -1
- package/dist/src/client/modify/cardinality.d.ts +2 -2
- package/dist/src/client/modify/cardinality.js +7 -5
- package/dist/src/client/modify/create.js +10 -5
- package/dist/src/client/modify/delete.d.ts +3 -1
- package/dist/src/client/modify/delete.js +12 -5
- package/dist/src/client/modify/expire.js +1 -1
- package/dist/src/client/modify/fixed.js +11 -4
- package/dist/src/client/modify/references/appendRefs.d.ts +4 -0
- package/dist/src/client/modify/references/appendRefs.js +27 -0
- package/dist/src/client/modify/references/edge.d.ts +0 -1
- package/dist/src/client/modify/references/edge.js +191 -71
- package/dist/src/client/modify/references/getEdgeSize.d.ts +3 -0
- package/dist/src/client/modify/references/getEdgeSize.js +27 -0
- package/dist/src/client/modify/references/reference.js +40 -26
- package/dist/src/client/modify/references/references.js +18 -6
- package/dist/src/client/modify/string.js +2 -1
- package/dist/src/client/modify/types.d.ts +1 -0
- package/dist/src/client/modify/types.js +1 -0
- package/dist/src/client/modify/update.js +3 -4
- package/dist/src/client/modify/upsert.js +1 -0
- package/dist/src/client/modify/vector.js +0 -2
- package/dist/src/client/query/BasedDbQuery.d.ts +4 -3
- package/dist/src/client/query/BasedDbQuery.js +12 -3
- package/dist/src/client/query/aggregationFn.d.ts +3 -0
- package/dist/src/client/query/aggregationFn.js +9 -0
- package/dist/src/client/query/debug.js +2 -6
- package/dist/src/client/query/display.js +5 -2
- package/dist/src/client/query/filter/createFixedFilterBuffer.d.ts +2 -2
- package/dist/src/client/query/filter/createFixedFilterBuffer.js +34 -26
- package/dist/src/client/query/filter/createReferenceFilter.d.ts +1 -1
- package/dist/src/client/query/filter/createReferenceFilter.js +17 -7
- package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +1 -1
- package/dist/src/client/query/filter/createVariableFilterBuffer.js +45 -22
- package/dist/src/client/query/filter/filter.d.ts +1 -1
- package/dist/src/client/query/filter/filter.js +5 -0
- package/dist/src/client/query/filter/parseFilterValue.js +1 -1
- package/dist/src/client/query/filter/toBuffer.d.ts +2 -2
- package/dist/src/client/query/filter/toBuffer.js +16 -9
- package/dist/src/client/query/include/props.js +18 -1
- package/dist/src/client/query/include/toBuffer.d.ts +1 -1
- package/dist/src/client/query/include/toBuffer.js +22 -12
- package/dist/src/client/query/include/walk.js +2 -1
- package/dist/src/client/query/query.d.ts +1 -0
- package/dist/src/client/query/query.js +1 -0
- package/dist/src/client/query/queryDef.js +2 -1
- package/dist/src/client/query/read/read.d.ts +2 -1
- package/dist/src/client/query/read/read.js +95 -62
- package/dist/src/client/query/registerQuery.d.ts +1 -1
- package/dist/src/client/query/registerQuery.js +2 -1
- package/dist/src/client/query/search/index.d.ts +1 -1
- package/dist/src/client/query/search/index.js +56 -24
- package/dist/src/client/query/subscription/markers.js +2 -1
- package/dist/src/client/query/toBuffer.d.ts +1 -1
- package/dist/src/client/query/toBuffer.js +80 -32
- package/dist/src/client/query/types.d.ts +25 -3
- package/dist/src/client/query/types.js +6 -0
- package/dist/src/client/query/validation.js +1 -1
- package/dist/src/client/string.d.ts +1 -1
- package/dist/src/client/string.js +1 -2
- package/dist/src/client/xxHash64.d.ts +1 -1
- package/dist/src/index.d.ts +4 -3
- package/dist/src/index.js +20 -11
- package/dist/src/native.d.ts +8 -13
- package/dist/src/native.js +9 -31
- package/dist/src/server/dbHash.d.ts +5 -0
- package/dist/src/server/dbHash.js +27 -0
- package/dist/src/server/index.d.ts +10 -10
- package/dist/src/server/index.js +54 -35
- package/dist/src/server/migrate/index.js +1 -1
- package/dist/src/server/migrate/worker.js +2 -2
- package/dist/src/server/save.d.ts +1 -1
- package/dist/src/server/save.js +10 -10
- package/dist/src/server/start.js +4 -2
- package/dist/src/server/tree.d.ts +1 -1
- package/dist/src/server/tree.js +1 -1
- package/dist/src/utils.d.ts +3 -0
- package/dist/src/utils.js +23 -5
- package/package.json +3 -2
- package/dist/src/client/query/read/types.d.ts +0 -4
- package/dist/src/client/query/read/types.js +0 -5
|
@@ -5,4 +5,4 @@ export type Search = string[] | {
|
|
|
5
5
|
} | string;
|
|
6
6
|
export declare const vectorSearch: (def: QueryDef, q: ArrayBufferView, field: string, opts: Omit<FilterOpts, "lowerCase">) => void;
|
|
7
7
|
export declare const search: (def: QueryDef, q: string, s?: Search) => void;
|
|
8
|
-
export declare const searchToBuffer: (search: QueryDefSearch) =>
|
|
8
|
+
export declare const searchToBuffer: (search: QueryDefSearch) => Uint8Array;
|
|
@@ -2,7 +2,7 @@ import { langCodesMap } from '@based/schema';
|
|
|
2
2
|
import { STRING, TEXT, VECTOR } from '@based/schema/def';
|
|
3
3
|
import { getVectorFn } from '../filter/types.js';
|
|
4
4
|
import { searchDoesNotExist, searchIncorrecQueryValue, searchIncorrectType, } from '../validation.js';
|
|
5
|
-
|
|
5
|
+
import { ENCODER, concatUint8Arr } from '../../../utils.js';
|
|
6
6
|
// vector
|
|
7
7
|
export const vectorSearch = (def, q, field, opts) => {
|
|
8
8
|
let prop = def.props[field];
|
|
@@ -24,16 +24,6 @@ export const vectorSearch = (def, q, field, opts) => {
|
|
|
24
24
|
opts,
|
|
25
25
|
};
|
|
26
26
|
};
|
|
27
|
-
function concatBufs(bufs, totalByteLength) {
|
|
28
|
-
const res = new Uint8Array(totalByteLength);
|
|
29
|
-
let off = 0;
|
|
30
|
-
for (let i = 0; i < bufs.length; i++) {
|
|
31
|
-
const buf = bufs[i];
|
|
32
|
-
res.set(buf, off);
|
|
33
|
-
off += buf.byteLength;
|
|
34
|
-
}
|
|
35
|
-
return res;
|
|
36
|
-
}
|
|
37
27
|
export const search = (def, q, s) => {
|
|
38
28
|
const bufs = [];
|
|
39
29
|
let nrBlocks = 0;
|
|
@@ -42,7 +32,12 @@ export const search = (def, q, s) => {
|
|
|
42
32
|
searchIncorrecQueryValue(def, q);
|
|
43
33
|
q = '';
|
|
44
34
|
}
|
|
45
|
-
const x = q
|
|
35
|
+
const x = q
|
|
36
|
+
.toLowerCase()
|
|
37
|
+
.normalize('NFKD')
|
|
38
|
+
.trim()
|
|
39
|
+
.split(' ')
|
|
40
|
+
.map((s) => ` ${s}`);
|
|
46
41
|
for (const s of x) {
|
|
47
42
|
if (s) {
|
|
48
43
|
const buf = ENCODER.encode(s);
|
|
@@ -55,7 +50,7 @@ export const search = (def, q, s) => {
|
|
|
55
50
|
}
|
|
56
51
|
}
|
|
57
52
|
bufs.unshift(Uint8Array.from([nrBlocks]));
|
|
58
|
-
const query =
|
|
53
|
+
const query = concatUint8Arr(bufs, totalByteLength);
|
|
59
54
|
def.search = {
|
|
60
55
|
size: query.byteLength + 3,
|
|
61
56
|
query,
|
|
@@ -104,8 +99,9 @@ export const search = (def, q, s) => {
|
|
|
104
99
|
if (prop.typeIndex !== STRING && prop.typeIndex !== TEXT) {
|
|
105
100
|
searchIncorrectType(def, prop);
|
|
106
101
|
}
|
|
107
|
-
def.search.size +=
|
|
102
|
+
def.search.size += 6;
|
|
108
103
|
def.search.fields.push({
|
|
104
|
+
typeIndex: prop.typeIndex,
|
|
109
105
|
weight: s[key],
|
|
110
106
|
lang,
|
|
111
107
|
field: prop.prop,
|
|
@@ -115,10 +111,20 @@ export const search = (def, q, s) => {
|
|
|
115
111
|
};
|
|
116
112
|
export const searchToBuffer = (search) => {
|
|
117
113
|
if (search.isVector) {
|
|
118
|
-
|
|
119
|
-
|
|
114
|
+
/* Vector Binary Schema:
|
|
115
|
+
| Offset | Field | Size (bytes) | Description |
|
|
116
|
+
|--------|----------|--------------|-------------------------------------------------|
|
|
117
|
+
| 0 | isVector | 1 | Indicates if search is a vector (always 1) |
|
|
118
|
+
| 1 | queryLen | 2 | Length of the query in bytes (u16) |
|
|
119
|
+
| 3 | field | 1 | Field identifier |
|
|
120
|
+
| 4 | func | 1 | Function identifier (enum) |
|
|
121
|
+
| 5 | score | 4 | Score value (f32) |
|
|
122
|
+
| 9 | query | queryLen | Query data (array of f32 values) |
|
|
123
|
+
*/
|
|
124
|
+
const result = new Uint8Array(search.size);
|
|
120
125
|
result[0] = 1; // search.isVector 1
|
|
121
|
-
result
|
|
126
|
+
result[1] = search.query.byteLength;
|
|
127
|
+
result[2] = search.query.byteLength >>> 8;
|
|
122
128
|
result[3] = search.prop;
|
|
123
129
|
result[4] = getVectorFn(search.opts.fn);
|
|
124
130
|
result.set(new Uint8Array(new Float32Array([search.opts.score ?? 0.5]).buffer), 5);
|
|
@@ -126,9 +132,33 @@ export const searchToBuffer = (search) => {
|
|
|
126
132
|
return result;
|
|
127
133
|
}
|
|
128
134
|
else {
|
|
129
|
-
|
|
135
|
+
/* Non-Vector Search Binary Schema:
|
|
136
|
+
|
|
137
|
+
| Offset | Field | Size (bytes) | Description |
|
|
138
|
+
|--------|----------|--------------|--------------------------------------------|
|
|
139
|
+
| 0 | isVector | 1 | Indicates if search is a vector (always 0) |
|
|
140
|
+
| 1 | queryLen | 2 | Length of the query in bytes (u16) |
|
|
141
|
+
| 3 | query | queryLen | Query data |
|
|
142
|
+
| X | fields | Variable | Sorted fields metadata |
|
|
143
|
+
|
|
144
|
+
### Fields Metadata Structure:
|
|
145
|
+
Each field entry consists of 6 bytes:
|
|
146
|
+
|
|
147
|
+
| Offset | Field | Size (bytes)| Description |
|
|
148
|
+
|--------|-----------|-------------|--------------------------------------|
|
|
149
|
+
| 0 | field | 1 | Field identifier |
|
|
150
|
+
| 1 | typeIndex | 1 | Type index of the field |
|
|
151
|
+
| 2 | weight | 1 | Field weight value |
|
|
152
|
+
| 3 | start | 2 | Start position in the query (u16) |
|
|
153
|
+
| 5 | lang | 1 | Language identifier |
|
|
154
|
+
|
|
155
|
+
### Notes:
|
|
156
|
+
- The number of field entries is inferred from the total packet size.
|
|
157
|
+
- `fields` are sorted by `weight` before being stored in the buffer.*/
|
|
158
|
+
const result = new Uint8Array(search.size);
|
|
130
159
|
result[0] = 0; // search.isVector 0
|
|
131
|
-
result
|
|
160
|
+
result[1] = search.query.byteLength;
|
|
161
|
+
result[2] = search.query.byteLength >>> 8;
|
|
132
162
|
result.set(search.query, 3);
|
|
133
163
|
const offset = search.query.byteLength + 3;
|
|
134
164
|
// @ts-ignore
|
|
@@ -136,13 +166,15 @@ export const searchToBuffer = (search) => {
|
|
|
136
166
|
return a.weight - b.weight;
|
|
137
167
|
});
|
|
138
168
|
// @ts-ignore
|
|
139
|
-
for (let i = 0; i < search.fields.length *
|
|
169
|
+
for (let i = 0; i < search.fields.length * 6; i += 6) {
|
|
140
170
|
// @ts-ignore
|
|
141
|
-
const f = search.fields[i /
|
|
171
|
+
const f = search.fields[Math.floor(i / 6)];
|
|
142
172
|
result[i + offset] = f.field;
|
|
143
|
-
result[i +
|
|
144
|
-
result
|
|
145
|
-
result[i +
|
|
173
|
+
result[i + offset + 1] = f.typeIndex;
|
|
174
|
+
result[i + offset + 2] = f.weight;
|
|
175
|
+
result[i + offset + 3] = f.start;
|
|
176
|
+
result[i + offset + 4] = f.start >>> 8;
|
|
177
|
+
result[i + offset + 5] = f.lang;
|
|
146
178
|
}
|
|
147
179
|
return result;
|
|
148
180
|
}
|
|
@@ -3,6 +3,8 @@ import { QueryDefType } from './types.js';
|
|
|
3
3
|
import { includeToBuffer } from './include/toBuffer.js';
|
|
4
4
|
import { filterToBuffer } from './query.js';
|
|
5
5
|
import { searchToBuffer } from './search/index.js';
|
|
6
|
+
import { createAggFnBuffer } from './aggregationFn.js';
|
|
7
|
+
import { ENCODER } from '../../utils.js';
|
|
6
8
|
const byteSize = (arr) => {
|
|
7
9
|
return arr.reduce((a, b) => {
|
|
8
10
|
return a + b.byteLength;
|
|
@@ -48,18 +50,21 @@ export function defToBuffer(db, def) {
|
|
|
48
50
|
if (def.target.resolvedAlias) {
|
|
49
51
|
// put this somehwere else at some point
|
|
50
52
|
const alias = def.target.resolvedAlias;
|
|
51
|
-
const
|
|
53
|
+
const aliasStr = ENCODER.encode(alias.value);
|
|
54
|
+
const aliasLen = aliasStr.byteLength;
|
|
52
55
|
// filter is nice for things like access
|
|
53
|
-
const buf =
|
|
56
|
+
const buf = new Uint8Array(8 + filterSize + aliasLen);
|
|
54
57
|
buf[0] = 3;
|
|
55
58
|
buf[1] = def.schema.idUint8[0];
|
|
56
59
|
buf[2] = def.schema.idUint8[1];
|
|
57
60
|
buf[3] = alias.def.prop;
|
|
58
|
-
buf
|
|
59
|
-
buf
|
|
60
|
-
buf.
|
|
61
|
+
buf[4] = aliasLen;
|
|
62
|
+
buf[5] = aliasLen >>> 8;
|
|
63
|
+
buf.set(aliasStr, 6);
|
|
64
|
+
buf[6 + aliasLen] = filterSize;
|
|
65
|
+
buf[7 + aliasLen] = filterSize >>> 8;
|
|
61
66
|
if (filterSize) {
|
|
62
|
-
buf.set(filter, 8 +
|
|
67
|
+
buf.set(filter, 8 + aliasLen);
|
|
63
68
|
}
|
|
64
69
|
result.push(buf);
|
|
65
70
|
}
|
|
@@ -67,12 +72,16 @@ export function defToBuffer(db, def) {
|
|
|
67
72
|
// type 0
|
|
68
73
|
// 0: 4 [id]
|
|
69
74
|
// 0: 2 [filterSize]
|
|
70
|
-
const buf =
|
|
75
|
+
const buf = new Uint8Array(9 + filterSize);
|
|
71
76
|
buf[0] = 0;
|
|
72
77
|
buf[1] = def.schema.idUint8[0];
|
|
73
78
|
buf[2] = def.schema.idUint8[1];
|
|
74
|
-
buf
|
|
75
|
-
buf.
|
|
79
|
+
buf[3] = def.target.id;
|
|
80
|
+
buf[4] = def.target.id >>> 8;
|
|
81
|
+
buf[5] = def.target.id >>> 16;
|
|
82
|
+
buf[6] = def.target.id >>> 24;
|
|
83
|
+
buf[7] = filterSize;
|
|
84
|
+
buf[8] = filterSize >>> 8;
|
|
76
85
|
if (filterSize) {
|
|
77
86
|
buf.set(filter, 9);
|
|
78
87
|
}
|
|
@@ -85,6 +94,8 @@ export function defToBuffer(db, def) {
|
|
|
85
94
|
sort = createSortBuffer(def.sort);
|
|
86
95
|
sortSize = sort.byteLength;
|
|
87
96
|
}
|
|
97
|
+
let aggregation;
|
|
98
|
+
aggregation = createAggFnBuffer(def.aggregation);
|
|
88
99
|
if (def.target.ids) {
|
|
89
100
|
// type 1
|
|
90
101
|
// 1: 4 + ids * 4 [ids len] [id,id,id]
|
|
@@ -98,23 +109,35 @@ export function defToBuffer(db, def) {
|
|
|
98
109
|
def.target.ids = def.target.ids.slice(def.range.offset, def.range.offset + def.range.limit);
|
|
99
110
|
}
|
|
100
111
|
const idsSize = def.target.ids.length * 4;
|
|
101
|
-
const buf =
|
|
112
|
+
const buf = new Uint8Array(21 + idsSize + filterSize + sortSize + searchSize);
|
|
102
113
|
buf[0] = 1;
|
|
103
114
|
buf[1] = def.schema.idUint8[0];
|
|
104
115
|
buf[2] = def.schema.idUint8[1];
|
|
105
|
-
buf
|
|
116
|
+
buf[3] = idsSize;
|
|
117
|
+
buf[4] = idsSize >>> 8;
|
|
118
|
+
buf[5] = idsSize >>> 16;
|
|
119
|
+
buf[6] = idsSize >>> 24;
|
|
106
120
|
buf.set(new Uint8Array(def.target.ids.buffer), 7);
|
|
107
|
-
buf
|
|
108
|
-
buf
|
|
109
|
-
buf
|
|
121
|
+
buf[idsSize + 7] = def.range.offset;
|
|
122
|
+
buf[idsSize + 8] = def.range.offset >>> 8;
|
|
123
|
+
buf[idsSize + 9] = def.range.offset >>> 16;
|
|
124
|
+
buf[idsSize + 10] = def.range.offset >>> 24;
|
|
125
|
+
buf[idsSize + 11] = def.range.limit;
|
|
126
|
+
buf[idsSize + 12] = def.range.limit >>> 8;
|
|
127
|
+
buf[idsSize + 13] = def.range.limit >>> 16;
|
|
128
|
+
buf[idsSize + 14] = def.range.limit >>> 24;
|
|
129
|
+
buf[idsSize + 15] = filterSize;
|
|
130
|
+
buf[idsSize + 16] = filterSize >>> 8;
|
|
110
131
|
if (filterSize) {
|
|
111
132
|
buf.set(filter, idsSize + 17);
|
|
112
133
|
}
|
|
113
|
-
buf
|
|
134
|
+
buf[17 + filterSize + idsSize] = sortSize;
|
|
135
|
+
buf[18 + filterSize + idsSize] = sortSize >>> 8;
|
|
114
136
|
if (sortSize) {
|
|
115
137
|
buf.set(sort, 19 + filterSize + idsSize);
|
|
116
138
|
}
|
|
117
|
-
buf
|
|
139
|
+
buf[19 + filterSize + idsSize + sortSize] = searchSize;
|
|
140
|
+
buf[20 + filterSize + idsSize + sortSize] = searchSize >>> 8;
|
|
118
141
|
if (searchSize) {
|
|
119
142
|
buf.set(search, 21 + filterSize + idsSize + sortSize);
|
|
120
143
|
}
|
|
@@ -129,24 +152,36 @@ export function defToBuffer(db, def) {
|
|
|
129
152
|
// ?filter
|
|
130
153
|
// 2: 2 [sort size]
|
|
131
154
|
// ?sort
|
|
132
|
-
const buf =
|
|
155
|
+
const buf = new Uint8Array(18 + filterSize + sortSize + searchSize);
|
|
133
156
|
buf[0] = 2;
|
|
134
157
|
buf[1] = def.schema.idUint8[0];
|
|
135
158
|
buf[2] = def.schema.idUint8[1];
|
|
136
|
-
buf
|
|
137
|
-
buf
|
|
138
|
-
buf.
|
|
159
|
+
buf[3] = def.range.offset;
|
|
160
|
+
buf[4] = def.range.offset >>> 8;
|
|
161
|
+
buf[5] = def.range.offset >>> 16;
|
|
162
|
+
buf[6] = def.range.offset >>> 24;
|
|
163
|
+
buf[7] = def.range.limit;
|
|
164
|
+
buf[8] = def.range.limit >>> 8;
|
|
165
|
+
buf[9] = def.range.limit >>> 16;
|
|
166
|
+
buf[10] = def.range.limit >>> 24;
|
|
167
|
+
buf[11] = filterSize;
|
|
168
|
+
buf[12] = filterSize >>> 8;
|
|
139
169
|
if (filterSize) {
|
|
140
170
|
buf.set(filter, 13);
|
|
141
171
|
}
|
|
142
|
-
buf
|
|
172
|
+
buf[13 + filterSize] = sortSize;
|
|
173
|
+
buf[14 + filterSize] = sortSize >>> 8;
|
|
143
174
|
if (sortSize) {
|
|
144
175
|
buf.set(sort, 15 + filterSize);
|
|
145
176
|
}
|
|
146
|
-
buf
|
|
177
|
+
buf[15 + filterSize + sortSize] = searchSize;
|
|
178
|
+
buf[16 + filterSize + sortSize] = searchSize >>> 8;
|
|
147
179
|
if (searchSize) {
|
|
148
180
|
buf.set(search, 17 + filterSize + sortSize);
|
|
149
181
|
}
|
|
182
|
+
if (aggregation) {
|
|
183
|
+
buf.set(aggregation, 17 + filterSize + sortSize + searchSize);
|
|
184
|
+
}
|
|
150
185
|
result.push(buf);
|
|
151
186
|
}
|
|
152
187
|
}
|
|
@@ -166,13 +201,23 @@ export function defToBuffer(db, def) {
|
|
|
166
201
|
const sortSize = sort?.byteLength ?? 0;
|
|
167
202
|
const filterSize = filter?.byteLength ?? 0;
|
|
168
203
|
const modsSize = filterSize + sortSize;
|
|
169
|
-
const meta =
|
|
204
|
+
const meta = new Uint8Array(modsSize + 10 + 8);
|
|
205
|
+
const sz = size + 7 + modsSize + 8;
|
|
170
206
|
meta[0] = 254;
|
|
171
|
-
meta
|
|
172
|
-
meta
|
|
173
|
-
meta
|
|
174
|
-
meta
|
|
175
|
-
meta
|
|
207
|
+
meta[1] = sz;
|
|
208
|
+
meta[2] = sz >>> 8;
|
|
209
|
+
meta[3] = filterSize;
|
|
210
|
+
meta[4] = filterSize >>> 8;
|
|
211
|
+
meta[5] = sortSize;
|
|
212
|
+
meta[6] = sortSize >>> 8;
|
|
213
|
+
meta[7] = def.range.offset;
|
|
214
|
+
meta[8] = def.range.offset >>> 8;
|
|
215
|
+
meta[9] = def.range.offset >>> 16;
|
|
216
|
+
meta[10] = def.range.offset >>> 24;
|
|
217
|
+
meta[11] = def.range.limit;
|
|
218
|
+
meta[12] = def.range.limit >>> 8;
|
|
219
|
+
meta[13] = def.range.limit >>> 16;
|
|
220
|
+
meta[14] = def.range.limit >>> 24;
|
|
176
221
|
if (filter) {
|
|
177
222
|
meta.set(filter, 15);
|
|
178
223
|
}
|
|
@@ -185,9 +230,11 @@ export function defToBuffer(db, def) {
|
|
|
185
230
|
result.push(meta);
|
|
186
231
|
}
|
|
187
232
|
else if (def.type === QueryDefType.Reference) {
|
|
188
|
-
const meta =
|
|
233
|
+
const meta = new Uint8Array(6);
|
|
234
|
+
const sz = size + 3;
|
|
189
235
|
meta[0] = 255;
|
|
190
|
-
meta
|
|
236
|
+
meta[1] = sz;
|
|
237
|
+
meta[2] = sz >>> 8;
|
|
191
238
|
meta[3] = def.schema.idUint8[0];
|
|
192
239
|
meta[4] = def.schema.idUint8[1];
|
|
193
240
|
meta[5] = def.target.propDef.prop;
|
|
@@ -195,9 +242,10 @@ export function defToBuffer(db, def) {
|
|
|
195
242
|
}
|
|
196
243
|
result.push(...include);
|
|
197
244
|
if (edges) {
|
|
198
|
-
const metaEdgeBuffer =
|
|
245
|
+
const metaEdgeBuffer = new Uint8Array(3);
|
|
199
246
|
metaEdgeBuffer[0] = 252;
|
|
200
|
-
metaEdgeBuffer
|
|
247
|
+
metaEdgeBuffer[1] = edgesSize;
|
|
248
|
+
metaEdgeBuffer[2] = edgesSize >>> 8;
|
|
201
249
|
result.push(metaEdgeBuffer, ...edges);
|
|
202
250
|
}
|
|
203
251
|
return result;
|
|
@@ -29,7 +29,7 @@ export type Target = {
|
|
|
29
29
|
export declare const isRefDef: (def: QueryDef) => def is QueryDefRest;
|
|
30
30
|
export type QueryDefFilter = {
|
|
31
31
|
size: number;
|
|
32
|
-
conditions: Map<number,
|
|
32
|
+
conditions: Map<number, Uint8Array[]>;
|
|
33
33
|
exists?: {
|
|
34
34
|
prop: PropDef | PropDefEdge;
|
|
35
35
|
negate: boolean;
|
|
@@ -37,7 +37,7 @@ export type QueryDefFilter = {
|
|
|
37
37
|
references?: Map<number, QueryDefFilter>;
|
|
38
38
|
fromRef?: PropDef;
|
|
39
39
|
schema?: SchemaTypeDef;
|
|
40
|
-
edges?: Map<number,
|
|
40
|
+
edges?: Map<number, Uint8Array[]>;
|
|
41
41
|
or?: QueryDefFilter;
|
|
42
42
|
and?: QueryDefFilter;
|
|
43
43
|
};
|
|
@@ -50,6 +50,7 @@ export type QueryDefSearch = {
|
|
|
50
50
|
field: number;
|
|
51
51
|
start: number;
|
|
52
52
|
lang: LangCode;
|
|
53
|
+
typeIndex: number;
|
|
53
54
|
}[];
|
|
54
55
|
} | {
|
|
55
56
|
size: number;
|
|
@@ -69,7 +70,6 @@ export type QueryDefShared = {
|
|
|
69
70
|
filter: QueryDefFilter;
|
|
70
71
|
search: null | QueryDefSearch;
|
|
71
72
|
sort: null | QueryDefSort;
|
|
72
|
-
reverseProps: any;
|
|
73
73
|
skipValidation: boolean;
|
|
74
74
|
range: {
|
|
75
75
|
offset: number;
|
|
@@ -92,6 +92,7 @@ export type QueryDefShared = {
|
|
|
92
92
|
};
|
|
93
93
|
references: Map<number, QueryDef>;
|
|
94
94
|
edges?: QueryDef;
|
|
95
|
+
aggregation: AggFn;
|
|
95
96
|
};
|
|
96
97
|
export type QueryDefEdges = {
|
|
97
98
|
type: QueryDefType.Edge;
|
|
@@ -113,3 +114,24 @@ export type QueryByAliasObj = {
|
|
|
113
114
|
[key: string]: string | QueryByAliasObj;
|
|
114
115
|
};
|
|
115
116
|
export declare const isAlias: (id: QueryByAliasObj | number | Uint32Array | (QueryByAliasObj | number)[]) => id is QueryByAliasObj;
|
|
117
|
+
export declare const READ_ID = 255;
|
|
118
|
+
export declare const READ_EDGE = 252;
|
|
119
|
+
export declare const READ_REFERENCES = 253;
|
|
120
|
+
export declare const READ_REFERENCE = 254;
|
|
121
|
+
export declare const CREATE_AGGREGATION = 250;
|
|
122
|
+
export declare const READ_AGGREGATION = 251;
|
|
123
|
+
export declare const enum AggFn {
|
|
124
|
+
AVG = 1,
|
|
125
|
+
CARDINALITY = 2,
|
|
126
|
+
CONCAT = 3,// string aggregation, delimiter should be an argument
|
|
127
|
+
COUNT = 4,
|
|
128
|
+
MAX = 5,
|
|
129
|
+
MIN = 6,
|
|
130
|
+
MODE = 7,// ordered-set
|
|
131
|
+
PERCENTILE = 8,// continuous or discrete should be optional parameters, default = discrete
|
|
132
|
+
RANK = 9,// hypothetical-set, dense should be optional parameter
|
|
133
|
+
STDDEV = 10,// population or sample should be optional parameters, default = sample
|
|
134
|
+
SUM = 11,
|
|
135
|
+
VARIANCE = 12,
|
|
136
|
+
NONE = -999
|
|
137
|
+
}
|
|
@@ -18,4 +18,10 @@ export const isAlias = (id) => {
|
|
|
18
18
|
!Array.isArray(id) &&
|
|
19
19
|
!ArrayBuffer.isView(id));
|
|
20
20
|
};
|
|
21
|
+
export const READ_ID = 255;
|
|
22
|
+
export const READ_EDGE = 252;
|
|
23
|
+
export const READ_REFERENCES = 253;
|
|
24
|
+
export const READ_REFERENCE = 254;
|
|
25
|
+
export const CREATE_AGGREGATION = 250;
|
|
26
|
+
export const READ_AGGREGATION = 251;
|
|
21
27
|
//# sourceMappingURL=types.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { LangCode } from '@based/schema';
|
|
2
|
-
export declare const write: (buf:
|
|
2
|
+
export declare const write: (buf: Uint8Array, value: string, offset: number, noCompression: boolean, lang?: LangCode) => number;
|
|
3
3
|
export declare const compress: (str: string) => Uint8Array;
|
|
4
4
|
export declare const decompress: (val: Uint8Array) => string;
|
|
5
5
|
export declare const read: (val: Uint8Array, offset: number, len: number) => string;
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import native from '../native.js';
|
|
2
2
|
import { readUint32 } from './bitWise.js';
|
|
3
3
|
import makeTmpBuffer from './tmpBuffer.js';
|
|
4
|
-
|
|
5
|
-
const ENCODER = new TextEncoder();
|
|
4
|
+
import { DECODER, ENCODER } from '../utils.js';
|
|
6
5
|
const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
|
|
7
6
|
// type 0 = no compression; 1 = deflate
|
|
8
7
|
// [lang] [type] [uncompressed size 4] [compressed string] [crc32]
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const xxHash64: (buf:
|
|
1
|
+
export declare const xxHash64: (buf: Uint8Array, target?: Uint8Array, index?: number) => Uint8Array;
|
package/dist/src/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { compress, decompress } from './client/string.js';
|
|
2
|
-
import { ModifyCtx } from './client/
|
|
2
|
+
import { ModifyCtx } from './client/flushModify.js';
|
|
3
3
|
import { DbServer } from './server/index.js';
|
|
4
4
|
import { DbClient } from './client/index.js';
|
|
5
5
|
export * from './client/modify/modify.js';
|
|
@@ -27,14 +27,15 @@ export declare class BasedDb {
|
|
|
27
27
|
upsert: DbClient['upsert'];
|
|
28
28
|
delete: DbClient['delete'];
|
|
29
29
|
expire: DbClient['expire'];
|
|
30
|
-
remove: DbClient['delete'];
|
|
31
30
|
query: DbClient['query'];
|
|
32
|
-
|
|
31
|
+
setSchema: DbClient['setSchema'];
|
|
32
|
+
putSchema: DbClient['setSchema'];
|
|
33
33
|
drain: DbClient['drain'];
|
|
34
34
|
start: DbServer['start'];
|
|
35
35
|
stop: DbServer['stop'];
|
|
36
36
|
save: DbServer['save'];
|
|
37
37
|
migrateSchema: DbServer['migrateSchema'];
|
|
38
|
+
isReady: DbClient['isModified'];
|
|
38
39
|
destroy(): Promise<void>;
|
|
39
40
|
wipe(): Promise<void>;
|
|
40
41
|
}
|
package/dist/src/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { compress, decompress } from './client/string.js';
|
|
2
|
-
import { ModifyCtx } from './client/
|
|
2
|
+
import { ModifyCtx } from './client/flushModify.js';
|
|
3
3
|
import { DbServer } from './server/index.js';
|
|
4
4
|
import { DbClient } from './client/index.js';
|
|
5
5
|
import picocolors from 'picocolors';
|
|
6
|
+
import { wait } from '@saulx/utils';
|
|
6
7
|
export * from './client/modify/modify.js';
|
|
7
8
|
export { compress, decompress };
|
|
8
9
|
export { ModifyCtx }; // TODO move this somewhere
|
|
@@ -45,8 +46,9 @@ export class BasedDb {
|
|
|
45
46
|
const client = new DbClient({
|
|
46
47
|
maxModifySize,
|
|
47
48
|
hooks: {
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
flushTime: 0,
|
|
50
|
+
setSchema(schema, fromStart) {
|
|
51
|
+
return Promise.resolve(server.setSchema(schema, fromStart));
|
|
50
52
|
},
|
|
51
53
|
flushModify(buf) {
|
|
52
54
|
const offsets = server.modify(buf);
|
|
@@ -54,6 +56,8 @@ export class BasedDb {
|
|
|
54
56
|
offsets,
|
|
55
57
|
});
|
|
56
58
|
},
|
|
59
|
+
flushReady: () => { },
|
|
60
|
+
flushIsReady: new Promise(() => { }),
|
|
57
61
|
getQueryBuf(buf) {
|
|
58
62
|
return Promise.resolve(server.getQueryBuf(buf));
|
|
59
63
|
},
|
|
@@ -81,15 +85,15 @@ export class BasedDb {
|
|
|
81
85
|
expire = function () {
|
|
82
86
|
return this.client.expire.apply(this.client, arguments);
|
|
83
87
|
};
|
|
84
|
-
remove = function () {
|
|
85
|
-
console.warn('WARNING: db.remove() is deprecated. Use db.delete() instead');
|
|
86
|
-
return this.client.delete.apply(this.client, arguments);
|
|
87
|
-
};
|
|
88
88
|
query = function () {
|
|
89
89
|
return this.client.query.apply(this.client, arguments);
|
|
90
90
|
};
|
|
91
|
+
setSchema = function () {
|
|
92
|
+
return this.client.setSchema.apply(this.client, arguments);
|
|
93
|
+
};
|
|
91
94
|
putSchema = function () {
|
|
92
|
-
|
|
95
|
+
console.warn('URGENT: putSchema will be removed in next release. Use setSchema instead!');
|
|
96
|
+
return this.setSchema.apply(this, arguments);
|
|
93
97
|
};
|
|
94
98
|
drain = function () {
|
|
95
99
|
return this.client.drain.apply(this.client, arguments);
|
|
@@ -108,10 +112,15 @@ export class BasedDb {
|
|
|
108
112
|
migrateSchema = function () {
|
|
109
113
|
return this.server.migrateSchema.apply(this.server, arguments);
|
|
110
114
|
};
|
|
111
|
-
|
|
112
|
-
|
|
115
|
+
isReady = function () {
|
|
116
|
+
return this.client.isReady.apply(this.client, arguments);
|
|
117
|
+
};
|
|
118
|
+
async destroy() {
|
|
119
|
+
// Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
|
|
120
|
+
// Todo: clear all active queries, queues ETC
|
|
121
|
+
await wait(Math.max(this.client.hooks.flushTime + 10, 10));
|
|
113
122
|
this.client.destroy();
|
|
114
|
-
|
|
123
|
+
await this.server.destroy();
|
|
115
124
|
}
|
|
116
125
|
async wipe() {
|
|
117
126
|
const opts = {
|
package/dist/src/native.d.ts
CHANGED
|
@@ -4,8 +4,8 @@ declare const _default: {
|
|
|
4
4
|
workerCtxInit: () => void;
|
|
5
5
|
externalFromInt(address: BigInt): any;
|
|
6
6
|
intFromExternal(external: any): BigInt;
|
|
7
|
-
modify: (data:
|
|
8
|
-
getQueryBuf: (q:
|
|
7
|
+
modify: (data: Uint8Array, types: Uint8Array, dbCtx: any) => any;
|
|
8
|
+
getQueryBuf: (q: Uint8Array, dbCtx: any) => ArrayBuffer | null;
|
|
9
9
|
start: (id: number) => any;
|
|
10
10
|
stop: (dbCtx: any) => any;
|
|
11
11
|
saveCommon: (path: string, dbCtx: any) => number;
|
|
@@ -15,17 +15,12 @@ declare const _default: {
|
|
|
15
15
|
updateSchemaType: (prefix: number, buf: Uint8Array, dbCtx: any) => any;
|
|
16
16
|
getTypeInfo: (typeId: number, dbCtx: any) => any;
|
|
17
17
|
getNodeRangeHash: (typeId: number, start: number, end: number, bufOut: Uint8Array, dbCtx: any) => any;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
decompress: (input: Buffer | Uint8Array, output: Buffer | Uint8Array, offset: number, len: number) => any;
|
|
25
|
-
crc32: (buf: Buffer | Uint8Array) => any;
|
|
26
|
-
createSortIndex: (buf: Buffer, dbCtx: any) => any;
|
|
27
|
-
destroySortIndex: (buf: Buffer, dbCtx: any) => any;
|
|
28
|
-
xxHash64: (buf: Buffer | Uint8Array, target: Buffer | Uint8Array, index: number) => any;
|
|
18
|
+
compress: (buf: Uint8Array, offset: number, stringSize: number) => any;
|
|
19
|
+
decompress: (input: Uint8Array, output: Uint8Array, offset: number, len: number) => any;
|
|
20
|
+
crc32: (buf: Uint8Array) => any;
|
|
21
|
+
createSortIndex: (buf: Uint8Array, dbCtx: any) => any;
|
|
22
|
+
destroySortIndex: (buf: Uint8Array, dbCtx: any) => any;
|
|
23
|
+
xxHash64: (buf: Uint8Array, target: Uint8Array, index: number) => any;
|
|
29
24
|
base64encode: (dst: Uint8Array, src: Uint8Array, lineMax: number) => Uint8Array;
|
|
30
25
|
equals: (a: Uint8Array, b: Uint8Array) => boolean;
|
|
31
26
|
};
|