@based/db 0.0.28 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -399
- package/dist/lib/darwin_aarch64/include/selva/db.h +8 -4
- package/dist/lib/darwin_aarch64/include/selva/fields.h +10 -2
- package/dist/lib/darwin_aarch64/include/selva/selva_hash128.h +17 -7
- package/dist/lib/darwin_aarch64/include/selva/sort.h +21 -14
- package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
- package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
- package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
- package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
- package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
- package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
- package/dist/lib/linux_aarch64/include/selva/db.h +8 -4
- package/dist/lib/linux_aarch64/include/selva/fields.h +10 -2
- package/dist/lib/linux_aarch64/include/selva/selva_hash128.h +17 -7
- package/dist/lib/linux_aarch64/include/selva/sort.h +21 -14
- package/dist/lib/linux_aarch64/libdeflate.so +0 -0
- package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
- package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
- package/dist/lib/linux_aarch64/libselva.so +0 -0
- package/dist/lib/linux_x86_64/include/selva/db.h +8 -4
- package/dist/lib/linux_x86_64/include/selva/fields.h +10 -2
- package/dist/lib/linux_x86_64/include/selva/selva_hash128.h +17 -7
- package/dist/lib/linux_x86_64/include/selva/sort.h +21 -14
- package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
- package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
- package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
- package/dist/lib/linux_x86_64/libselva.so +0 -0
- package/dist/src/client/flushModify.d.ts +1 -1
- package/dist/src/client/flushModify.js +15 -19
- package/dist/src/client/index.d.ts +8 -10
- package/dist/src/client/index.js +19 -8
- package/dist/src/client/modify/ModifyRes.d.ts +2 -4
- package/dist/src/client/modify/ModifyRes.js +15 -21
- package/dist/src/client/modify/create.js +1 -2
- package/dist/src/client/modify/delete.js +1 -2
- package/dist/src/client/modify/fixed.js +43 -8
- package/dist/src/client/modify/modify.js +0 -5
- package/dist/src/client/modify/references/{appendRefs.d.ts → appendEdgeRefs.d.ts} +1 -1
- package/dist/src/client/modify/references/{appendRefs.js → appendEdgeRefs.js} +5 -2
- package/dist/src/client/modify/references/edge.js +182 -175
- package/dist/src/client/modify/references/reference.js +4 -8
- package/dist/src/client/modify/references/references.js +18 -14
- package/dist/src/client/modify/setCursor.js +1 -1
- package/dist/src/client/modify/string.js +0 -3
- package/dist/src/client/modify/text.js +11 -3
- package/dist/src/client/modify/types.d.ts +11 -0
- package/dist/src/client/modify/types.js +10 -0
- package/dist/src/client/modify/update.js +5 -3
- package/dist/src/client/modify/vector.js +13 -4
- package/dist/src/client/query/BasedDbQuery.d.ts +1 -1
- package/dist/src/client/query/BasedDbQuery.js +2 -2
- package/dist/src/client/query/BasedIterable.d.ts +1 -1
- package/dist/src/client/query/BasedIterable.js +7 -2
- package/dist/src/client/query/filter/createFixedFilterBuffer.d.ts +2 -1
- package/dist/src/client/query/filter/createFixedFilterBuffer.js +11 -28
- package/dist/src/client/query/filter/createReferenceFilter.d.ts +2 -1
- package/dist/src/client/query/filter/createReferenceFilter.js +10 -9
- package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -1
- package/dist/src/client/query/filter/createVariableFilterBuffer.js +8 -10
- package/dist/src/client/query/filter/parseFilterValue.js +1 -1
- package/dist/src/client/query/filter/primitiveFilter.js +9 -9
- package/dist/src/client/query/filter/toBuffer.js +0 -15
- package/dist/src/client/query/filter/types.d.ts +1 -0
- package/dist/src/client/query/filter/types.js +1 -0
- package/dist/src/client/query/include/walk.js +0 -1
- package/dist/src/client/query/read/read.js +16 -7
- package/dist/src/client/query/search/index.js +11 -15
- package/dist/src/client/query/subscription/index.d.ts +1 -2
- package/dist/src/client/query/subscription/index.js +3 -50
- package/dist/src/client/query/subscription/markers.js +1 -2
- package/dist/src/client/query/subscription/run.js +0 -2
- package/dist/src/client/query/subscription/types.d.ts +1 -29
- package/dist/src/client/query/subscription/types.js +8 -1
- package/dist/src/client/query/thresholds.d.ts +0 -2
- package/dist/src/client/query/thresholds.js +0 -2
- package/dist/src/client/query/toBuffer.js +16 -42
- package/dist/src/client/query/types.d.ts +3 -2
- package/dist/src/client/query/validation.d.ts +1 -3
- package/dist/src/client/query/validation.js +6 -18
- package/dist/src/client/string.d.ts +2 -0
- package/dist/src/client/string.js +10 -14
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +11 -15
- package/dist/src/native.d.ts +3 -3
- package/dist/src/native.js +6 -4
- package/dist/src/server/csmt/draw-dot.js +2 -2
- package/dist/src/server/csmt/tree.js +57 -6
- package/dist/src/server/csmt/types.d.ts +5 -0
- package/dist/src/server/index.d.ts +4 -3
- package/dist/src/server/index.js +44 -44
- package/dist/src/server/migrate/index.js +47 -29
- package/dist/src/server/migrate/worker.js +2 -2
- package/dist/src/server/save.js +40 -28
- package/dist/src/server/start.js +7 -19
- package/dist/src/server/tree.d.ts +2 -0
- package/dist/src/server/tree.js +34 -2
- package/dist/src/server/worker.js +3 -3
- package/dist/src/utils.d.ts +3 -1
- package/dist/src/utils.js +43 -19
- package/package.json +9 -3
- package/dist/lib/darwin_aarch64/include/selva/base64.h +0 -59
- package/dist/lib/darwin_aarch64/include/selva/base64url.h +0 -59
- package/dist/lib/linux_aarch64/include/selva/base64.h +0 -59
- package/dist/lib/linux_aarch64/include/selva/base64url.h +0 -59
- package/dist/lib/linux_x86_64/include/selva/base64.h +0 -59
- package/dist/lib/linux_x86_64/include/selva/base64url.h +0 -59
- package/dist/src/client/timestamp.d.ts +0 -1
- package/dist/src/client/timestamp.js +0 -68
|
@@ -20,7 +20,6 @@ export const walkDefs = (db, def, f) => {
|
|
|
20
20
|
def.edges.lang = def.lang;
|
|
21
21
|
}
|
|
22
22
|
const edgeProp = def.edges.props[p];
|
|
23
|
-
// console.log(p, !!edgeProp, Object.keys(def.edges.props), def.target)
|
|
24
23
|
if (edgeProp.typeIndex === REFERENCE ||
|
|
25
24
|
edgeProp.typeIndex === REFERENCES) {
|
|
26
25
|
const refDef = createOrGetRefQueryDef(db, def.edges, edgeProp);
|
|
@@ -135,7 +135,7 @@ const handleUndefinedProps = (id, q, item) => {
|
|
|
135
135
|
if (q.aggregation == 0 /* AggFlag.NONE */ || q.aggregation == null) {
|
|
136
136
|
for (const k in q.include.propsRead) {
|
|
137
137
|
if (q.include.propsRead[k] !== id) {
|
|
138
|
-
// Only
|
|
138
|
+
// Only relevant for seperate props
|
|
139
139
|
const prop = q.schema.reverseProps[k];
|
|
140
140
|
if (prop.typeIndex === CARDINALITY) {
|
|
141
141
|
addField(prop, 0, item);
|
|
@@ -146,7 +146,8 @@ const handleUndefinedProps = (id, q, item) => {
|
|
|
146
146
|
if (lang.has(0)) {
|
|
147
147
|
for (const locale in q.schema.locales) {
|
|
148
148
|
if (!lan[locale]) {
|
|
149
|
-
|
|
149
|
+
// console.log('???', locale, prop.default)
|
|
150
|
+
lan[locale] = prop.default[locale] || '';
|
|
150
151
|
}
|
|
151
152
|
}
|
|
152
153
|
}
|
|
@@ -154,13 +155,24 @@ const handleUndefinedProps = (id, q, item) => {
|
|
|
154
155
|
for (const code of lang) {
|
|
155
156
|
const locale = inverseLangMap.get(code);
|
|
156
157
|
if (!lan[locale]) {
|
|
157
|
-
lan[locale] = '';
|
|
158
|
+
lan[locale] = prop.default[locale] || '';
|
|
158
159
|
}
|
|
159
160
|
}
|
|
160
161
|
}
|
|
161
162
|
}
|
|
163
|
+
else if (prop.typeIndex === BINARY) {
|
|
164
|
+
addField(prop, prop.default, item);
|
|
165
|
+
}
|
|
166
|
+
else if (prop.typeIndex === TEXT) {
|
|
167
|
+
// q.lang
|
|
168
|
+
// console.log('???')
|
|
169
|
+
addField(prop, '', item);
|
|
170
|
+
}
|
|
162
171
|
else {
|
|
163
|
-
|
|
172
|
+
1;
|
|
173
|
+
if (prop.default !== undefined) {
|
|
174
|
+
addField(prop, prop.default, item);
|
|
175
|
+
}
|
|
164
176
|
}
|
|
165
177
|
}
|
|
166
178
|
}
|
|
@@ -314,9 +326,6 @@ export const readAllFields = (q, result, offset, end, item, id) => {
|
|
|
314
326
|
}
|
|
315
327
|
else {
|
|
316
328
|
const prop = q.schema.reverseProps[index];
|
|
317
|
-
if (!prop) {
|
|
318
|
-
console.log({ prop: !!prop }, index);
|
|
319
|
-
}
|
|
320
329
|
if (prop.typeIndex === CARDINALITY) {
|
|
321
330
|
q.include.propsRead[index] = id;
|
|
322
331
|
const size = readUint32(result, i);
|
|
@@ -12,8 +12,7 @@ export const vectorSearch = (def, q, field, opts) => {
|
|
|
12
12
|
if (prop.typeIndex !== VECTOR) {
|
|
13
13
|
searchIncorrectType(def, prop);
|
|
14
14
|
}
|
|
15
|
-
|
|
16
|
-
let size = 9;
|
|
15
|
+
let size = 17;
|
|
17
16
|
const vec = new Uint8Array(q.buffer, 0, q.byteLength);
|
|
18
17
|
size += vec.byteLength;
|
|
19
18
|
def.search = {
|
|
@@ -32,21 +31,17 @@ export const search = (def, q, s) => {
|
|
|
32
31
|
searchIncorrecQueryValue(def, q);
|
|
33
32
|
q = '';
|
|
34
33
|
}
|
|
35
|
-
const x = q
|
|
36
|
-
.toLowerCase()
|
|
37
|
-
.normalize('NFKD')
|
|
38
|
-
.trim()
|
|
39
|
-
.split(' ')
|
|
40
|
-
.map((s) => ` ${s}`);
|
|
34
|
+
const x = q.toLowerCase().normalize('NFKD').trim().split(' ');
|
|
41
35
|
for (const s of x) {
|
|
42
36
|
if (s) {
|
|
43
37
|
const buf = ENCODER.encode(s);
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
38
|
+
const lenBuf = new Uint8Array(2);
|
|
39
|
+
let len = buf.byteLength;
|
|
40
|
+
lenBuf[0] = len;
|
|
41
|
+
lenBuf[1] = len >>> 8;
|
|
42
|
+
bufs.push(lenBuf, buf);
|
|
48
43
|
nrBlocks++;
|
|
49
|
-
totalByteLength +=
|
|
44
|
+
totalByteLength += buf.byteLength + 2;
|
|
50
45
|
}
|
|
51
46
|
}
|
|
52
47
|
bufs.unshift(Uint8Array.from([nrBlocks]));
|
|
@@ -119,7 +114,8 @@ export const searchToBuffer = (search) => {
|
|
|
119
114
|
| 3 | field | 1 | Field identifier |
|
|
120
115
|
| 4 | func | 1 | Function identifier (enum) |
|
|
121
116
|
| 5 | score | 4 | Score value (f32) |
|
|
122
|
-
| 9 |
|
|
117
|
+
| 9 | align | 8 | Space for alignment |
|
|
118
|
+
| 17 | query | queryLen | Query data (array of f32 values) |
|
|
123
119
|
*/
|
|
124
120
|
const result = new Uint8Array(search.size);
|
|
125
121
|
result[0] = 1; // search.isVector 1
|
|
@@ -128,7 +124,7 @@ export const searchToBuffer = (search) => {
|
|
|
128
124
|
result[3] = search.prop;
|
|
129
125
|
result[4] = getVectorFn(search.opts.fn);
|
|
130
126
|
result.set(new Uint8Array(new Float32Array([search.opts.score ?? 0.5]).buffer), 5);
|
|
131
|
-
result.set(search.query,
|
|
127
|
+
result.set(search.query, 17);
|
|
132
128
|
return result;
|
|
133
129
|
}
|
|
134
130
|
else {
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { BasedDbQuery } from '../BasedDbQuery.js';
|
|
2
2
|
import { OnData, OnError, OnClose } from './types.js';
|
|
3
|
+
export declare const subscribe: (q: BasedDbQuery, onData: OnData, onError?: OnError) => OnClose;
|
|
3
4
|
export * from './types.js';
|
|
4
|
-
export * from './markers.js';
|
|
5
|
-
export declare const subscribe: (q: BasedDbQuery, onData: OnData, onError: OnError) => OnClose;
|
|
@@ -1,58 +1,11 @@
|
|
|
1
1
|
import { includeField } from '../query.js';
|
|
2
2
|
import { registerQuery } from '../registerQuery.js';
|
|
3
|
-
import { runSubscription } from './run.js';
|
|
4
|
-
import { addSubscriptionMarkers, deleteSubscriptionMarkers } from './markers.js';
|
|
5
|
-
export * from './types.js';
|
|
6
|
-
export * from './markers.js';
|
|
7
3
|
export const subscribe = (q, onData, onError) => {
|
|
8
|
-
let closed = false;
|
|
9
4
|
if (!q.def.include.stringFields.size && !q.def.references.size) {
|
|
10
5
|
includeField(q.def, '*');
|
|
11
6
|
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
}
|
|
15
|
-
catch (err) {
|
|
16
|
-
onError(err);
|
|
17
|
-
return () => q;
|
|
18
|
-
}
|
|
19
|
-
if (!q.db.subscriptions.has(q.id)) {
|
|
20
|
-
const subscription = {
|
|
21
|
-
query: q,
|
|
22
|
-
subs: new Set(),
|
|
23
|
-
inProgress: false,
|
|
24
|
-
closed: false,
|
|
25
|
-
};
|
|
26
|
-
q.db.subscriptions.set(q.id, subscription);
|
|
27
|
-
addSubscriptionMarkers(q, subscription);
|
|
28
|
-
}
|
|
29
|
-
const fn = (res, err) => {
|
|
30
|
-
if (!closed) {
|
|
31
|
-
if (err) {
|
|
32
|
-
onError(err);
|
|
33
|
-
}
|
|
34
|
-
else {
|
|
35
|
-
onData(res);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
};
|
|
39
|
-
const sub = q.db.subscriptions.get(q.id);
|
|
40
|
-
const close = () => {
|
|
41
|
-
sub.subs.delete(fn);
|
|
42
|
-
if (sub.subs.size === 0) {
|
|
43
|
-
q.db.subscriptions.delete(q.id);
|
|
44
|
-
deleteSubscriptionMarkers(q);
|
|
45
|
-
}
|
|
46
|
-
closed = true;
|
|
47
|
-
return q;
|
|
48
|
-
};
|
|
49
|
-
sub.subs.add(fn);
|
|
50
|
-
if (sub.res) {
|
|
51
|
-
onData(sub.res);
|
|
52
|
-
}
|
|
53
|
-
if (!sub.inProgress) {
|
|
54
|
-
runSubscription(sub);
|
|
55
|
-
}
|
|
56
|
-
return close;
|
|
7
|
+
registerQuery(q);
|
|
8
|
+
return q.db.hooks.subscribe(q, onData, onError);
|
|
57
9
|
};
|
|
10
|
+
export * from './types.js';
|
|
58
11
|
//# sourceMappingURL=index.js.map
|
|
@@ -95,7 +95,7 @@ const getFilterFields = (filter, results = {
|
|
|
95
95
|
conditions.forEach((v, k) => {
|
|
96
96
|
if (k === 0) {
|
|
97
97
|
for (const buf of v) {
|
|
98
|
-
const x = buf[4] | buf[5] << 8;
|
|
98
|
+
const x = buf[4] | (buf[5] << 8);
|
|
99
99
|
results.main.add(x);
|
|
100
100
|
}
|
|
101
101
|
}
|
|
@@ -209,6 +209,5 @@ export const resetSubscriptionMarkers = (db) => {
|
|
|
209
209
|
};
|
|
210
210
|
export const deleteSubscriptionMarkers = (q) => {
|
|
211
211
|
// derp
|
|
212
|
-
console.log('delete markers!');
|
|
213
212
|
};
|
|
214
213
|
//# sourceMappingURL=markers.js.map
|
|
@@ -61,11 +61,9 @@ export const runSubscription = (subscription) => {
|
|
|
61
61
|
}
|
|
62
62
|
};
|
|
63
63
|
export const startSubscription = (db) => {
|
|
64
|
-
console.log('start subscription');
|
|
65
64
|
if (!db.subscriptionsInProgress) {
|
|
66
65
|
db.subscriptionsInProgress = true;
|
|
67
66
|
setTimeout(() => {
|
|
68
|
-
console.log('run it');
|
|
69
67
|
db.subscriptionsToRun.forEach((s) => {
|
|
70
68
|
runSubscription(s);
|
|
71
69
|
});
|
|
@@ -1,33 +1,5 @@
|
|
|
1
1
|
import { BasedQueryResponse } from '../BasedIterable.js';
|
|
2
|
-
import { BasedDbQuery } from '../BasedDbQuery.js';
|
|
3
2
|
export type OnData = (res: BasedQueryResponse) => any;
|
|
4
3
|
export type OnError = (err: Error) => any;
|
|
5
|
-
export type OnClose = () =>
|
|
4
|
+
export type OnClose = () => any;
|
|
6
5
|
export type OnSubscription = (res: any, err?: Error) => void;
|
|
7
|
-
export type Subscription = {
|
|
8
|
-
query: BasedDbQuery;
|
|
9
|
-
subs: Set<OnSubscription>;
|
|
10
|
-
res?: BasedQueryResponse;
|
|
11
|
-
closed: boolean;
|
|
12
|
-
inProgress: boolean;
|
|
13
|
-
};
|
|
14
|
-
export type SubscriptionsMap = Map<number, Subscription>;
|
|
15
|
-
export type SubscriptionsToRun = Subscription[];
|
|
16
|
-
export type SubscriptionMarkers = {
|
|
17
|
-
main: {
|
|
18
|
-
[start: string]: Subscription[];
|
|
19
|
-
};
|
|
20
|
-
props: {
|
|
21
|
-
[prop: string]: Subscription[];
|
|
22
|
-
};
|
|
23
|
-
};
|
|
24
|
-
export type SubscriptionMarkerMap = {
|
|
25
|
-
[typeID: string]: {
|
|
26
|
-
ids: Map<number, SubscriptionMarkers>;
|
|
27
|
-
collection: SubscriptionMarkers;
|
|
28
|
-
};
|
|
29
|
-
};
|
|
30
|
-
export type SubscriptionMarkersCheck = {
|
|
31
|
-
collection: SubscriptionMarkers | false;
|
|
32
|
-
ids: SubscriptionMarkers | false;
|
|
33
|
-
};
|
|
@@ -1,3 +1,10 @@
|
|
|
1
1
|
export {};
|
|
2
|
-
//
|
|
2
|
+
// export type Subscription = {
|
|
3
|
+
// query: BasedDbQuery
|
|
4
|
+
// subs: Set<OnSubscription>
|
|
5
|
+
// res?: BasedQueryResponse
|
|
6
|
+
// closed: boolean
|
|
7
|
+
// inProgress: boolean // dont need to check
|
|
8
|
+
// // filter - realy nice to add
|
|
9
|
+
// }
|
|
3
10
|
//# sourceMappingURL=types.js.map
|
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
export declare const MAX_IDS_PER_QUERY = 1000000;
|
|
2
2
|
export declare const DEF_RANGE_PROP_LIMIT = 1000;
|
|
3
3
|
export declare const DEF_RANGE_REF_LIMIT = 10000;
|
|
4
|
-
export declare const MAX_ID = 4294967295;
|
|
5
4
|
export declare const MIN_ID_VALUE = 1;
|
|
6
|
-
export declare const MAX_ID_VALUE: number;
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
export const MAX_IDS_PER_QUERY = 1e6; // Max number of IDs that can be queried at once
|
|
3
3
|
export const DEF_RANGE_PROP_LIMIT = 1e3;
|
|
4
4
|
export const DEF_RANGE_REF_LIMIT = 1e4;
|
|
5
|
-
export const MAX_ID = 4294967295;
|
|
6
5
|
// Data validation thresholds
|
|
7
6
|
export const MIN_ID_VALUE = 1; // Minimum allowed value for an ID (id array stared)
|
|
8
|
-
export const MAX_ID_VALUE = 2 ** 32 - 1; /// Maximum allowed value for an ID (Uint32)
|
|
9
7
|
//# sourceMappingURL=thresholds.js.map
|
|
@@ -13,6 +13,7 @@ const byteSize = (arr) => {
|
|
|
13
13
|
export function defToBuffer(db, def) {
|
|
14
14
|
const result = [];
|
|
15
15
|
const include = includeToBuffer(db, def);
|
|
16
|
+
// ---------------------------------------
|
|
16
17
|
def.references.forEach((ref) => {
|
|
17
18
|
include.push(...defToBuffer(db, ref));
|
|
18
19
|
if (ref.errors) {
|
|
@@ -32,27 +33,21 @@ export function defToBuffer(db, def) {
|
|
|
32
33
|
edgesSize = byteSize(edges);
|
|
33
34
|
}
|
|
34
35
|
const size = (edges ? edgesSize + 3 : 0) + byteSize(include);
|
|
36
|
+
// ---------------------------------------
|
|
37
|
+
// move down and will handle size after store the size Var
|
|
38
|
+
// only for references | edges
|
|
35
39
|
if (def.type === QueryDefType.Root) {
|
|
36
|
-
let filter;
|
|
37
|
-
let filterSize = 0;
|
|
38
40
|
let search;
|
|
39
41
|
let searchSize = 0;
|
|
42
|
+
const filterSize = def.filter.size || 0;
|
|
40
43
|
if (def.search) {
|
|
41
44
|
search = searchToBuffer(def.search);
|
|
42
45
|
searchSize = def.search.size;
|
|
43
46
|
}
|
|
44
|
-
if (def.filter.size) {
|
|
45
|
-
filter = filterToBuffer(def.filter);
|
|
46
|
-
filterSize = filter.byteLength;
|
|
47
|
-
}
|
|
48
|
-
// [type,type]
|
|
49
|
-
// [q type] 0 == id, 1 === ids, 2 === type only
|
|
50
47
|
if (def.target.resolvedAlias) {
|
|
51
|
-
// put this somehwere else at some point
|
|
52
48
|
const alias = def.target.resolvedAlias;
|
|
53
49
|
const aliasStr = ENCODER.encode(alias.value);
|
|
54
50
|
const aliasLen = aliasStr.byteLength;
|
|
55
|
-
// filter is nice for things like access
|
|
56
51
|
const buf = new Uint8Array(8 + filterSize + aliasLen);
|
|
57
52
|
buf[0] = 3;
|
|
58
53
|
buf[1] = def.schema.idUint8[0];
|
|
@@ -64,14 +59,11 @@ export function defToBuffer(db, def) {
|
|
|
64
59
|
buf[6 + aliasLen] = filterSize;
|
|
65
60
|
buf[7 + aliasLen] = filterSize >>> 8;
|
|
66
61
|
if (filterSize) {
|
|
67
|
-
buf.set(filter, 8 + aliasLen);
|
|
62
|
+
buf.set(filterToBuffer(def.filter), 8 + aliasLen);
|
|
68
63
|
}
|
|
69
64
|
result.push(buf);
|
|
70
65
|
}
|
|
71
66
|
else if (def.target.id) {
|
|
72
|
-
// type 0
|
|
73
|
-
// 0: 4 [id]
|
|
74
|
-
// 0: 2 [filterSize]
|
|
75
67
|
const buf = new Uint8Array(9 + filterSize);
|
|
76
68
|
buf[0] = 0;
|
|
77
69
|
buf[1] = def.schema.idUint8[0];
|
|
@@ -83,7 +75,7 @@ export function defToBuffer(db, def) {
|
|
|
83
75
|
buf[7] = filterSize;
|
|
84
76
|
buf[8] = filterSize >>> 8;
|
|
85
77
|
if (filterSize) {
|
|
86
|
-
buf.set(filter, 9);
|
|
78
|
+
buf.set(filterToBuffer(def.filter), 9);
|
|
87
79
|
}
|
|
88
80
|
result.push(buf);
|
|
89
81
|
}
|
|
@@ -95,15 +87,10 @@ export function defToBuffer(db, def) {
|
|
|
95
87
|
sortSize = sort.byteLength;
|
|
96
88
|
}
|
|
97
89
|
let aggregation;
|
|
98
|
-
|
|
90
|
+
if (def.aggregation) {
|
|
91
|
+
aggregation = createAggFlagBuffer(def.aggregation);
|
|
92
|
+
}
|
|
99
93
|
if (def.target.ids) {
|
|
100
|
-
// type 1
|
|
101
|
-
// 1: 4 + ids * 4 [ids len] [id,id,id]
|
|
102
|
-
// 1: 8 [offset, limit]
|
|
103
|
-
// 1: 2 [filter size]
|
|
104
|
-
// ?filter
|
|
105
|
-
// 1: 2 [sort size]
|
|
106
|
-
// ?sort
|
|
107
94
|
if (!sortSize &&
|
|
108
95
|
(def.range.offset || def.range.limit < def.target.ids.length)) {
|
|
109
96
|
def.target.ids = def.target.ids.slice(def.range.offset, def.range.offset + def.range.limit);
|
|
@@ -129,7 +116,8 @@ export function defToBuffer(db, def) {
|
|
|
129
116
|
buf[idsSize + 15] = filterSize;
|
|
130
117
|
buf[idsSize + 16] = filterSize >>> 8;
|
|
131
118
|
if (filterSize) {
|
|
132
|
-
|
|
119
|
+
// just write it here dont do set
|
|
120
|
+
buf.set(filterToBuffer(def.filter), idsSize + 17);
|
|
133
121
|
}
|
|
134
122
|
buf[17 + filterSize + idsSize] = sortSize;
|
|
135
123
|
buf[18 + filterSize + idsSize] = sortSize >>> 8;
|
|
@@ -145,13 +133,6 @@ export function defToBuffer(db, def) {
|
|
|
145
133
|
result.push(buf);
|
|
146
134
|
}
|
|
147
135
|
else {
|
|
148
|
-
// type 2
|
|
149
|
-
// 2: 0
|
|
150
|
-
// 2: 8 [offset, limit]
|
|
151
|
-
// 2: 2 [filter size]
|
|
152
|
-
// ?filter
|
|
153
|
-
// 2: 2 [sort size]
|
|
154
|
-
// ?sort
|
|
155
136
|
const buf = new Uint8Array(18 + filterSize + sortSize + searchSize);
|
|
156
137
|
buf[0] = 2;
|
|
157
138
|
buf[1] = def.schema.idUint8[0];
|
|
@@ -167,7 +148,7 @@ export function defToBuffer(db, def) {
|
|
|
167
148
|
buf[11] = filterSize;
|
|
168
149
|
buf[12] = filterSize >>> 8;
|
|
169
150
|
if (filterSize) {
|
|
170
|
-
buf.set(filter, 13);
|
|
151
|
+
buf.set(filterToBuffer(def.filter), 13);
|
|
171
152
|
}
|
|
172
153
|
buf[13 + filterSize] = sortSize;
|
|
173
154
|
buf[14 + filterSize] = sortSize >>> 8;
|
|
@@ -187,19 +168,12 @@ export function defToBuffer(db, def) {
|
|
|
187
168
|
}
|
|
188
169
|
}
|
|
189
170
|
else if (def.type === QueryDefType.References) {
|
|
190
|
-
|
|
191
|
-
let filter;
|
|
192
|
-
if (def.filter.size) {
|
|
193
|
-
filter = filterToBuffer(def.filter);
|
|
194
|
-
}
|
|
171
|
+
const filterSize = def.filter.size || 0;
|
|
195
172
|
let sort;
|
|
196
173
|
if (def.sort) {
|
|
197
174
|
sort = createSortBuffer(def.sort);
|
|
198
175
|
}
|
|
199
|
-
// ADD IT
|
|
200
|
-
// TODO: ADD RANGE [offset,limit] (8 bytes)
|
|
201
176
|
const sortSize = sort?.byteLength ?? 0;
|
|
202
|
-
const filterSize = filter?.byteLength ?? 0;
|
|
203
177
|
const modsSize = filterSize + sortSize;
|
|
204
178
|
const meta = new Uint8Array(modsSize + 10 + 8);
|
|
205
179
|
const sz = size + 7 + modsSize + 8;
|
|
@@ -218,8 +192,8 @@ export function defToBuffer(db, def) {
|
|
|
218
192
|
meta[12] = def.range.limit >>> 8;
|
|
219
193
|
meta[13] = def.range.limit >>> 16;
|
|
220
194
|
meta[14] = def.range.limit >>> 24;
|
|
221
|
-
if (
|
|
222
|
-
meta.set(filter, 15);
|
|
195
|
+
if (filterSize) {
|
|
196
|
+
meta.set(filterToBuffer(def.filter), 15);
|
|
223
197
|
}
|
|
224
198
|
if (sort) {
|
|
225
199
|
meta.set(sort, 15 + filterSize);
|
|
@@ -27,9 +27,10 @@ export type Target = {
|
|
|
27
27
|
};
|
|
28
28
|
};
|
|
29
29
|
export declare const isRefDef: (def: QueryDef) => def is QueryDefRest;
|
|
30
|
+
export type FilterCondition = Uint8Array;
|
|
30
31
|
export type QueryDefFilter = {
|
|
31
32
|
size: number;
|
|
32
|
-
conditions: Map<number,
|
|
33
|
+
conditions: Map<number, FilterCondition[]>;
|
|
33
34
|
exists?: {
|
|
34
35
|
prop: PropDef | PropDefEdge;
|
|
35
36
|
negate: boolean;
|
|
@@ -37,7 +38,7 @@ export type QueryDefFilter = {
|
|
|
37
38
|
references?: Map<number, QueryDefFilter>;
|
|
38
39
|
fromRef?: PropDef;
|
|
39
40
|
schema?: SchemaTypeDef;
|
|
40
|
-
edges?: Map<number,
|
|
41
|
+
edges?: Map<number, FilterCondition[]>;
|
|
41
42
|
or?: QueryDefFilter;
|
|
42
43
|
and?: QueryDefFilter;
|
|
43
44
|
};
|
|
@@ -61,9 +61,7 @@ export declare const searchIncorrecQueryValue: (def: QueryDef, payload: any) =>
|
|
|
61
61
|
export declare const searchIncorrectType: (def: QueryDef, payload: PropDef | PropDefEdge) => void;
|
|
62
62
|
export declare const searchDoesNotExist: (def: QueryDef, field: string, isVector: boolean) => PropDef;
|
|
63
63
|
export declare const validateRange: (def: QueryDef, offset: number, limit: number) => boolean;
|
|
64
|
-
export declare const
|
|
65
|
-
export declare const isValidString: (v: any) => boolean;
|
|
66
|
-
export declare const validateVal: (def: QueryDef, f: Filter, validate: (v: any) => boolean) => boolean;
|
|
64
|
+
export declare const validateVal: (def: QueryDef, f: Filter, validate: (v: any, p: PropDef | PropDefEdge) => boolean) => boolean;
|
|
67
65
|
export declare const validateFilter: (def: QueryDef, prop: PropDef | PropDefEdge, f: Filter) => boolean;
|
|
68
66
|
export declare const validateType: (db: DbClient, def: QueryDef, type: string) => SchemaTypeDef;
|
|
69
67
|
export declare const filterOperatorDoesNotExist: (def: QueryDef, field: string) => void;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import picocolors from 'picocolors';
|
|
2
2
|
import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, DEFAULT_MAP, } from '@based/schema/def';
|
|
3
3
|
import { EQUAL, EXISTS, HAS, isNumerical, LIKE, operatorReverseMap, VECTOR_FNS, } from './filter/types.js';
|
|
4
|
-
import {
|
|
4
|
+
import { MAX_IDS_PER_QUERY } from './thresholds.js';
|
|
5
5
|
import { displayTarget, safeStringify } from './display.js';
|
|
6
|
-
import { langCodesMap } from '@based/schema';
|
|
6
|
+
import { isValidId, isValidString, langCodesMap, MAX_ID, } from '@based/schema';
|
|
7
7
|
export const ERR_TARGET_INVAL_TYPE = 1;
|
|
8
8
|
export const ERR_TARGET_INVAL_ALIAS = 2;
|
|
9
9
|
export const ERR_TARGET_EXCEED_MAX_IDS = 3;
|
|
@@ -85,18 +85,6 @@ export const validateRange = (def, offset, limit) => {
|
|
|
85
85
|
}
|
|
86
86
|
return r;
|
|
87
87
|
};
|
|
88
|
-
export const isValidId = (id) => {
|
|
89
|
-
if (typeof id != 'number' || id < MIN_ID_VALUE || id > MAX_ID_VALUE) {
|
|
90
|
-
return false;
|
|
91
|
-
}
|
|
92
|
-
return true;
|
|
93
|
-
};
|
|
94
|
-
export const isValidString = (v) => {
|
|
95
|
-
const isVal = typeof v === 'string' ||
|
|
96
|
-
v instanceof Uint8Array ||
|
|
97
|
-
ArrayBuffer.isView(v);
|
|
98
|
-
return isVal;
|
|
99
|
-
};
|
|
100
88
|
export const validateVal = (def, f, validate) => {
|
|
101
89
|
if (def.skipValidation) {
|
|
102
90
|
return false;
|
|
@@ -104,13 +92,13 @@ export const validateVal = (def, f, validate) => {
|
|
|
104
92
|
const value = f[2];
|
|
105
93
|
if (Array.isArray(value)) {
|
|
106
94
|
for (const v of value) {
|
|
107
|
-
if (!validate(v)) {
|
|
95
|
+
if (!validate(v, f[2])) {
|
|
108
96
|
def.errors.push({ code: ERR_FILTER_INVALID_VAL, payload: f });
|
|
109
97
|
return true;
|
|
110
98
|
}
|
|
111
99
|
}
|
|
112
100
|
}
|
|
113
|
-
else if (!validate(value)) {
|
|
101
|
+
else if (!validate(value, f[2])) {
|
|
114
102
|
def.errors.push({
|
|
115
103
|
code: ERR_FILTER_INVALID_VAL,
|
|
116
104
|
payload: f,
|
|
@@ -147,7 +135,7 @@ export const validateFilter = (def, prop, f) => {
|
|
|
147
135
|
});
|
|
148
136
|
return true;
|
|
149
137
|
}
|
|
150
|
-
if (validateVal(def, f,
|
|
138
|
+
if (validateVal(def, f, prop.validation)) {
|
|
151
139
|
return true;
|
|
152
140
|
}
|
|
153
141
|
}
|
|
@@ -287,7 +275,7 @@ export const validateSort = (def, field, orderInput) => {
|
|
|
287
275
|
}
|
|
288
276
|
}
|
|
289
277
|
const type = propDef.typeIndex;
|
|
290
|
-
if (type === REFERENCES || type === REFERENCE) {
|
|
278
|
+
if (type === REFERENCES || type === REFERENCE || type === VECTOR) {
|
|
291
279
|
def.errors.push({
|
|
292
280
|
code: ERR_SORT_TYPE,
|
|
293
281
|
payload: propDef,
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { LangCode } from '@based/schema';
|
|
2
|
+
export declare const COMPRESSED = 1;
|
|
3
|
+
export declare const NOT_COMPRESSED = 0;
|
|
2
4
|
export declare const write: (buf: Uint8Array, value: string, offset: number, noCompression: boolean, lang?: LangCode) => number;
|
|
3
5
|
export declare const compress: (str: string) => Uint8Array;
|
|
4
6
|
export declare const decompress: (val: Uint8Array) => string;
|
|
@@ -3,23 +3,19 @@ import { readUint32 } from './../utils.js';
|
|
|
3
3
|
import makeTmpBuffer from './tmpBuffer.js';
|
|
4
4
|
import { DECODER, ENCODER } from '../utils.js';
|
|
5
5
|
const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
// var cnt = 0
|
|
9
|
-
// var bytesSaved = 0
|
|
10
|
-
// make this into a package
|
|
11
|
-
// write the type Byte
|
|
6
|
+
export const COMPRESSED = 1;
|
|
7
|
+
export const NOT_COMPRESSED = 0;
|
|
12
8
|
export const write = (buf, value, offset, noCompression, lang) => {
|
|
13
9
|
value = value.normalize('NFKD');
|
|
14
10
|
buf[offset] = lang || 0;
|
|
15
11
|
const { written: l } = ENCODER.encodeInto(value, buf.subarray(offset + 2));
|
|
16
12
|
let crc = native.crc32(buf.subarray(offset + 2, offset + 2 + l));
|
|
17
|
-
// 50 maybe if lvl 1
|
|
13
|
+
// 50 len maybe if lvl 1
|
|
18
14
|
if (value.length > 200 && !noCompression) {
|
|
19
15
|
buf.copyWithin(offset + 6 + l, offset + 2, offset + 2 + l);
|
|
20
16
|
const size = native.compress(buf, offset + 6, l);
|
|
21
17
|
if (size === 0) {
|
|
22
|
-
buf[offset + 1] =
|
|
18
|
+
buf[offset + 1] = NOT_COMPRESSED;
|
|
23
19
|
ENCODER.encodeInto(value, buf.subarray(offset + 2));
|
|
24
20
|
buf[offset + l + 2] = crc;
|
|
25
21
|
buf[offset + l + 3] = crc >>>= 8;
|
|
@@ -29,7 +25,7 @@ export const write = (buf, value, offset, noCompression, lang) => {
|
|
|
29
25
|
}
|
|
30
26
|
else {
|
|
31
27
|
let len = l;
|
|
32
|
-
buf[offset + 1] =
|
|
28
|
+
buf[offset + 1] = COMPRESSED;
|
|
33
29
|
buf[offset + 2] = len;
|
|
34
30
|
buf[offset + 3] = len >>>= 8;
|
|
35
31
|
buf[offset + 4] = len >>>= 8;
|
|
@@ -38,11 +34,11 @@ export const write = (buf, value, offset, noCompression, lang) => {
|
|
|
38
34
|
buf[offset + size + 7] = crc >>>= 8;
|
|
39
35
|
buf[offset + size + 8] = crc >>>= 8;
|
|
40
36
|
buf[offset + size + 9] = crc >>>= 8;
|
|
41
|
-
return size + 10;
|
|
37
|
+
return size + 10;
|
|
42
38
|
}
|
|
43
39
|
}
|
|
44
40
|
else {
|
|
45
|
-
buf[offset + 1] =
|
|
41
|
+
buf[offset + 1] = NOT_COMPRESSED;
|
|
46
42
|
buf[offset + l + 2] = crc;
|
|
47
43
|
buf[offset + l + 3] = crc >>>= 8;
|
|
48
44
|
buf[offset + l + 4] = crc >>>= 8;
|
|
@@ -63,14 +59,14 @@ export const decompress = (val) => {
|
|
|
63
59
|
};
|
|
64
60
|
export const read = (val, offset, len) => {
|
|
65
61
|
const type = val[offset + 1];
|
|
66
|
-
if (type ==
|
|
62
|
+
if (type == COMPRESSED) {
|
|
67
63
|
const origSize = readUint32(val, offset + 2);
|
|
68
64
|
const newBuffer = getTmpBuffer(origSize);
|
|
69
|
-
//
|
|
65
|
+
// Browser fallback required for this
|
|
70
66
|
native.decompress(val, newBuffer, offset + 6, len - 6);
|
|
71
67
|
return DECODER.decode(newBuffer);
|
|
72
68
|
}
|
|
73
|
-
else if (type ==
|
|
69
|
+
else if (type == NOT_COMPRESSED) {
|
|
74
70
|
return DECODER.decode(val.subarray(offset + 2, len + offset - 4));
|
|
75
71
|
}
|
|
76
72
|
return '';
|