@based/db 0.0.28 → 0.0.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/README.md +1 -399
  2. package/dist/lib/darwin_aarch64/include/selva/db.h +8 -4
  3. package/dist/lib/darwin_aarch64/include/selva/fields.h +10 -2
  4. package/dist/lib/darwin_aarch64/include/selva/selva_hash128.h +17 -7
  5. package/dist/lib/darwin_aarch64/include/selva/sort.h +21 -14
  6. package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
  7. package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
  8. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  9. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  10. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  11. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  12. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  13. package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
  14. package/dist/lib/linux_aarch64/include/selva/db.h +8 -4
  15. package/dist/lib/linux_aarch64/include/selva/fields.h +10 -2
  16. package/dist/lib/linux_aarch64/include/selva/selva_hash128.h +17 -7
  17. package/dist/lib/linux_aarch64/include/selva/sort.h +21 -14
  18. package/dist/lib/linux_aarch64/libdeflate.so +0 -0
  19. package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
  20. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  21. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  22. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  23. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  24. package/dist/lib/linux_aarch64/libselva.so +0 -0
  25. package/dist/lib/linux_x86_64/include/selva/db.h +8 -4
  26. package/dist/lib/linux_x86_64/include/selva/fields.h +10 -2
  27. package/dist/lib/linux_x86_64/include/selva/selva_hash128.h +17 -7
  28. package/dist/lib/linux_x86_64/include/selva/sort.h +21 -14
  29. package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
  30. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  31. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  32. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  33. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  34. package/dist/lib/linux_x86_64/libselva.so +0 -0
  35. package/dist/src/client/flushModify.d.ts +1 -1
  36. package/dist/src/client/flushModify.js +15 -19
  37. package/dist/src/client/index.d.ts +8 -10
  38. package/dist/src/client/index.js +19 -8
  39. package/dist/src/client/modify/ModifyRes.d.ts +2 -4
  40. package/dist/src/client/modify/ModifyRes.js +15 -21
  41. package/dist/src/client/modify/create.js +1 -2
  42. package/dist/src/client/modify/delete.js +1 -2
  43. package/dist/src/client/modify/fixed.js +43 -8
  44. package/dist/src/client/modify/modify.js +0 -5
  45. package/dist/src/client/modify/references/{appendRefs.d.ts → appendEdgeRefs.d.ts} +1 -1
  46. package/dist/src/client/modify/references/{appendRefs.js → appendEdgeRefs.js} +5 -2
  47. package/dist/src/client/modify/references/edge.js +182 -175
  48. package/dist/src/client/modify/references/reference.js +4 -8
  49. package/dist/src/client/modify/references/references.js +18 -14
  50. package/dist/src/client/modify/setCursor.js +1 -1
  51. package/dist/src/client/modify/string.js +0 -3
  52. package/dist/src/client/modify/text.js +11 -3
  53. package/dist/src/client/modify/types.d.ts +11 -0
  54. package/dist/src/client/modify/types.js +10 -0
  55. package/dist/src/client/modify/update.js +5 -3
  56. package/dist/src/client/modify/vector.js +13 -4
  57. package/dist/src/client/query/BasedDbQuery.d.ts +1 -1
  58. package/dist/src/client/query/BasedDbQuery.js +2 -2
  59. package/dist/src/client/query/BasedIterable.d.ts +1 -1
  60. package/dist/src/client/query/BasedIterable.js +7 -2
  61. package/dist/src/client/query/filter/createFixedFilterBuffer.d.ts +2 -1
  62. package/dist/src/client/query/filter/createFixedFilterBuffer.js +11 -28
  63. package/dist/src/client/query/filter/createReferenceFilter.d.ts +2 -1
  64. package/dist/src/client/query/filter/createReferenceFilter.js +10 -9
  65. package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -1
  66. package/dist/src/client/query/filter/createVariableFilterBuffer.js +8 -10
  67. package/dist/src/client/query/filter/parseFilterValue.js +1 -1
  68. package/dist/src/client/query/filter/primitiveFilter.js +9 -9
  69. package/dist/src/client/query/filter/toBuffer.js +0 -15
  70. package/dist/src/client/query/filter/types.d.ts +1 -0
  71. package/dist/src/client/query/filter/types.js +1 -0
  72. package/dist/src/client/query/include/walk.js +0 -1
  73. package/dist/src/client/query/read/read.js +16 -7
  74. package/dist/src/client/query/search/index.js +11 -15
  75. package/dist/src/client/query/subscription/index.d.ts +1 -2
  76. package/dist/src/client/query/subscription/index.js +3 -50
  77. package/dist/src/client/query/subscription/markers.js +1 -2
  78. package/dist/src/client/query/subscription/run.js +0 -2
  79. package/dist/src/client/query/subscription/types.d.ts +1 -29
  80. package/dist/src/client/query/subscription/types.js +8 -1
  81. package/dist/src/client/query/thresholds.d.ts +0 -2
  82. package/dist/src/client/query/thresholds.js +0 -2
  83. package/dist/src/client/query/toBuffer.js +16 -42
  84. package/dist/src/client/query/types.d.ts +3 -2
  85. package/dist/src/client/query/validation.d.ts +1 -3
  86. package/dist/src/client/query/validation.js +6 -18
  87. package/dist/src/client/string.d.ts +2 -0
  88. package/dist/src/client/string.js +10 -14
  89. package/dist/src/index.d.ts +1 -1
  90. package/dist/src/index.js +11 -15
  91. package/dist/src/native.d.ts +3 -3
  92. package/dist/src/native.js +6 -4
  93. package/dist/src/server/csmt/draw-dot.js +2 -2
  94. package/dist/src/server/csmt/tree.js +57 -6
  95. package/dist/src/server/csmt/types.d.ts +5 -0
  96. package/dist/src/server/index.d.ts +4 -3
  97. package/dist/src/server/index.js +44 -44
  98. package/dist/src/server/migrate/index.js +47 -29
  99. package/dist/src/server/migrate/worker.js +2 -2
  100. package/dist/src/server/save.js +40 -28
  101. package/dist/src/server/start.js +7 -19
  102. package/dist/src/server/tree.d.ts +2 -0
  103. package/dist/src/server/tree.js +34 -2
  104. package/dist/src/server/worker.js +3 -3
  105. package/dist/src/utils.d.ts +3 -1
  106. package/dist/src/utils.js +43 -19
  107. package/package.json +9 -3
  108. package/dist/lib/darwin_aarch64/include/selva/base64.h +0 -59
  109. package/dist/lib/darwin_aarch64/include/selva/base64url.h +0 -59
  110. package/dist/lib/linux_aarch64/include/selva/base64.h +0 -59
  111. package/dist/lib/linux_aarch64/include/selva/base64url.h +0 -59
  112. package/dist/lib/linux_x86_64/include/selva/base64.h +0 -59
  113. package/dist/lib/linux_x86_64/include/selva/base64url.h +0 -59
  114. package/dist/src/client/timestamp.d.ts +0 -1
  115. package/dist/src/client/timestamp.js +0 -68
@@ -20,7 +20,6 @@ export const walkDefs = (db, def, f) => {
20
20
  def.edges.lang = def.lang;
21
21
  }
22
22
  const edgeProp = def.edges.props[p];
23
- // console.log(p, !!edgeProp, Object.keys(def.edges.props), def.target)
24
23
  if (edgeProp.typeIndex === REFERENCE ||
25
24
  edgeProp.typeIndex === REFERENCES) {
26
25
  const refDef = createOrGetRefQueryDef(db, def.edges, edgeProp);
@@ -135,7 +135,7 @@ const handleUndefinedProps = (id, q, item) => {
135
135
  if (q.aggregation == 0 /* AggFlag.NONE */ || q.aggregation == null) {
136
136
  for (const k in q.include.propsRead) {
137
137
  if (q.include.propsRead[k] !== id) {
138
- // Only relvant for seperate props
138
+ // Only relevant for seperate props
139
139
  const prop = q.schema.reverseProps[k];
140
140
  if (prop.typeIndex === CARDINALITY) {
141
141
  addField(prop, 0, item);
@@ -146,7 +146,8 @@ const handleUndefinedProps = (id, q, item) => {
146
146
  if (lang.has(0)) {
147
147
  for (const locale in q.schema.locales) {
148
148
  if (!lan[locale]) {
149
- lan[locale] = '';
149
+ // console.log('???', locale, prop.default)
150
+ lan[locale] = prop.default[locale] || '';
150
151
  }
151
152
  }
152
153
  }
@@ -154,13 +155,24 @@ const handleUndefinedProps = (id, q, item) => {
154
155
  for (const code of lang) {
155
156
  const locale = inverseLangMap.get(code);
156
157
  if (!lan[locale]) {
157
- lan[locale] = '';
158
+ lan[locale] = prop.default[locale] || '';
158
159
  }
159
160
  }
160
161
  }
161
162
  }
163
+ else if (prop.typeIndex === BINARY) {
164
+ addField(prop, prop.default, item);
165
+ }
166
+ else if (prop.typeIndex === TEXT) {
167
+ // q.lang
168
+ // console.log('???')
169
+ addField(prop, '', item);
170
+ }
162
171
  else {
163
- addField(prop, prop.typeIndex === JSON ? null : '', item);
172
+ 1;
173
+ if (prop.default !== undefined) {
174
+ addField(prop, prop.default, item);
175
+ }
164
176
  }
165
177
  }
166
178
  }
@@ -314,9 +326,6 @@ export const readAllFields = (q, result, offset, end, item, id) => {
314
326
  }
315
327
  else {
316
328
  const prop = q.schema.reverseProps[index];
317
- if (!prop) {
318
- console.log({ prop: !!prop }, index);
319
- }
320
329
  if (prop.typeIndex === CARDINALITY) {
321
330
  q.include.propsRead[index] = id;
322
331
  const size = readUint32(result, i);
@@ -12,8 +12,7 @@ export const vectorSearch = (def, q, field, opts) => {
12
12
  if (prop.typeIndex !== VECTOR) {
13
13
  searchIncorrectType(def, prop);
14
14
  }
15
- // [isVec] [q len] [q len] [field] [fn] [score] [score] [score] [score] [q..]
16
- let size = 9;
15
+ let size = 17;
17
16
  const vec = new Uint8Array(q.buffer, 0, q.byteLength);
18
17
  size += vec.byteLength;
19
18
  def.search = {
@@ -32,21 +31,17 @@ export const search = (def, q, s) => {
32
31
  searchIncorrecQueryValue(def, q);
33
32
  q = '';
34
33
  }
35
- const x = q
36
- .toLowerCase()
37
- .normalize('NFKD')
38
- .trim()
39
- .split(' ')
40
- .map((s) => ` ${s}`);
34
+ const x = q.toLowerCase().normalize('NFKD').trim().split(' ');
41
35
  for (const s of x) {
42
36
  if (s) {
43
37
  const buf = ENCODER.encode(s);
44
- let len = buf.byteLength - 2;
45
- buf[0] = len;
46
- buf[1] = len >>> 8;
47
- bufs.push(buf);
38
+ const lenBuf = new Uint8Array(2);
39
+ let len = buf.byteLength;
40
+ lenBuf[0] = len;
41
+ lenBuf[1] = len >>> 8;
42
+ bufs.push(lenBuf, buf);
48
43
  nrBlocks++;
49
- totalByteLength += len + 2;
44
+ totalByteLength += buf.byteLength + 2;
50
45
  }
51
46
  }
52
47
  bufs.unshift(Uint8Array.from([nrBlocks]));
@@ -119,7 +114,8 @@ export const searchToBuffer = (search) => {
119
114
  | 3 | field | 1 | Field identifier |
120
115
  | 4 | func | 1 | Function identifier (enum) |
121
116
  | 5 | score | 4 | Score value (f32) |
122
- | 9 | query | queryLen | Query data (array of f32 values) |
117
+ | 9 | align | 8 | Space for alignment |
118
+ | 17 | query | queryLen | Query data (array of f32 values) |
123
119
  */
124
120
  const result = new Uint8Array(search.size);
125
121
  result[0] = 1; // search.isVector 1
@@ -128,7 +124,7 @@ export const searchToBuffer = (search) => {
128
124
  result[3] = search.prop;
129
125
  result[4] = getVectorFn(search.opts.fn);
130
126
  result.set(new Uint8Array(new Float32Array([search.opts.score ?? 0.5]).buffer), 5);
131
- result.set(search.query, 9);
127
+ result.set(search.query, 17);
132
128
  return result;
133
129
  }
134
130
  else {
@@ -1,5 +1,4 @@
1
1
  import { BasedDbQuery } from '../BasedDbQuery.js';
2
2
  import { OnData, OnError, OnClose } from './types.js';
3
+ export declare const subscribe: (q: BasedDbQuery, onData: OnData, onError?: OnError) => OnClose;
3
4
  export * from './types.js';
4
- export * from './markers.js';
5
- export declare const subscribe: (q: BasedDbQuery, onData: OnData, onError: OnError) => OnClose;
@@ -1,58 +1,11 @@
1
1
  import { includeField } from '../query.js';
2
2
  import { registerQuery } from '../registerQuery.js';
3
- import { runSubscription } from './run.js';
4
- import { addSubscriptionMarkers, deleteSubscriptionMarkers } from './markers.js';
5
- export * from './types.js';
6
- export * from './markers.js';
7
3
  export const subscribe = (q, onData, onError) => {
8
- let closed = false;
9
4
  if (!q.def.include.stringFields.size && !q.def.references.size) {
10
5
  includeField(q.def, '*');
11
6
  }
12
- try {
13
- registerQuery(q);
14
- }
15
- catch (err) {
16
- onError(err);
17
- return () => q;
18
- }
19
- if (!q.db.subscriptions.has(q.id)) {
20
- const subscription = {
21
- query: q,
22
- subs: new Set(),
23
- inProgress: false,
24
- closed: false,
25
- };
26
- q.db.subscriptions.set(q.id, subscription);
27
- addSubscriptionMarkers(q, subscription);
28
- }
29
- const fn = (res, err) => {
30
- if (!closed) {
31
- if (err) {
32
- onError(err);
33
- }
34
- else {
35
- onData(res);
36
- }
37
- }
38
- };
39
- const sub = q.db.subscriptions.get(q.id);
40
- const close = () => {
41
- sub.subs.delete(fn);
42
- if (sub.subs.size === 0) {
43
- q.db.subscriptions.delete(q.id);
44
- deleteSubscriptionMarkers(q);
45
- }
46
- closed = true;
47
- return q;
48
- };
49
- sub.subs.add(fn);
50
- if (sub.res) {
51
- onData(sub.res);
52
- }
53
- if (!sub.inProgress) {
54
- runSubscription(sub);
55
- }
56
- return close;
7
+ registerQuery(q);
8
+ return q.db.hooks.subscribe(q, onData, onError);
57
9
  };
10
+ export * from './types.js';
58
11
  //# sourceMappingURL=index.js.map
@@ -95,7 +95,7 @@ const getFilterFields = (filter, results = {
95
95
  conditions.forEach((v, k) => {
96
96
  if (k === 0) {
97
97
  for (const buf of v) {
98
- const x = buf[4] | buf[5] << 8;
98
+ const x = buf[4] | (buf[5] << 8);
99
99
  results.main.add(x);
100
100
  }
101
101
  }
@@ -209,6 +209,5 @@ export const resetSubscriptionMarkers = (db) => {
209
209
  };
210
210
  export const deleteSubscriptionMarkers = (q) => {
211
211
  // derp
212
- console.log('delete markers!');
213
212
  };
214
213
  //# sourceMappingURL=markers.js.map
@@ -61,11 +61,9 @@ export const runSubscription = (subscription) => {
61
61
  }
62
62
  };
63
63
  export const startSubscription = (db) => {
64
- console.log('start subscription');
65
64
  if (!db.subscriptionsInProgress) {
66
65
  db.subscriptionsInProgress = true;
67
66
  setTimeout(() => {
68
- console.log('run it');
69
67
  db.subscriptionsToRun.forEach((s) => {
70
68
  runSubscription(s);
71
69
  });
@@ -1,33 +1,5 @@
1
1
  import { BasedQueryResponse } from '../BasedIterable.js';
2
- import { BasedDbQuery } from '../BasedDbQuery.js';
3
2
  export type OnData = (res: BasedQueryResponse) => any;
4
3
  export type OnError = (err: Error) => any;
5
- export type OnClose = () => BasedDbQuery;
4
+ export type OnClose = () => any;
6
5
  export type OnSubscription = (res: any, err?: Error) => void;
7
- export type Subscription = {
8
- query: BasedDbQuery;
9
- subs: Set<OnSubscription>;
10
- res?: BasedQueryResponse;
11
- closed: boolean;
12
- inProgress: boolean;
13
- };
14
- export type SubscriptionsMap = Map<number, Subscription>;
15
- export type SubscriptionsToRun = Subscription[];
16
- export type SubscriptionMarkers = {
17
- main: {
18
- [start: string]: Subscription[];
19
- };
20
- props: {
21
- [prop: string]: Subscription[];
22
- };
23
- };
24
- export type SubscriptionMarkerMap = {
25
- [typeID: string]: {
26
- ids: Map<number, SubscriptionMarkers>;
27
- collection: SubscriptionMarkers;
28
- };
29
- };
30
- export type SubscriptionMarkersCheck = {
31
- collection: SubscriptionMarkers | false;
32
- ids: SubscriptionMarkers | false;
33
- };
@@ -1,3 +1,10 @@
1
1
  export {};
2
- // has sub
2
+ // export type Subscription = {
3
+ // query: BasedDbQuery
4
+ // subs: Set<OnSubscription>
5
+ // res?: BasedQueryResponse
6
+ // closed: boolean
7
+ // inProgress: boolean // dont need to check
8
+ // // filter - realy nice to add
9
+ // }
3
10
  //# sourceMappingURL=types.js.map
@@ -1,6 +1,4 @@
1
1
  export declare const MAX_IDS_PER_QUERY = 1000000;
2
2
  export declare const DEF_RANGE_PROP_LIMIT = 1000;
3
3
  export declare const DEF_RANGE_REF_LIMIT = 10000;
4
- export declare const MAX_ID = 4294967295;
5
4
  export declare const MIN_ID_VALUE = 1;
6
- export declare const MAX_ID_VALUE: number;
@@ -2,8 +2,6 @@
2
2
  export const MAX_IDS_PER_QUERY = 1e6; // Max number of IDs that can be queried at once
3
3
  export const DEF_RANGE_PROP_LIMIT = 1e3;
4
4
  export const DEF_RANGE_REF_LIMIT = 1e4;
5
- export const MAX_ID = 4294967295;
6
5
  // Data validation thresholds
7
6
  export const MIN_ID_VALUE = 1; // Minimum allowed value for an ID (id array stared)
8
- export const MAX_ID_VALUE = 2 ** 32 - 1; /// Maximum allowed value for an ID (Uint32)
9
7
  //# sourceMappingURL=thresholds.js.map
@@ -13,6 +13,7 @@ const byteSize = (arr) => {
13
13
  export function defToBuffer(db, def) {
14
14
  const result = [];
15
15
  const include = includeToBuffer(db, def);
16
+ // ---------------------------------------
16
17
  def.references.forEach((ref) => {
17
18
  include.push(...defToBuffer(db, ref));
18
19
  if (ref.errors) {
@@ -32,27 +33,21 @@ export function defToBuffer(db, def) {
32
33
  edgesSize = byteSize(edges);
33
34
  }
34
35
  const size = (edges ? edgesSize + 3 : 0) + byteSize(include);
36
+ // ---------------------------------------
37
+ // move down and will handle size after store the size Var
38
+ // only for references | edges
35
39
  if (def.type === QueryDefType.Root) {
36
- let filter;
37
- let filterSize = 0;
38
40
  let search;
39
41
  let searchSize = 0;
42
+ const filterSize = def.filter.size || 0;
40
43
  if (def.search) {
41
44
  search = searchToBuffer(def.search);
42
45
  searchSize = def.search.size;
43
46
  }
44
- if (def.filter.size) {
45
- filter = filterToBuffer(def.filter);
46
- filterSize = filter.byteLength;
47
- }
48
- // [type,type]
49
- // [q type] 0 == id, 1 === ids, 2 === type only
50
47
  if (def.target.resolvedAlias) {
51
- // put this somehwere else at some point
52
48
  const alias = def.target.resolvedAlias;
53
49
  const aliasStr = ENCODER.encode(alias.value);
54
50
  const aliasLen = aliasStr.byteLength;
55
- // filter is nice for things like access
56
51
  const buf = new Uint8Array(8 + filterSize + aliasLen);
57
52
  buf[0] = 3;
58
53
  buf[1] = def.schema.idUint8[0];
@@ -64,14 +59,11 @@ export function defToBuffer(db, def) {
64
59
  buf[6 + aliasLen] = filterSize;
65
60
  buf[7 + aliasLen] = filterSize >>> 8;
66
61
  if (filterSize) {
67
- buf.set(filter, 8 + aliasLen);
62
+ buf.set(filterToBuffer(def.filter), 8 + aliasLen);
68
63
  }
69
64
  result.push(buf);
70
65
  }
71
66
  else if (def.target.id) {
72
- // type 0
73
- // 0: 4 [id]
74
- // 0: 2 [filterSize]
75
67
  const buf = new Uint8Array(9 + filterSize);
76
68
  buf[0] = 0;
77
69
  buf[1] = def.schema.idUint8[0];
@@ -83,7 +75,7 @@ export function defToBuffer(db, def) {
83
75
  buf[7] = filterSize;
84
76
  buf[8] = filterSize >>> 8;
85
77
  if (filterSize) {
86
- buf.set(filter, 9);
78
+ buf.set(filterToBuffer(def.filter), 9);
87
79
  }
88
80
  result.push(buf);
89
81
  }
@@ -95,15 +87,10 @@ export function defToBuffer(db, def) {
95
87
  sortSize = sort.byteLength;
96
88
  }
97
89
  let aggregation;
98
- aggregation = createAggFlagBuffer(def.aggregation);
90
+ if (def.aggregation) {
91
+ aggregation = createAggFlagBuffer(def.aggregation);
92
+ }
99
93
  if (def.target.ids) {
100
- // type 1
101
- // 1: 4 + ids * 4 [ids len] [id,id,id]
102
- // 1: 8 [offset, limit]
103
- // 1: 2 [filter size]
104
- // ?filter
105
- // 1: 2 [sort size]
106
- // ?sort
107
94
  if (!sortSize &&
108
95
  (def.range.offset || def.range.limit < def.target.ids.length)) {
109
96
  def.target.ids = def.target.ids.slice(def.range.offset, def.range.offset + def.range.limit);
@@ -129,7 +116,8 @@ export function defToBuffer(db, def) {
129
116
  buf[idsSize + 15] = filterSize;
130
117
  buf[idsSize + 16] = filterSize >>> 8;
131
118
  if (filterSize) {
132
- buf.set(filter, idsSize + 17);
119
+ // just write it here dont do set
120
+ buf.set(filterToBuffer(def.filter), idsSize + 17);
133
121
  }
134
122
  buf[17 + filterSize + idsSize] = sortSize;
135
123
  buf[18 + filterSize + idsSize] = sortSize >>> 8;
@@ -145,13 +133,6 @@ export function defToBuffer(db, def) {
145
133
  result.push(buf);
146
134
  }
147
135
  else {
148
- // type 2
149
- // 2: 0
150
- // 2: 8 [offset, limit]
151
- // 2: 2 [filter size]
152
- // ?filter
153
- // 2: 2 [sort size]
154
- // ?sort
155
136
  const buf = new Uint8Array(18 + filterSize + sortSize + searchSize);
156
137
  buf[0] = 2;
157
138
  buf[1] = def.schema.idUint8[0];
@@ -167,7 +148,7 @@ export function defToBuffer(db, def) {
167
148
  buf[11] = filterSize;
168
149
  buf[12] = filterSize >>> 8;
169
150
  if (filterSize) {
170
- buf.set(filter, 13);
151
+ buf.set(filterToBuffer(def.filter), 13);
171
152
  }
172
153
  buf[13 + filterSize] = sortSize;
173
154
  buf[14 + filterSize] = sortSize >>> 8;
@@ -187,19 +168,12 @@ export function defToBuffer(db, def) {
187
168
  }
188
169
  }
189
170
  else if (def.type === QueryDefType.References) {
190
- // TODO filter edge
191
- let filter;
192
- if (def.filter.size) {
193
- filter = filterToBuffer(def.filter);
194
- }
171
+ const filterSize = def.filter.size || 0;
195
172
  let sort;
196
173
  if (def.sort) {
197
174
  sort = createSortBuffer(def.sort);
198
175
  }
199
- // ADD IT
200
- // TODO: ADD RANGE [offset,limit] (8 bytes)
201
176
  const sortSize = sort?.byteLength ?? 0;
202
- const filterSize = filter?.byteLength ?? 0;
203
177
  const modsSize = filterSize + sortSize;
204
178
  const meta = new Uint8Array(modsSize + 10 + 8);
205
179
  const sz = size + 7 + modsSize + 8;
@@ -218,8 +192,8 @@ export function defToBuffer(db, def) {
218
192
  meta[12] = def.range.limit >>> 8;
219
193
  meta[13] = def.range.limit >>> 16;
220
194
  meta[14] = def.range.limit >>> 24;
221
- if (filter) {
222
- meta.set(filter, 15);
195
+ if (filterSize) {
196
+ meta.set(filterToBuffer(def.filter), 15);
223
197
  }
224
198
  if (sort) {
225
199
  meta.set(sort, 15 + filterSize);
@@ -27,9 +27,10 @@ export type Target = {
27
27
  };
28
28
  };
29
29
  export declare const isRefDef: (def: QueryDef) => def is QueryDefRest;
30
+ export type FilterCondition = Uint8Array;
30
31
  export type QueryDefFilter = {
31
32
  size: number;
32
- conditions: Map<number, Uint8Array[]>;
33
+ conditions: Map<number, FilterCondition[]>;
33
34
  exists?: {
34
35
  prop: PropDef | PropDefEdge;
35
36
  negate: boolean;
@@ -37,7 +38,7 @@ export type QueryDefFilter = {
37
38
  references?: Map<number, QueryDefFilter>;
38
39
  fromRef?: PropDef;
39
40
  schema?: SchemaTypeDef;
40
- edges?: Map<number, Uint8Array[]>;
41
+ edges?: Map<number, FilterCondition[]>;
41
42
  or?: QueryDefFilter;
42
43
  and?: QueryDefFilter;
43
44
  };
@@ -61,9 +61,7 @@ export declare const searchIncorrecQueryValue: (def: QueryDef, payload: any) =>
61
61
  export declare const searchIncorrectType: (def: QueryDef, payload: PropDef | PropDefEdge) => void;
62
62
  export declare const searchDoesNotExist: (def: QueryDef, field: string, isVector: boolean) => PropDef;
63
63
  export declare const validateRange: (def: QueryDef, offset: number, limit: number) => boolean;
64
- export declare const isValidId: (id: number) => boolean;
65
- export declare const isValidString: (v: any) => boolean;
66
- export declare const validateVal: (def: QueryDef, f: Filter, validate: (v: any) => boolean) => boolean;
64
+ export declare const validateVal: (def: QueryDef, f: Filter, validate: (v: any, p: PropDef | PropDefEdge) => boolean) => boolean;
67
65
  export declare const validateFilter: (def: QueryDef, prop: PropDef | PropDefEdge, f: Filter) => boolean;
68
66
  export declare const validateType: (db: DbClient, def: QueryDef, type: string) => SchemaTypeDef;
69
67
  export declare const filterOperatorDoesNotExist: (def: QueryDef, field: string) => void;
@@ -1,9 +1,9 @@
1
1
  import picocolors from 'picocolors';
2
2
  import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, DEFAULT_MAP, } from '@based/schema/def';
3
3
  import { EQUAL, EXISTS, HAS, isNumerical, LIKE, operatorReverseMap, VECTOR_FNS, } from './filter/types.js';
4
- import { MAX_ID, MAX_ID_VALUE, MAX_IDS_PER_QUERY, MIN_ID_VALUE, } from './thresholds.js';
4
+ import { MAX_IDS_PER_QUERY } from './thresholds.js';
5
5
  import { displayTarget, safeStringify } from './display.js';
6
- import { langCodesMap } from '@based/schema';
6
+ import { isValidId, isValidString, langCodesMap, MAX_ID, } from '@based/schema';
7
7
  export const ERR_TARGET_INVAL_TYPE = 1;
8
8
  export const ERR_TARGET_INVAL_ALIAS = 2;
9
9
  export const ERR_TARGET_EXCEED_MAX_IDS = 3;
@@ -85,18 +85,6 @@ export const validateRange = (def, offset, limit) => {
85
85
  }
86
86
  return r;
87
87
  };
88
- export const isValidId = (id) => {
89
- if (typeof id != 'number' || id < MIN_ID_VALUE || id > MAX_ID_VALUE) {
90
- return false;
91
- }
92
- return true;
93
- };
94
- export const isValidString = (v) => {
95
- const isVal = typeof v === 'string' ||
96
- v instanceof Uint8Array ||
97
- ArrayBuffer.isView(v);
98
- return isVal;
99
- };
100
88
  export const validateVal = (def, f, validate) => {
101
89
  if (def.skipValidation) {
102
90
  return false;
@@ -104,13 +92,13 @@ export const validateVal = (def, f, validate) => {
104
92
  const value = f[2];
105
93
  if (Array.isArray(value)) {
106
94
  for (const v of value) {
107
- if (!validate(v)) {
95
+ if (!validate(v, f[2])) {
108
96
  def.errors.push({ code: ERR_FILTER_INVALID_VAL, payload: f });
109
97
  return true;
110
98
  }
111
99
  }
112
100
  }
113
- else if (!validate(value)) {
101
+ else if (!validate(value, f[2])) {
114
102
  def.errors.push({
115
103
  code: ERR_FILTER_INVALID_VAL,
116
104
  payload: f,
@@ -147,7 +135,7 @@ export const validateFilter = (def, prop, f) => {
147
135
  });
148
136
  return true;
149
137
  }
150
- if (validateVal(def, f, isValidId)) {
138
+ if (validateVal(def, f, prop.validation)) {
151
139
  return true;
152
140
  }
153
141
  }
@@ -287,7 +275,7 @@ export const validateSort = (def, field, orderInput) => {
287
275
  }
288
276
  }
289
277
  const type = propDef.typeIndex;
290
- if (type === REFERENCES || type === REFERENCE) {
278
+ if (type === REFERENCES || type === REFERENCE || type === VECTOR) {
291
279
  def.errors.push({
292
280
  code: ERR_SORT_TYPE,
293
281
  payload: propDef,
@@ -1,4 +1,6 @@
1
1
  import { LangCode } from '@based/schema';
2
+ export declare const COMPRESSED = 1;
3
+ export declare const NOT_COMPRESSED = 0;
2
4
  export declare const write: (buf: Uint8Array, value: string, offset: number, noCompression: boolean, lang?: LangCode) => number;
3
5
  export declare const compress: (str: string) => Uint8Array;
4
6
  export declare const decompress: (val: Uint8Array) => string;
@@ -3,23 +3,19 @@ import { readUint32 } from './../utils.js';
3
3
  import makeTmpBuffer from './tmpBuffer.js';
4
4
  import { DECODER, ENCODER } from '../utils.js';
5
5
  const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
6
- // type 0 = no compression; 1 = deflate
7
- // [lang] [type] [uncompressed size 4] [compressed string] [crc32]
8
- // var cnt = 0
9
- // var bytesSaved = 0
10
- // make this into a package
11
- // write the type Byte
6
+ export const COMPRESSED = 1;
7
+ export const NOT_COMPRESSED = 0;
12
8
  export const write = (buf, value, offset, noCompression, lang) => {
13
9
  value = value.normalize('NFKD');
14
10
  buf[offset] = lang || 0;
15
11
  const { written: l } = ENCODER.encodeInto(value, buf.subarray(offset + 2));
16
12
  let crc = native.crc32(buf.subarray(offset + 2, offset + 2 + l));
17
- // 50 maybe if lvl 1
13
+ // 50 len maybe if lvl 1
18
14
  if (value.length > 200 && !noCompression) {
19
15
  buf.copyWithin(offset + 6 + l, offset + 2, offset + 2 + l);
20
16
  const size = native.compress(buf, offset + 6, l);
21
17
  if (size === 0) {
22
- buf[offset + 1] = 0; // not compressed
18
+ buf[offset + 1] = NOT_COMPRESSED;
23
19
  ENCODER.encodeInto(value, buf.subarray(offset + 2));
24
20
  buf[offset + l + 2] = crc;
25
21
  buf[offset + l + 3] = crc >>>= 8;
@@ -29,7 +25,7 @@ export const write = (buf, value, offset, noCompression, lang) => {
29
25
  }
30
26
  else {
31
27
  let len = l;
32
- buf[offset + 1] = 1; // compressed
28
+ buf[offset + 1] = COMPRESSED;
33
29
  buf[offset + 2] = len;
34
30
  buf[offset + 3] = len >>>= 8;
35
31
  buf[offset + 4] = len >>>= 8;
@@ -38,11 +34,11 @@ export const write = (buf, value, offset, noCompression, lang) => {
38
34
  buf[offset + size + 7] = crc >>>= 8;
39
35
  buf[offset + size + 8] = crc >>>= 8;
40
36
  buf[offset + size + 9] = crc >>>= 8;
41
- return size + 10; // 0 C 4 4
37
+ return size + 10;
42
38
  }
43
39
  }
44
40
  else {
45
- buf[offset + 1] = 0; // not compressed
41
+ buf[offset + 1] = NOT_COMPRESSED;
46
42
  buf[offset + l + 2] = crc;
47
43
  buf[offset + l + 3] = crc >>>= 8;
48
44
  buf[offset + l + 4] = crc >>>= 8;
@@ -63,14 +59,14 @@ export const decompress = (val) => {
63
59
  };
64
60
  export const read = (val, offset, len) => {
65
61
  const type = val[offset + 1];
66
- if (type == 1) {
62
+ if (type == COMPRESSED) {
67
63
  const origSize = readUint32(val, offset + 2);
68
64
  const newBuffer = getTmpBuffer(origSize);
69
- // deflate in browser for this...
65
+ // Browser fallback required for this
70
66
  native.decompress(val, newBuffer, offset + 6, len - 6);
71
67
  return DECODER.decode(newBuffer);
72
68
  }
73
- else if (type == 0) {
69
+ else if (type == NOT_COMPRESSED) {
74
70
  return DECODER.decode(val.subarray(offset + 2, len + offset - 4));
75
71
  }
76
72
  return '';
@@ -22,7 +22,7 @@ export declare class BasedDb {
22
22
  constructor(opts: {
23
23
  path: string;
24
24
  maxModifySize?: number;
25
- debug?: boolean;
25
+ debug?: boolean | 'server';
26
26
  });
27
27
  create: DbClient['create'];
28
28
  copy: DbClient['copy'];