@based/db 0.0.66 → 0.0.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +2 -2
  2. package/dist/lib/darwin_aarch64/include/selva/colvec.h +71 -0
  3. package/dist/lib/darwin_aarch64/include/selva/db.h +3 -0
  4. package/dist/lib/darwin_aarch64/include/selva/fields.h +1 -1
  5. package/dist/lib/darwin_aarch64/include/selva/types.h +7 -0
  6. package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
  7. package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
  8. package/dist/lib/darwin_aarch64/libjemalloc_selva.so.2 +0 -0
  9. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  10. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  11. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  12. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  13. package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
  14. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  15. package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
  16. package/dist/lib/linux_aarch64/include/selva/colvec.h +71 -0
  17. package/dist/lib/linux_aarch64/include/selva/db.h +3 -0
  18. package/dist/lib/linux_aarch64/include/selva/fields.h +1 -1
  19. package/dist/lib/linux_aarch64/include/selva/types.h +7 -0
  20. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  21. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  22. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  23. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  24. package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
  25. package/dist/lib/linux_aarch64/libselva.so +0 -0
  26. package/dist/lib/linux_x86_64/include/selva/colvec.h +71 -0
  27. package/dist/lib/linux_x86_64/include/selva/db.h +3 -0
  28. package/dist/lib/linux_x86_64/include/selva/fields.h +1 -1
  29. package/dist/lib/linux_x86_64/include/selva/types.h +7 -0
  30. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  31. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  32. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  33. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  34. package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
  35. package/dist/lib/linux_x86_64/libselva.so +0 -0
  36. package/dist/src/client/flushModify.d.ts +2 -1
  37. package/dist/src/client/flushModify.js +12 -4
  38. package/dist/src/client/modify/create.js +11 -0
  39. package/dist/src/client/modify/delete.js +3 -0
  40. package/dist/src/client/modify/modify.js +2 -2
  41. package/dist/src/client/modify/setCursor.d.ts +2 -1
  42. package/dist/src/client/query/BasedDbQuery.d.ts +7 -2
  43. package/dist/src/client/query/BasedDbQuery.js +111 -14
  44. package/dist/src/client/query/aggregates/aggregation.js +24 -11
  45. package/dist/src/client/query/aggregates/types.d.ts +21 -2
  46. package/dist/src/client/query/aggregates/types.js +34 -1
  47. package/dist/src/client/query/display.js +8 -2
  48. package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +2 -3
  49. package/dist/src/client/query/filter/createVariableFilterBuffer.js +20 -7
  50. package/dist/src/client/query/filter/filter.js +13 -3
  51. package/dist/src/client/query/filter/primitiveFilter.d.ts +1 -2
  52. package/dist/src/client/query/include/props.js +18 -2
  53. package/dist/src/client/query/include/toBuffer.js +11 -3
  54. package/dist/src/client/query/include/walk.js +5 -1
  55. package/dist/src/client/query/queryDef.js +4 -1
  56. package/dist/src/client/query/read/read.js +50 -23
  57. package/dist/src/client/query/registerQuery.js +1 -0
  58. package/dist/src/client/query/search/index.d.ts +1 -1
  59. package/dist/src/client/query/search/index.js +21 -7
  60. package/dist/src/client/query/sort.d.ts +1 -1
  61. package/dist/src/client/query/toByteCode/default.d.ts +1 -1
  62. package/dist/src/client/query/toByteCode/default.js +0 -2
  63. package/dist/src/client/query/toByteCode/toBuffer.js +0 -7
  64. package/dist/src/client/query/types.d.ts +16 -5
  65. package/dist/src/client/query/validation.js +25 -2
  66. package/dist/src/client/xxHash64.d.ts +1 -1
  67. package/dist/src/index.d.ts +0 -1
  68. package/dist/src/index.js +0 -1
  69. package/dist/src/native.d.ts +2 -2
  70. package/dist/src/native.js +7 -8
  71. package/dist/src/server/IoWorker.d.ts +8 -0
  72. package/dist/src/server/IoWorker.js +39 -0
  73. package/dist/src/server/QueryWorker.d.ts +8 -0
  74. package/dist/src/server/QueryWorker.js +26 -0
  75. package/dist/src/server/blocks.d.ts +24 -0
  76. package/dist/src/server/blocks.js +112 -0
  77. package/dist/src/server/dbHash.d.ts +1 -1
  78. package/dist/src/server/index.d.ts +9 -16
  79. package/dist/src/server/index.js +37 -15
  80. package/dist/src/server/migrate/index.d.ts +5 -0
  81. package/dist/src/server/migrate/index.js +10 -7
  82. package/dist/src/server/save.d.ts +8 -6
  83. package/dist/src/server/save.js +34 -78
  84. package/dist/src/server/schema.js +6 -5
  85. package/dist/src/server/start.js +57 -60
  86. package/dist/src/server/tree.d.ts +24 -13
  87. package/dist/src/server/tree.js +95 -66
  88. package/dist/src/server/workers/DbWorker.d.ts +17 -0
  89. package/dist/src/server/{DbWorker.js → workers/DbWorker.js} +15 -17
  90. package/dist/src/server/workers/io_worker.js +39 -0
  91. package/dist/src/server/workers/io_worker_types.d.ts +12 -0
  92. package/dist/src/server/workers/io_worker_types.js +2 -0
  93. package/dist/src/server/workers/query_worker.d.ts +1 -0
  94. package/dist/src/server/workers/query_worker.js +4 -0
  95. package/dist/src/server/workers/worker.d.ts +1 -0
  96. package/dist/src/server/workers/worker.js +41 -0
  97. package/dist/src/shared/Emitter.d.ts +1 -0
  98. package/dist/src/types.d.ts +1 -1
  99. package/dist/src/types.js +1 -1
  100. package/package.json +3 -3
  101. package/dist/lib/darwin_aarch64/include/selva/history.h +0 -64
  102. package/dist/lib/linux_aarch64/include/selva/history.h +0 -64
  103. package/dist/lib/linux_x86_64/include/selva/history.h +0 -64
  104. package/dist/src/client/query/serialize.d.ts +0 -4
  105. package/dist/src/client/query/serialize.js +0 -26
  106. package/dist/src/server/DbWorker.d.ts +0 -13
  107. package/dist/src/server/csmt/draw-dot.d.ts +0 -4
  108. package/dist/src/server/csmt/draw-dot.js +0 -38
  109. package/dist/src/server/csmt/index.d.ts +0 -4
  110. package/dist/src/server/csmt/index.js +0 -5
  111. package/dist/src/server/csmt/match.d.ts +0 -7
  112. package/dist/src/server/csmt/match.js +0 -10
  113. package/dist/src/server/csmt/memebership-proof.d.ts +0 -7
  114. package/dist/src/server/csmt/memebership-proof.js +0 -122
  115. package/dist/src/server/csmt/tree-utils.d.ts +0 -6
  116. package/dist/src/server/csmt/tree-utils.js +0 -33
  117. package/dist/src/server/csmt/tree.d.ts +0 -3
  118. package/dist/src/server/csmt/tree.js +0 -270
  119. package/dist/src/server/csmt/types.d.ts +0 -46
  120. package/dist/src/server/csmt/types.js +0 -2
  121. package/dist/src/server/worker.js +0 -33
  122. /package/dist/src/server/{worker.d.ts → workers/io_worker.d.ts} +0 -0
@@ -1,9 +1,10 @@
1
- import { ALIAS, ALIASES, BINARY, BOOLEAN, ENUM, INT16, INT32, INT8, NUMBER, STRING, TEXT, TIMESTAMP, UINT16, UINT32, UINT8, VECTOR, JSON, CARDINALITY, } from '@based/schema/def';
1
+ import { ALIAS, ALIASES, BINARY, BOOLEAN, ENUM, INT16, INT32, INT8, NUMBER, STRING, TEXT, TIMESTAMP, UINT16, UINT32, UINT8, VECTOR, JSON, CARDINALITY, COLVEC, } from '@based/schema/def';
2
2
  import { QueryDefType } from '../types.js';
3
3
  import { read, readUtf8 } from '../../string.js';
4
4
  import { DECODER, readDoubleLE, readFloatLE, readInt16, readInt32, readUint16, readUint32, setByPath, } from '@saulx/utils';
5
5
  import { inverseLangMap } from '@based/schema';
6
6
  import { READ_EDGE, READ_ID, READ_REFERENCE, READ_REFERENCES, READ_AGGREGATION, } from '../types.js';
7
+ import { AggregateType } from '../aggregates/types.js';
7
8
  const readAggregate = (q, result, offset, len) => {
8
9
  const results = {};
9
10
  if (q.aggregate.groupBy) {
@@ -21,24 +22,47 @@ const readAggregate = (q, result, offset, len) => {
21
22
  i += 2;
22
23
  }
23
24
  else {
24
- keyLen = readUint16(result, i);
25
- i += 2;
26
- key = DECODER.decode(result.subarray(i, i + keyLen));
27
- i += keyLen;
25
+ if (q.aggregate.groupBy.typeIndex == ENUM) {
26
+ i += 2;
27
+ key = q.aggregate.groupBy.enum[result[i] - 1];
28
+ i++;
29
+ }
30
+ else {
31
+ keyLen = readUint16(result, i);
32
+ i += 2;
33
+ key = DECODER.decode(result.subarray(i, i + keyLen));
34
+ i += keyLen;
35
+ }
28
36
  }
29
37
  const resultKey = (results[key] = {});
30
38
  for (const aggregatesArray of q.aggregate.aggregates.values()) {
31
39
  for (const agg of aggregatesArray) {
32
- setByPath(resultKey, agg.propDef.path, readUint32(result, agg.resultPos + i));
40
+ var val = undefined;
41
+ if (agg.type === AggregateType.CARDINALITY ||
42
+ agg.type === AggregateType.COUNT) {
43
+ val = readUint32(result, agg.resultPos + i);
44
+ }
45
+ else {
46
+ val = readDoubleLE(result, agg.resultPos + i);
47
+ }
48
+ setByPath(resultKey, agg.propDef.path, val);
33
49
  }
34
50
  }
35
- i += q.aggregate.totalResultsPos;
51
+ i += q.aggregate.totalResultsSize;
36
52
  }
37
53
  }
38
54
  else {
39
55
  for (const aggregatesArray of q.aggregate.aggregates.values()) {
40
56
  for (const agg of aggregatesArray) {
41
- setByPath(results, agg.propDef.path, readUint32(result, agg.resultPos + offset));
57
+ var val = undefined;
58
+ if (agg.type === AggregateType.CARDINALITY ||
59
+ agg.type === AggregateType.COUNT) {
60
+ val = readUint32(result, agg.resultPos + offset);
61
+ }
62
+ else {
63
+ val = readDoubleLE(result, agg.resultPos + offset);
64
+ }
65
+ setByPath(results, agg.propDef.path, val);
42
66
  }
43
67
  }
44
68
  }
@@ -172,13 +196,13 @@ const handleUndefinedProps = (id, q, item) => {
172
196
  if (prop.typeIndex === CARDINALITY) {
173
197
  addField(prop, 0, item);
174
198
  }
175
- else if (prop.typeIndex === TEXT && q.lang == 0) {
199
+ else if (prop.typeIndex === TEXT && q.lang.lang == 0) {
176
200
  const lan = getEmptyField(prop, item);
177
201
  const lang = q.include.langTextFields.get(prop.prop).codes;
178
202
  if (lang.has(0)) {
179
203
  for (const locale in q.schema.locales) {
180
204
  if (lan[locale] == undefined) {
181
- lan[locale] = prop.default[locale] || '';
205
+ lan[locale] = '';
182
206
  }
183
207
  }
184
208
  }
@@ -186,23 +210,26 @@ const handleUndefinedProps = (id, q, item) => {
186
210
  for (const code of lang) {
187
211
  const locale = inverseLangMap.get(code);
188
212
  if (!lan[locale]) {
189
- lan[locale] = prop.default[locale] || '';
213
+ lan[locale] = '';
190
214
  }
191
215
  }
192
216
  }
193
217
  }
194
- else if (prop.typeIndex === BINARY) {
195
- addField(prop, prop.default, item);
196
- }
197
- else if (prop.typeIndex === TEXT) {
218
+ // else if (prop.typeIndex === BINARY) {
219
+ // addField(prop, prop.default, item)
220
+ // }
221
+ else if (prop.typeIndex === TEXT ||
222
+ prop.typeIndex === STRING ||
223
+ prop.typeIndex === ALIAS) {
198
224
  addField(prop, '', item);
199
225
  }
200
- else {
201
- 1;
202
- if (prop.default !== undefined) {
203
- addField(prop, prop.default, item);
204
- }
205
- }
226
+ // else if (prop.typeIndex === JSON) {
227
+ // // addField(prop, null, item)
228
+ // } else {
229
+ // // if (prop.default !== undefined) {
230
+ // // addField(prop, '', item)
231
+ // // }
232
+ // }
206
233
  }
207
234
  }
208
235
  };
@@ -380,7 +407,7 @@ export const readAllFields = (q, result, offset, end, item, id) => {
380
407
  // do nothing
381
408
  }
382
409
  else {
383
- if (q.lang != 0) {
410
+ if (q.lang.lang != 0) {
384
411
  q.include.propsRead[index] = id;
385
412
  addField(prop, read(result, i + 4, size, true), item);
386
413
  }
@@ -403,7 +430,7 @@ export const readAllFields = (q, result, offset, end, item, id) => {
403
430
  addField(prop, string, item);
404
431
  }
405
432
  }
406
- else if (prop.typeIndex == VECTOR) {
433
+ else if (prop.typeIndex == VECTOR || prop.typeIndex == COLVEC) {
407
434
  q.include.propsRead[index] = id;
408
435
  const size = readUint32(result, i);
409
436
  const arr = new Float32Array(size / 4);
@@ -16,6 +16,7 @@ export const registerQuery = (q) => {
16
16
  for (const command of commands) {
17
17
  q[command.method](...command.args);
18
18
  }
19
+ // locale first...
19
20
  if (!q.def.include.stringFields.size && !q.def.references.size) {
20
21
  includeField(q.def, '*');
21
22
  }
@@ -5,4 +5,4 @@ export type Search = string[] | {
5
5
  } | string;
6
6
  export declare const vectorSearch: (def: QueryDef, q: ArrayBufferView, field: string, opts: Omit<FilterOpts, "lowerCase">) => void;
7
7
  export declare const search: (def: QueryDef, q: string, s?: Search) => void;
8
- export declare const searchToBuffer: (search: QueryDefSearch) => Uint8Array;
8
+ export declare const searchToBuffer: (search: QueryDefSearch) => Uint8Array<ArrayBuffer>;
@@ -74,14 +74,16 @@ export const search = (def, q, s) => {
74
74
  }
75
75
  for (const key in s) {
76
76
  let prop = def.props[key];
77
- let lang = def.lang;
77
+ let lang = def.lang.lang;
78
+ let fallback = def.lang.fallback;
78
79
  if (!prop) {
79
80
  if (key.includes('.')) {
80
81
  const k = key.split('.');
81
82
  prop = def.props[k.slice(0, -1).join('.')];
82
83
  if (prop && prop.typeIndex === TEXT) {
83
84
  lang = langCodesMap.get(k[k.length - 1]);
84
- // incorrect LANG
85
+ fallback = [];
86
+ // handle incorrect LANG
85
87
  }
86
88
  else {
87
89
  prop = searchDoesNotExist(def, key, false);
@@ -94,11 +96,16 @@ export const search = (def, q, s) => {
94
96
  if (prop.typeIndex !== STRING && prop.typeIndex !== TEXT) {
95
97
  searchIncorrectType(def, prop);
96
98
  }
97
- def.search.size += 6;
99
+ def.search.size += 10;
100
+ if (fallback.length > 3) {
101
+ // Max 4 lang fallback size (tmp)
102
+ console.warn('Search has a max of 4 lang fallbacks!');
103
+ fallback = fallback.slice(0, 3);
104
+ }
98
105
  def.search.fields.push({
99
106
  typeIndex: prop.typeIndex,
100
107
  weight: s[key],
101
- lang,
108
+ lang: { lang, fallback },
102
109
  field: prop.prop,
103
110
  start: prop.start ?? 0, // also need lang ofc if you have start
104
111
  });
@@ -147,6 +154,8 @@ export const searchToBuffer = (search) => {
147
154
  | 2 | weight | 1 | Field weight value |
148
155
  | 3 | start | 2 | Start position in the query (u16) |
149
156
  | 5 | lang | 1 | Language identifier |
157
+ | 6 | fallbackL | 1 | Language fallback size |
158
+ | 7 | fallback | 3 | Language fallback |
150
159
 
151
160
  ### Notes:
152
161
  - The number of field entries is inferred from the total packet size.
@@ -162,15 +171,20 @@ export const searchToBuffer = (search) => {
162
171
  return a.weight - b.weight;
163
172
  });
164
173
  // @ts-ignore
165
- for (let i = 0; i < search.fields.length * 6; i += 6) {
174
+ for (let i = 0; i < search.fields.length * 10; i += 10) {
166
175
  // @ts-ignore
167
- const f = search.fields[Math.floor(i / 6)];
176
+ const f = search.fields[Math.floor(i / 10)];
168
177
  result[i + offset] = f.field;
169
178
  result[i + offset + 1] = f.typeIndex;
170
179
  result[i + offset + 2] = f.weight;
171
180
  result[i + offset + 3] = f.start;
172
181
  result[i + offset + 4] = f.start >>> 8;
173
- result[i + offset + 5] = f.lang;
182
+ // fallback
183
+ result[i + offset + 5] = f.lang.lang;
184
+ result[i + offset + 6] = f.lang.fallback.length;
185
+ for (let j = 0; j < f.lang.fallback.length; j++) {
186
+ result[i + j + offset + 7] = f.lang.fallback[j];
187
+ }
174
188
  }
175
189
  return result;
176
190
  }
@@ -1,3 +1,3 @@
1
1
  import { QueryDef, QueryDefSort } from './types.js';
2
- export declare const createSortBuffer: (sort: QueryDefSort) => Uint8Array;
2
+ export declare const createSortBuffer: (sort: QueryDefSort) => Uint8Array<ArrayBuffer>;
3
3
  export declare const sort: (def: QueryDef, field: string, order?: "asc" | "desc") => void;
@@ -1,2 +1,2 @@
1
1
  import { QueryDef } from '../types.js';
2
- export declare const defaultQuery: (def: QueryDef, filterSize: number, sortSize: number, searchSize: number, sort: Uint8Array, search: Uint8Array) => Uint8Array;
2
+ export declare const defaultQuery: (def: QueryDef, filterSize: number, sortSize: number, searchSize: number, sort: Uint8Array, search: Uint8Array) => Uint8Array<ArrayBuffer>;
@@ -17,8 +17,6 @@ export const defaultQuery = (def, filterSize, sortSize, searchSize, sort, search
17
17
  buf[index++] = filterSize;
18
18
  buf[index++] = filterSize >>> 8;
19
19
  buf[index++] = filterSize && isSimpleMainFilter(def.filter) ? 1 : 0;
20
- // if (filterSize && isSimpleMainFilter(def.filter)) {
21
- // console.log('SIMPLE FILTER!')
22
20
  if (filterSize) {
23
21
  buf.set(filterToBuffer(def.filter), index);
24
22
  index += filterSize;
@@ -14,7 +14,6 @@ const byteSize = (arr) => {
14
14
  export function defToBuffer(db, def) {
15
15
  const result = [];
16
16
  const include = includeToBuffer(db, def);
17
- // ---------------------------------------
18
17
  def.references.forEach((ref) => {
19
18
  include.push(...defToBuffer(db, ref));
20
19
  if (ref.errors) {
@@ -46,7 +45,6 @@ export function defToBuffer(db, def) {
46
45
  buf[0] = 251 /* includeOp.REFERENCES_AGGREGATION */;
47
46
  buf[1] = sz;
48
47
  buf[2] = sz >>> 8;
49
- // ---
50
48
  buf[3] = filterSize;
51
49
  buf[4] = filterSize >>> 8;
52
50
  buf[5] = def.range.offset;
@@ -62,8 +60,6 @@ export function defToBuffer(db, def) {
62
60
  buf[9 + 2 + filterSize] = def.target.propDef.prop; // refField
63
61
  const aggregateBuffer = aggregateToBuffer(def.aggregate);
64
62
  buf.set(aggregateBuffer, 9 + 3 + filterSize);
65
- // buf[12 + filterSize] = aggregateSize
66
- // buf[12 + 1 + filterSize] = aggregateSize >>> 8
67
63
  result.push(buf);
68
64
  }
69
65
  else {
@@ -92,8 +88,6 @@ export function defToBuffer(db, def) {
92
88
  buf.set(aggregateBuffer, 16 + filterSize);
93
89
  result.push(buf);
94
90
  }
95
- // ignore this for now...
96
- // result.push(...include)
97
91
  if (def.type === QueryDefType.Root) {
98
92
  const checksum = new Uint8Array(8);
99
93
  writeUint64(checksum, def.schemaChecksum ?? 0, 0);
@@ -175,7 +169,6 @@ export function defToBuffer(db, def) {
175
169
  buf[idsSize + 13] = def.range.limit >>> 16;
176
170
  buf[idsSize + 14] = def.range.limit >>> 24;
177
171
  // if (filterSize && isSimpleMainFilter(def.filter)) {
178
- // console.log('SIMPLE FILTER!')
179
172
  buf[idsSize + 15] = filterSize;
180
173
  buf[idsSize + 16] = filterSize >>> 8;
181
174
  if (filterSize) {
@@ -1,4 +1,4 @@
1
- import { LangCode } from '@based/schema';
1
+ import { LangCode, LangName } from '@based/schema';
2
2
  import { PropDef, PropDefEdge, SchemaTypeDef } from '@based/schema/def';
3
3
  import { FilterOpts } from './filter/types.js';
4
4
  import { QueryError } from './validation.js';
@@ -59,7 +59,10 @@ export type QueryDefSearch = {
59
59
  weight: number;
60
60
  field: number;
61
61
  start: number;
62
- lang: LangCode;
62
+ lang: {
63
+ lang: LangCode;
64
+ fallback: LangCode[];
65
+ };
63
66
  typeIndex: number;
64
67
  }[];
65
68
  } | {
@@ -78,17 +81,23 @@ export type Aggregation = {
78
81
  type: AggregateType;
79
82
  propDef: PropDef;
80
83
  resultPos: number;
84
+ accumulatorPos: number;
81
85
  };
82
86
  export type QueryDefAggregation = {
83
87
  size: number;
84
88
  groupBy?: PropDef;
85
89
  aggregates: Map<number, Aggregation[]>;
86
- totalResultsPos: number;
90
+ totalResultsSize: number;
91
+ totalAccumulatorSize: number;
87
92
  };
93
+ export type LangFallback = LangName | false;
88
94
  export type QueryDefShared = {
89
95
  schemaChecksum?: number;
90
96
  errors: QueryError[];
91
- lang: LangCode;
97
+ lang: {
98
+ lang: LangCode;
99
+ fallback: LangCode[];
100
+ };
92
101
  filter: QueryDefFilter;
93
102
  aggregate: null | QueryDefAggregation;
94
103
  search: null | QueryDefSearch;
@@ -99,9 +108,11 @@ export type QueryDefShared = {
99
108
  limit: number;
100
109
  };
101
110
  include: {
102
- langTextFields: Map<number, {
111
+ langTextFields: Map<number, // prop name
112
+ {
103
113
  def: PropDef | PropDefEdge;
104
114
  codes: Set<LangCode>;
115
+ fallBacks: LangCode[];
105
116
  }>;
106
117
  stringFields: Set<string>;
107
118
  props: Map<number, PropDef | PropDefEdge>;
@@ -139,6 +139,28 @@ export const validateFilter = (def, prop, f) => {
139
139
  });
140
140
  return true;
141
141
  }
142
+ // map { id: } format for filter
143
+ const values = f[2];
144
+ if (Array.isArray(values)) {
145
+ let hasObject = false;
146
+ for (const v of values) {
147
+ if (typeof v === 'object' && 'id' in v) {
148
+ hasObject = true;
149
+ break;
150
+ }
151
+ }
152
+ if (hasObject) {
153
+ f[2] = values.map((v) => {
154
+ if (typeof v === 'object' && 'id' in v) {
155
+ return v.id;
156
+ }
157
+ return v;
158
+ });
159
+ }
160
+ }
161
+ else if (typeof values === 'object' && 'id' in values) {
162
+ f[2] = values.id;
163
+ }
142
164
  if (validateVal(def, f, prop.validation)) {
143
165
  return true;
144
166
  }
@@ -274,7 +296,7 @@ export const validateSort = (def, field, orderInput) => {
274
296
  return {
275
297
  prop: EMPTY_ALIAS_PROP_DEF,
276
298
  order,
277
- lang: def.lang,
299
+ lang: def.lang?.lang,
278
300
  };
279
301
  }
280
302
  }
@@ -287,7 +309,7 @@ export const validateSort = (def, field, orderInput) => {
287
309
  }
288
310
  else if (type === TEXT) {
289
311
  if (lang === 0) {
290
- lang = def.lang ?? 0;
312
+ lang = def.lang?.lang ?? 0;
291
313
  if (lang === 0) {
292
314
  def.errors.push({
293
315
  code: ERR_SORT_LANG,
@@ -445,6 +467,7 @@ export const EMPTY_SCHEMA_DEF = {
445
467
  propNames: new Uint8Array([]),
446
468
  idUint8: new Uint8Array([0, 0]),
447
469
  mainEmptyAllZeroes: true,
470
+ hasSeperateDefaults: false,
448
471
  };
449
472
  export const aggregationFieldDoesNotExist = (def, field) => {
450
473
  def.errors.push({
@@ -1 +1 @@
1
- export declare const xxHash64: (buf: Uint8Array, target?: Uint8Array, index?: number) => Uint8Array;
1
+ export declare const xxHash64: (buf: Uint8Array, target?: Uint8Array, index?: number) => Uint8Array<ArrayBufferLike>;
@@ -12,7 +12,6 @@ export { DbClient, DbServer };
12
12
  export { xxHash64 } from './client/xxHash64.js';
13
13
  export { crc32 } from './client/crc32.js';
14
14
  export { default as createHash } from './server/dbHash.js';
15
- export * from './client/query/serialize.js';
16
15
  export * from './utils.js';
17
16
  export * from './client/query/query.js';
18
17
  export * from './client/query/BasedDbQuery.js';
package/dist/src/index.js CHANGED
@@ -13,7 +13,6 @@ export { DbClient, DbServer };
13
13
  export { xxHash64 } from './client/xxHash64.js';
14
14
  export { crc32 } from './client/crc32.js';
15
15
  export { default as createHash } from './server/dbHash.js';
16
- export * from './client/query/serialize.js';
17
16
  export * from './utils.js';
18
17
  export * from './client/query/query.js';
19
18
  export * from './client/query/BasedDbQuery.js';
@@ -1,7 +1,5 @@
1
1
  declare const native: {
2
2
  threadCtx: any;
3
- historyAppend(history: any, typeId: number, nodeId: number, dbCtx: any): any;
4
- historyCreate(pathname: string, mainLen: number): any;
5
3
  workerCtxInit: () => void;
6
4
  externalFromInt(address: BigInt): any;
7
5
  intFromExternal(external: any): BigInt;
@@ -13,6 +11,7 @@ declare const native: {
13
11
  saveBlock: (path: string, typeCode: number, start: number, dbCtx: any, hashOut: Uint8Array) => number;
14
12
  loadCommon: (path: string, dbCtx: any) => void;
15
13
  loadBlock: (path: string, dbCtx: any) => void;
14
+ delBlock: (dbCtx: any, typeId: number, block: number) => void;
16
15
  updateSchemaType: (prefix: number, buf: Uint8Array, dbCtx: any) => any;
17
16
  getTypeInfo: (typeId: number, dbCtx: any) => any;
18
17
  getNodeRangeHash: (typeId: number, start: number, end: number, bufOut: Uint8Array, dbCtx: any) => any;
@@ -25,5 +24,6 @@ declare const native: {
25
24
  equals: (a: Uint8Array, b: Uint8Array) => boolean;
26
25
  membarSyncRead: () => void;
27
26
  membarSyncWrite: () => void;
27
+ colvecTest: (dbCtx: any, typeId: number, field: number, nodeId: number, len: number) => any;
28
28
  };
29
29
  export default native;
@@ -17,13 +17,6 @@ function SelvaIoErrlogToString(buf) {
17
17
  // then add ThreadCtx to modify ctx and query ctx
18
18
  const native = {
19
19
  threadCtx: null, // add compressors here as well!
20
- historyAppend(history, typeId, nodeId, dbCtx) {
21
- return db.historyAppend(history, typeId, nodeId, dbCtx);
22
- },
23
- historyCreate(pathname, mainLen) {
24
- const pathBuf = ENCODER.encode(pathname + '\0');
25
- return db.historyCreate(pathBuf, mainLen + 16 - (mainLen % 16));
26
- },
27
20
  workerCtxInit: () => {
28
21
  return db.workerCtxInit();
29
22
  },
@@ -65,9 +58,12 @@ const native = {
65
58
  const pathBuf = ENCODER.encode(path + '\0');
66
59
  const err = db.loadBlock(pathBuf, dbCtx, selvaIoErrlog);
67
60
  if (err) {
68
- throw new Error(`Failed to load a range. selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
61
+ throw new Error(`Failed to load a range "${path}". selvaError: ${err} cause:\n${SelvaIoErrlogToString(selvaIoErrlog)}`);
69
62
  }
70
63
  },
64
+ delBlock: (dbCtx, typeId, block) => {
65
+ db.delBlock(dbCtx, typeId, block);
66
+ },
71
67
  updateSchemaType: (prefix, buf, dbCtx) => {
72
68
  return db.updateSchema(prefix, buf, dbCtx);
73
69
  },
@@ -104,6 +100,9 @@ const native = {
104
100
  membarSyncWrite: () => {
105
101
  db.membarSyncWrite();
106
102
  },
103
+ colvecTest: (dbCtx, typeId, field, nodeId, len) => {
104
+ return db.colvecTest(dbCtx, typeId, field, nodeId, len);
105
+ }
107
106
  };
108
107
  global.__basedDb__native__ = native;
109
108
  export default native;
@@ -0,0 +1,8 @@
1
+ import { DbWorker } from './workers/DbWorker.js';
2
+ import { DbServer } from './index.js';
3
+ export declare class IoWorker extends DbWorker {
4
+ constructor(address: BigInt, db: DbServer);
5
+ handleMsg(_buf: any): void;
6
+ loadBlock(filepath: string): Promise<void>;
7
+ unloadBlock(filepath: string, typeId: number, start: number): Promise<Uint8Array>;
8
+ }
@@ -0,0 +1,39 @@
1
+ import { DbWorker } from './workers/DbWorker.js';
2
+ import { DECODER, readInt32 } from '@saulx/utils';
3
+ export class IoWorker extends DbWorker {
4
+ constructor(address, db) {
5
+ const onExit = (_code) => {
6
+ this.db.ioWorker = new IoWorker(address, db);
7
+ };
8
+ super(address, db, onExit, 'io_worker.js');
9
+ }
10
+ handleMsg(_buf) {
11
+ }
12
+ async loadBlock(filepath) {
13
+ const job = {
14
+ type: 'load',
15
+ filepath
16
+ };
17
+ const resBuf = await this.call(job);
18
+ if (resBuf.length) {
19
+ throw new Error(DECODER.decode(resBuf));
20
+ }
21
+ }
22
+ async unloadBlock(filepath, typeId, start) {
23
+ const job = {
24
+ type: 'unload',
25
+ filepath,
26
+ typeId,
27
+ start
28
+ };
29
+ const resBuf = await this.call(job);
30
+ const err = readInt32(resBuf, 0);
31
+ if (err) {
32
+ throw new Error(`selva error: ${err}`);
33
+ }
34
+ // Note that this shares the original buffer which may not be 100% optimal,
35
+ // as the first 4 bytes are no longer needed.
36
+ return new Uint8Array(resBuf.buffer, 4);
37
+ }
38
+ }
39
+ //# sourceMappingURL=IoWorker.js.map
@@ -0,0 +1,8 @@
1
+ import { DbWorker } from './workers/DbWorker.js';
2
+ import { DbServer } from './index.js';
3
+ export declare class QueryWorker extends DbWorker {
4
+ constructor(address: BigInt, db: DbServer, workerIndex: number);
5
+ handleMsg(_buf: any): void;
6
+ protected callback: (resolve: (x: any) => any) => void;
7
+ getQueryBuf(buf: Uint8Array): Promise<Uint8Array>;
8
+ }
@@ -0,0 +1,26 @@
1
+ import { DbWorker } from './workers/DbWorker.js';
2
+ import { readUint64 } from '@saulx/utils';
3
+ export class QueryWorker extends DbWorker {
4
+ constructor(address, db, workerIndex) {
5
+ const onExit = (_code) => {
6
+ this.db.workers[workerIndex] = new QueryWorker(address, db, workerIndex);
7
+ };
8
+ super(address, db, onExit, 'query_worker.js');
9
+ }
10
+ handleMsg(_buf) {
11
+ this.db.processingQueries--;
12
+ this.db.onQueryEnd();
13
+ }
14
+ callback = (resolve) => {
15
+ this.db.processingQueries++;
16
+ this.resolvers.push(resolve);
17
+ };
18
+ getQueryBuf(buf) {
19
+ const schemaChecksum = readUint64(buf, buf.byteLength - 8);
20
+ if (schemaChecksum !== this.db.schema?.hash) {
21
+ return Promise.resolve(new Uint8Array(1));
22
+ }
23
+ return this.call(buf);
24
+ }
25
+ }
26
+ //# sourceMappingURL=QueryWorker.js.map
@@ -0,0 +1,24 @@
1
+ import { SchemaTypeDef } from '@based/schema/def';
2
+ import { DbServer } from './index.js';
3
+ /**
4
+ * Save a block.
5
+ */
6
+ export declare function saveBlock(db: DbServer, typeId: number, start: number, end: number): void;
7
+ /**
8
+ * Load a block (typically of a partial type) back to memory.
9
+ */
10
+ export declare function loadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
11
+ /**
12
+ * Save a block and remove it from memory.
13
+ */
14
+ export declare function unloadBlock(db: DbServer, def: SchemaTypeDef, start: number): Promise<void>;
15
+ /**
16
+ * Execute cb() for each block in memory.
17
+ */
18
+ export declare function foreachBlock(db: DbServer, def: SchemaTypeDef, cb: (start: number, end: number, hash: Uint8Array) => void, includeEmptyBlocks?: boolean): void;
19
+ /**
20
+ * Execute cb() for each dirty block.
21
+ * A dirty block is one that is changed in memory but not yet persisted in the
22
+ * file system.
23
+ */
24
+ export declare function foreachDirtyBlock(db: DbServer, cb: (mtKey: number, typeId: number, start: number, end: number) => void): void;