@based/db 0.0.21 → 0.0.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +10 -10
  2. package/dist/lib/darwin_aarch64/include/selva/types.h +9 -18
  3. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  4. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  5. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  6. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  7. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  8. package/dist/lib/linux_aarch64/include/selva/types.h +9 -18
  9. package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
  10. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  11. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  12. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  13. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  14. package/dist/lib/linux_aarch64/libselva.so +0 -0
  15. package/dist/lib/linux_x86_64/include/selva/types.h +9 -18
  16. package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
  17. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  18. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  19. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  20. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  21. package/dist/lib/linux_x86_64/libselva.so +0 -0
  22. package/dist/src/client/bitWise.d.ts +0 -1
  23. package/dist/src/client/bitWise.js +0 -10
  24. package/dist/src/client/{operations.d.ts → flushModify.d.ts} +4 -4
  25. package/dist/src/client/{operations.js → flushModify.js} +39 -15
  26. package/dist/src/client/index.d.ts +10 -6
  27. package/dist/src/client/index.js +12 -4
  28. package/dist/src/client/modify/ModifyRes.d.ts +2 -8
  29. package/dist/src/client/modify/ModifyRes.js +23 -29
  30. package/dist/src/client/modify/alias.d.ts +1 -1
  31. package/dist/src/client/modify/alias.js +5 -2
  32. package/dist/src/client/modify/binary.d.ts +1 -1
  33. package/dist/src/client/modify/binary.js +1 -1
  34. package/dist/src/client/modify/cardinality.d.ts +2 -2
  35. package/dist/src/client/modify/cardinality.js +7 -5
  36. package/dist/src/client/modify/create.js +10 -5
  37. package/dist/src/client/modify/delete.d.ts +3 -1
  38. package/dist/src/client/modify/delete.js +12 -5
  39. package/dist/src/client/modify/expire.js +1 -1
  40. package/dist/src/client/modify/fixed.js +11 -4
  41. package/dist/src/client/modify/references/appendRefs.d.ts +4 -0
  42. package/dist/src/client/modify/references/appendRefs.js +27 -0
  43. package/dist/src/client/modify/references/edge.d.ts +0 -1
  44. package/dist/src/client/modify/references/edge.js +191 -71
  45. package/dist/src/client/modify/references/getEdgeSize.d.ts +3 -0
  46. package/dist/src/client/modify/references/getEdgeSize.js +27 -0
  47. package/dist/src/client/modify/references/reference.js +40 -26
  48. package/dist/src/client/modify/references/references.js +18 -6
  49. package/dist/src/client/modify/string.js +2 -1
  50. package/dist/src/client/modify/types.d.ts +1 -0
  51. package/dist/src/client/modify/types.js +1 -0
  52. package/dist/src/client/modify/update.js +3 -4
  53. package/dist/src/client/modify/upsert.js +1 -0
  54. package/dist/src/client/modify/vector.js +0 -2
  55. package/dist/src/client/query/BasedDbQuery.d.ts +4 -3
  56. package/dist/src/client/query/BasedDbQuery.js +12 -3
  57. package/dist/src/client/query/aggregationFn.d.ts +3 -0
  58. package/dist/src/client/query/aggregationFn.js +9 -0
  59. package/dist/src/client/query/debug.js +2 -6
  60. package/dist/src/client/query/display.js +5 -2
  61. package/dist/src/client/query/filter/createFixedFilterBuffer.d.ts +2 -2
  62. package/dist/src/client/query/filter/createFixedFilterBuffer.js +34 -26
  63. package/dist/src/client/query/filter/createReferenceFilter.d.ts +1 -1
  64. package/dist/src/client/query/filter/createReferenceFilter.js +17 -7
  65. package/dist/src/client/query/filter/createVariableFilterBuffer.d.ts +1 -1
  66. package/dist/src/client/query/filter/createVariableFilterBuffer.js +45 -22
  67. package/dist/src/client/query/filter/filter.d.ts +1 -1
  68. package/dist/src/client/query/filter/filter.js +5 -0
  69. package/dist/src/client/query/filter/parseFilterValue.js +1 -1
  70. package/dist/src/client/query/filter/toBuffer.d.ts +2 -2
  71. package/dist/src/client/query/filter/toBuffer.js +16 -9
  72. package/dist/src/client/query/include/props.js +18 -1
  73. package/dist/src/client/query/include/toBuffer.d.ts +1 -1
  74. package/dist/src/client/query/include/toBuffer.js +22 -12
  75. package/dist/src/client/query/include/walk.js +2 -1
  76. package/dist/src/client/query/query.d.ts +1 -0
  77. package/dist/src/client/query/query.js +1 -0
  78. package/dist/src/client/query/queryDef.js +2 -1
  79. package/dist/src/client/query/read/read.d.ts +2 -1
  80. package/dist/src/client/query/read/read.js +95 -62
  81. package/dist/src/client/query/registerQuery.d.ts +1 -1
  82. package/dist/src/client/query/registerQuery.js +2 -1
  83. package/dist/src/client/query/search/index.d.ts +1 -1
  84. package/dist/src/client/query/search/index.js +56 -24
  85. package/dist/src/client/query/subscription/markers.js +2 -1
  86. package/dist/src/client/query/toBuffer.d.ts +1 -1
  87. package/dist/src/client/query/toBuffer.js +80 -32
  88. package/dist/src/client/query/types.d.ts +25 -3
  89. package/dist/src/client/query/types.js +6 -0
  90. package/dist/src/client/query/validation.js +1 -1
  91. package/dist/src/client/string.d.ts +1 -1
  92. package/dist/src/client/string.js +1 -2
  93. package/dist/src/client/xxHash64.d.ts +1 -1
  94. package/dist/src/index.d.ts +4 -3
  95. package/dist/src/index.js +20 -11
  96. package/dist/src/native.d.ts +8 -13
  97. package/dist/src/native.js +9 -31
  98. package/dist/src/server/dbHash.d.ts +5 -0
  99. package/dist/src/server/dbHash.js +27 -0
  100. package/dist/src/server/index.d.ts +10 -10
  101. package/dist/src/server/index.js +54 -35
  102. package/dist/src/server/migrate/index.js +1 -1
  103. package/dist/src/server/migrate/worker.js +2 -2
  104. package/dist/src/server/save.d.ts +1 -1
  105. package/dist/src/server/save.js +10 -10
  106. package/dist/src/server/start.js +4 -2
  107. package/dist/src/server/tree.d.ts +1 -1
  108. package/dist/src/server/tree.js +1 -1
  109. package/dist/src/utils.d.ts +3 -0
  110. package/dist/src/utils.js +23 -5
  111. package/package.json +3 -2
  112. package/dist/src/client/query/read/types.d.ts +0 -4
  113. package/dist/src/client/query/read/types.js +0 -5
@@ -5,4 +5,4 @@ export type Search = string[] | {
5
5
  } | string;
6
6
  export declare const vectorSearch: (def: QueryDef, q: ArrayBufferView, field: string, opts: Omit<FilterOpts, "lowerCase">) => void;
7
7
  export declare const search: (def: QueryDef, q: string, s?: Search) => void;
8
- export declare const searchToBuffer: (search: QueryDefSearch) => Buffer;
8
+ export declare const searchToBuffer: (search: QueryDefSearch) => Uint8Array;
@@ -2,7 +2,7 @@ import { langCodesMap } from '@based/schema';
2
2
  import { STRING, TEXT, VECTOR } from '@based/schema/def';
3
3
  import { getVectorFn } from '../filter/types.js';
4
4
  import { searchDoesNotExist, searchIncorrecQueryValue, searchIncorrectType, } from '../validation.js';
5
- const ENCODER = new TextEncoder();
5
+ import { ENCODER, concatUint8Arr } from '../../../utils.js';
6
6
  // vector
7
7
  export const vectorSearch = (def, q, field, opts) => {
8
8
  let prop = def.props[field];
@@ -24,16 +24,6 @@ export const vectorSearch = (def, q, field, opts) => {
24
24
  opts,
25
25
  };
26
26
  };
27
- function concatBufs(bufs, totalByteLength) {
28
- const res = new Uint8Array(totalByteLength);
29
- let off = 0;
30
- for (let i = 0; i < bufs.length; i++) {
31
- const buf = bufs[i];
32
- res.set(buf, off);
33
- off += buf.byteLength;
34
- }
35
- return res;
36
- }
37
27
  export const search = (def, q, s) => {
38
28
  const bufs = [];
39
29
  let nrBlocks = 0;
@@ -42,7 +32,12 @@ export const search = (def, q, s) => {
42
32
  searchIncorrecQueryValue(def, q);
43
33
  q = '';
44
34
  }
45
- const x = q.toLowerCase().normalize('NFKD').trim().split(' ').map((s) => ` ${s}`);
35
+ const x = q
36
+ .toLowerCase()
37
+ .normalize('NFKD')
38
+ .trim()
39
+ .split(' ')
40
+ .map((s) => ` ${s}`);
46
41
  for (const s of x) {
47
42
  if (s) {
48
43
  const buf = ENCODER.encode(s);
@@ -55,7 +50,7 @@ export const search = (def, q, s) => {
55
50
  }
56
51
  }
57
52
  bufs.unshift(Uint8Array.from([nrBlocks]));
58
- const query = concatBufs(bufs, totalByteLength);
53
+ const query = concatUint8Arr(bufs, totalByteLength);
59
54
  def.search = {
60
55
  size: query.byteLength + 3,
61
56
  query,
@@ -104,8 +99,9 @@ export const search = (def, q, s) => {
104
99
  if (prop.typeIndex !== STRING && prop.typeIndex !== TEXT) {
105
100
  searchIncorrectType(def, prop);
106
101
  }
107
- def.search.size += 5;
102
+ def.search.size += 6;
108
103
  def.search.fields.push({
104
+ typeIndex: prop.typeIndex,
109
105
  weight: s[key],
110
106
  lang,
111
107
  field: prop.prop,
@@ -115,10 +111,20 @@ export const search = (def, q, s) => {
115
111
  };
116
112
  export const searchToBuffer = (search) => {
117
113
  if (search.isVector) {
118
- // [isVec] [q len] [q len] [field] [fn] [score] [score] [score] [score] [q..]
119
- const result = Buffer.allocUnsafe(search.size);
114
+ /* Vector Binary Schema:
115
+ | Offset | Field | Size (bytes) | Description |
116
+ |--------|----------|--------------|-------------------------------------------------|
117
+ | 0 | isVector | 1 | Indicates if search is a vector (always 1) |
118
+ | 1 | queryLen | 2 | Length of the query in bytes (u16) |
119
+ | 3 | field | 1 | Field identifier |
120
+ | 4 | func | 1 | Function identifier (enum) |
121
+ | 5 | score | 4 | Score value (f32) |
122
+ | 9 | query | queryLen | Query data (array of f32 values) |
123
+ */
124
+ const result = new Uint8Array(search.size);
120
125
  result[0] = 1; // search.isVector 1
121
- result.writeUint16LE(search.query.byteLength, 1);
126
+ result[1] = search.query.byteLength;
127
+ result[2] = search.query.byteLength >>> 8;
122
128
  result[3] = search.prop;
123
129
  result[4] = getVectorFn(search.opts.fn);
124
130
  result.set(new Uint8Array(new Float32Array([search.opts.score ?? 0.5]).buffer), 5);
@@ -126,9 +132,33 @@ export const searchToBuffer = (search) => {
126
132
  return result;
127
133
  }
128
134
  else {
129
- const result = Buffer.allocUnsafe(search.size);
135
+ /* Non-Vector Search Binary Schema:
136
+
137
+ | Offset | Field | Size (bytes) | Description |
138
+ |--------|----------|--------------|--------------------------------------------|
139
+ | 0 | isVector | 1 | Indicates if search is a vector (always 0) |
140
+ | 1 | queryLen | 2 | Length of the query in bytes (u16) |
141
+ | 3 | query | queryLen | Query data |
142
+ | X | fields | Variable | Sorted fields metadata |
143
+
144
+ ### Fields Metadata Structure:
145
+ Each field entry consists of 6 bytes:
146
+
147
+ | Offset | Field | Size (bytes)| Description |
148
+ |--------|-----------|-------------|--------------------------------------|
149
+ | 0 | field | 1 | Field identifier |
150
+ | 1 | typeIndex | 1 | Type index of the field |
151
+ | 2 | weight | 1 | Field weight value |
152
+ | 3 | start | 2 | Start position in the query (u16) |
153
+ | 5 | lang | 1 | Language identifier |
154
+
155
+ ### Notes:
156
+ - The number of field entries is inferred from the total packet size.
157
+ - `fields` are sorted by `weight` before being stored in the buffer.*/
158
+ const result = new Uint8Array(search.size);
130
159
  result[0] = 0; // search.isVector 0
131
- result.writeUint16LE(search.query.byteLength, 1);
160
+ result[1] = search.query.byteLength;
161
+ result[2] = search.query.byteLength >>> 8;
132
162
  result.set(search.query, 3);
133
163
  const offset = search.query.byteLength + 3;
134
164
  // @ts-ignore
@@ -136,13 +166,15 @@ export const searchToBuffer = (search) => {
136
166
  return a.weight - b.weight;
137
167
  });
138
168
  // @ts-ignore
139
- for (let i = 0; i < search.fields.length * 5; i += 5) {
169
+ for (let i = 0; i < search.fields.length * 6; i += 6) {
140
170
  // @ts-ignore
141
- const f = search.fields[i / 5];
171
+ const f = search.fields[Math.floor(i / 6)];
142
172
  result[i + offset] = f.field;
143
- result[i + 1 + offset] = f.weight;
144
- result.writeUInt16LE(f.start, i + 2 + offset);
145
- result[i + 4 + offset] = f.lang;
173
+ result[i + offset + 1] = f.typeIndex;
174
+ result[i + offset + 2] = f.weight;
175
+ result[i + offset + 3] = f.start;
176
+ result[i + offset + 4] = f.start >>> 8;
177
+ result[i + offset + 5] = f.lang;
146
178
  }
147
179
  return result;
148
180
  }
@@ -95,7 +95,8 @@ const getFilterFields = (filter, results = {
95
95
  conditions.forEach((v, k) => {
96
96
  if (k === 0) {
97
97
  for (const buf of v) {
98
- results.main.add(buf.readUint16LE(4));
98
+ const x = buf[4] | buf[5] << 8;
99
+ results.main.add(x);
99
100
  }
100
101
  }
101
102
  else {
@@ -1,3 +1,3 @@
1
1
  import { QueryDef } from './types.js';
2
2
  import { DbClient } from '../index.js';
3
- export declare function defToBuffer(db: DbClient, def: QueryDef): Buffer[];
3
+ export declare function defToBuffer(db: DbClient, def: QueryDef): Uint8Array[];
@@ -3,6 +3,8 @@ import { QueryDefType } from './types.js';
3
3
  import { includeToBuffer } from './include/toBuffer.js';
4
4
  import { filterToBuffer } from './query.js';
5
5
  import { searchToBuffer } from './search/index.js';
6
+ import { createAggFnBuffer } from './aggregationFn.js';
7
+ import { ENCODER } from '../../utils.js';
6
8
  const byteSize = (arr) => {
7
9
  return arr.reduce((a, b) => {
8
10
  return a + b.byteLength;
@@ -48,18 +50,21 @@ export function defToBuffer(db, def) {
48
50
  if (def.target.resolvedAlias) {
49
51
  // put this somehwere else at some point
50
52
  const alias = def.target.resolvedAlias;
51
- const s = Buffer.byteLength(alias.value);
53
+ const aliasStr = ENCODER.encode(alias.value);
54
+ const aliasLen = aliasStr.byteLength;
52
55
  // filter is nice for things like access
53
- const buf = Buffer.allocUnsafe(8 + filterSize + s);
56
+ const buf = new Uint8Array(8 + filterSize + aliasLen);
54
57
  buf[0] = 3;
55
58
  buf[1] = def.schema.idUint8[0];
56
59
  buf[2] = def.schema.idUint8[1];
57
60
  buf[3] = alias.def.prop;
58
- buf.writeUint16LE(s, 4);
59
- buf.write(alias.value, 6);
60
- buf.writeUint16LE(filterSize, s + 6);
61
+ buf[4] = aliasLen;
62
+ buf[5] = aliasLen >>> 8;
63
+ buf.set(aliasStr, 6);
64
+ buf[6 + aliasLen] = filterSize;
65
+ buf[7 + aliasLen] = filterSize >>> 8;
61
66
  if (filterSize) {
62
- buf.set(filter, 8 + s);
67
+ buf.set(filter, 8 + aliasLen);
63
68
  }
64
69
  result.push(buf);
65
70
  }
@@ -67,12 +72,16 @@ export function defToBuffer(db, def) {
67
72
  // type 0
68
73
  // 0: 4 [id]
69
74
  // 0: 2 [filterSize]
70
- const buf = Buffer.allocUnsafe(9 + filterSize);
75
+ const buf = new Uint8Array(9 + filterSize);
71
76
  buf[0] = 0;
72
77
  buf[1] = def.schema.idUint8[0];
73
78
  buf[2] = def.schema.idUint8[1];
74
- buf.writeUInt32LE(def.target.id, 3);
75
- buf.writeUint16LE(filterSize, 7);
79
+ buf[3] = def.target.id;
80
+ buf[4] = def.target.id >>> 8;
81
+ buf[5] = def.target.id >>> 16;
82
+ buf[6] = def.target.id >>> 24;
83
+ buf[7] = filterSize;
84
+ buf[8] = filterSize >>> 8;
76
85
  if (filterSize) {
77
86
  buf.set(filter, 9);
78
87
  }
@@ -85,6 +94,8 @@ export function defToBuffer(db, def) {
85
94
  sort = createSortBuffer(def.sort);
86
95
  sortSize = sort.byteLength;
87
96
  }
97
+ let aggregation;
98
+ aggregation = createAggFnBuffer(def.aggregation);
88
99
  if (def.target.ids) {
89
100
  // type 1
90
101
  // 1: 4 + ids * 4 [ids len] [id,id,id]
@@ -98,23 +109,35 @@ export function defToBuffer(db, def) {
98
109
  def.target.ids = def.target.ids.slice(def.range.offset, def.range.offset + def.range.limit);
99
110
  }
100
111
  const idsSize = def.target.ids.length * 4;
101
- const buf = Buffer.allocUnsafe(21 + idsSize + filterSize + sortSize + searchSize);
112
+ const buf = new Uint8Array(21 + idsSize + filterSize + sortSize + searchSize);
102
113
  buf[0] = 1;
103
114
  buf[1] = def.schema.idUint8[0];
104
115
  buf[2] = def.schema.idUint8[1];
105
- buf.writeUint32LE(idsSize, 3);
116
+ buf[3] = idsSize;
117
+ buf[4] = idsSize >>> 8;
118
+ buf[5] = idsSize >>> 16;
119
+ buf[6] = idsSize >>> 24;
106
120
  buf.set(new Uint8Array(def.target.ids.buffer), 7);
107
- buf.writeUint32LE(def.range.offset, idsSize + 7);
108
- buf.writeUint32LE(def.range.limit, idsSize + 11);
109
- buf.writeUint16LE(filterSize, idsSize + 15);
121
+ buf[idsSize + 7] = def.range.offset;
122
+ buf[idsSize + 8] = def.range.offset >>> 8;
123
+ buf[idsSize + 9] = def.range.offset >>> 16;
124
+ buf[idsSize + 10] = def.range.offset >>> 24;
125
+ buf[idsSize + 11] = def.range.limit;
126
+ buf[idsSize + 12] = def.range.limit >>> 8;
127
+ buf[idsSize + 13] = def.range.limit >>> 16;
128
+ buf[idsSize + 14] = def.range.limit >>> 24;
129
+ buf[idsSize + 15] = filterSize;
130
+ buf[idsSize + 16] = filterSize >>> 8;
110
131
  if (filterSize) {
111
132
  buf.set(filter, idsSize + 17);
112
133
  }
113
- buf.writeUint16LE(sortSize, 17 + filterSize + idsSize);
134
+ buf[17 + filterSize + idsSize] = sortSize;
135
+ buf[18 + filterSize + idsSize] = sortSize >>> 8;
114
136
  if (sortSize) {
115
137
  buf.set(sort, 19 + filterSize + idsSize);
116
138
  }
117
- buf.writeUint16LE(searchSize, 19 + filterSize + idsSize + sortSize);
139
+ buf[19 + filterSize + idsSize + sortSize] = searchSize;
140
+ buf[20 + filterSize + idsSize + sortSize] = searchSize >>> 8;
118
141
  if (searchSize) {
119
142
  buf.set(search, 21 + filterSize + idsSize + sortSize);
120
143
  }
@@ -129,24 +152,36 @@ export function defToBuffer(db, def) {
129
152
  // ?filter
130
153
  // 2: 2 [sort size]
131
154
  // ?sort
132
- const buf = Buffer.allocUnsafe(17 + filterSize + sortSize + searchSize);
155
+ const buf = new Uint8Array(18 + filterSize + sortSize + searchSize);
133
156
  buf[0] = 2;
134
157
  buf[1] = def.schema.idUint8[0];
135
158
  buf[2] = def.schema.idUint8[1];
136
- buf.writeUint32LE(def.range.offset, 3);
137
- buf.writeUint32LE(def.range.limit, 7);
138
- buf.writeUint16LE(filterSize, 11);
159
+ buf[3] = def.range.offset;
160
+ buf[4] = def.range.offset >>> 8;
161
+ buf[5] = def.range.offset >>> 16;
162
+ buf[6] = def.range.offset >>> 24;
163
+ buf[7] = def.range.limit;
164
+ buf[8] = def.range.limit >>> 8;
165
+ buf[9] = def.range.limit >>> 16;
166
+ buf[10] = def.range.limit >>> 24;
167
+ buf[11] = filterSize;
168
+ buf[12] = filterSize >>> 8;
139
169
  if (filterSize) {
140
170
  buf.set(filter, 13);
141
171
  }
142
- buf.writeUint16LE(sortSize, 13 + filterSize);
172
+ buf[13 + filterSize] = sortSize;
173
+ buf[14 + filterSize] = sortSize >>> 8;
143
174
  if (sortSize) {
144
175
  buf.set(sort, 15 + filterSize);
145
176
  }
146
- buf.writeUint16LE(searchSize, 15 + filterSize + sortSize);
177
+ buf[15 + filterSize + sortSize] = searchSize;
178
+ buf[16 + filterSize + sortSize] = searchSize >>> 8;
147
179
  if (searchSize) {
148
180
  buf.set(search, 17 + filterSize + sortSize);
149
181
  }
182
+ if (aggregation) {
183
+ buf.set(aggregation, 17 + filterSize + sortSize + searchSize);
184
+ }
150
185
  result.push(buf);
151
186
  }
152
187
  }
@@ -166,13 +201,23 @@ export function defToBuffer(db, def) {
166
201
  const sortSize = sort?.byteLength ?? 0;
167
202
  const filterSize = filter?.byteLength ?? 0;
168
203
  const modsSize = filterSize + sortSize;
169
- const meta = Buffer.allocUnsafe(modsSize + 10 + 8);
204
+ const meta = new Uint8Array(modsSize + 10 + 8);
205
+ const sz = size + 7 + modsSize + 8;
170
206
  meta[0] = 254;
171
- meta.writeUint16LE(size + 7 + modsSize + 8, 1);
172
- meta.writeUint16LE(filterSize, 3);
173
- meta.writeUint16LE(sortSize, 5);
174
- meta.writeUint32LE(def.range.offset, 7);
175
- meta.writeUint32LE(def.range.limit, 7 + 4);
207
+ meta[1] = sz;
208
+ meta[2] = sz >>> 8;
209
+ meta[3] = filterSize;
210
+ meta[4] = filterSize >>> 8;
211
+ meta[5] = sortSize;
212
+ meta[6] = sortSize >>> 8;
213
+ meta[7] = def.range.offset;
214
+ meta[8] = def.range.offset >>> 8;
215
+ meta[9] = def.range.offset >>> 16;
216
+ meta[10] = def.range.offset >>> 24;
217
+ meta[11] = def.range.limit;
218
+ meta[12] = def.range.limit >>> 8;
219
+ meta[13] = def.range.limit >>> 16;
220
+ meta[14] = def.range.limit >>> 24;
176
221
  if (filter) {
177
222
  meta.set(filter, 15);
178
223
  }
@@ -185,9 +230,11 @@ export function defToBuffer(db, def) {
185
230
  result.push(meta);
186
231
  }
187
232
  else if (def.type === QueryDefType.Reference) {
188
- const meta = Buffer.allocUnsafe(6);
233
+ const meta = new Uint8Array(6);
234
+ const sz = size + 3;
189
235
  meta[0] = 255;
190
- meta.writeUint16LE(size + 3, 1);
236
+ meta[1] = sz;
237
+ meta[2] = sz >>> 8;
191
238
  meta[3] = def.schema.idUint8[0];
192
239
  meta[4] = def.schema.idUint8[1];
193
240
  meta[5] = def.target.propDef.prop;
@@ -195,9 +242,10 @@ export function defToBuffer(db, def) {
195
242
  }
196
243
  result.push(...include);
197
244
  if (edges) {
198
- const metaEdgeBuffer = Buffer.allocUnsafe(3);
245
+ const metaEdgeBuffer = new Uint8Array(3);
199
246
  metaEdgeBuffer[0] = 252;
200
- metaEdgeBuffer.writeUint16LE(edgesSize, 1);
247
+ metaEdgeBuffer[1] = edgesSize;
248
+ metaEdgeBuffer[2] = edgesSize >>> 8;
201
249
  result.push(metaEdgeBuffer, ...edges);
202
250
  }
203
251
  return result;
@@ -29,7 +29,7 @@ export type Target = {
29
29
  export declare const isRefDef: (def: QueryDef) => def is QueryDefRest;
30
30
  export type QueryDefFilter = {
31
31
  size: number;
32
- conditions: Map<number, Buffer[]>;
32
+ conditions: Map<number, Uint8Array[]>;
33
33
  exists?: {
34
34
  prop: PropDef | PropDefEdge;
35
35
  negate: boolean;
@@ -37,7 +37,7 @@ export type QueryDefFilter = {
37
37
  references?: Map<number, QueryDefFilter>;
38
38
  fromRef?: PropDef;
39
39
  schema?: SchemaTypeDef;
40
- edges?: Map<number, Buffer[]>;
40
+ edges?: Map<number, Uint8Array[]>;
41
41
  or?: QueryDefFilter;
42
42
  and?: QueryDefFilter;
43
43
  };
@@ -50,6 +50,7 @@ export type QueryDefSearch = {
50
50
  field: number;
51
51
  start: number;
52
52
  lang: LangCode;
53
+ typeIndex: number;
53
54
  }[];
54
55
  } | {
55
56
  size: number;
@@ -69,7 +70,6 @@ export type QueryDefShared = {
69
70
  filter: QueryDefFilter;
70
71
  search: null | QueryDefSearch;
71
72
  sort: null | QueryDefSort;
72
- reverseProps: any;
73
73
  skipValidation: boolean;
74
74
  range: {
75
75
  offset: number;
@@ -92,6 +92,7 @@ export type QueryDefShared = {
92
92
  };
93
93
  references: Map<number, QueryDef>;
94
94
  edges?: QueryDef;
95
+ aggregation: AggFn;
95
96
  };
96
97
  export type QueryDefEdges = {
97
98
  type: QueryDefType.Edge;
@@ -113,3 +114,24 @@ export type QueryByAliasObj = {
113
114
  [key: string]: string | QueryByAliasObj;
114
115
  };
115
116
  export declare const isAlias: (id: QueryByAliasObj | number | Uint32Array | (QueryByAliasObj | number)[]) => id is QueryByAliasObj;
117
+ export declare const READ_ID = 255;
118
+ export declare const READ_EDGE = 252;
119
+ export declare const READ_REFERENCES = 253;
120
+ export declare const READ_REFERENCE = 254;
121
+ export declare const CREATE_AGGREGATION = 250;
122
+ export declare const READ_AGGREGATION = 251;
123
+ export declare const enum AggFn {
124
+ AVG = 1,
125
+ CARDINALITY = 2,
126
+ CONCAT = 3,// string aggregation, delimiter should be an argument
127
+ COUNT = 4,
128
+ MAX = 5,
129
+ MIN = 6,
130
+ MODE = 7,// ordered-set
131
+ PERCENTILE = 8,// continuous or discrete should be optional parameters, default = discrete
132
+ RANK = 9,// hypothetical-set, dense should be optional parameter
133
+ STDDEV = 10,// population or sample should be optional parameters, default = sample
134
+ SUM = 11,
135
+ VARIANCE = 12,
136
+ NONE = -999
137
+ }
@@ -18,4 +18,10 @@ export const isAlias = (id) => {
18
18
  !Array.isArray(id) &&
19
19
  !ArrayBuffer.isView(id));
20
20
  };
21
+ export const READ_ID = 255;
22
+ export const READ_EDGE = 252;
23
+ export const READ_REFERENCES = 253;
24
+ export const READ_REFERENCE = 254;
25
+ export const CREATE_AGGREGATION = 250;
26
+ export const READ_AGGREGATION = 251;
21
27
  //# sourceMappingURL=types.js.map
@@ -93,7 +93,7 @@ export const isValidId = (id) => {
93
93
  };
94
94
  export const isValidString = (v) => {
95
95
  const isVal = typeof v === 'string' ||
96
- v instanceof Buffer ||
96
+ v instanceof Uint8Array ||
97
97
  ArrayBuffer.isView(v);
98
98
  return isVal;
99
99
  };
@@ -1,5 +1,5 @@
1
1
  import { LangCode } from '@based/schema';
2
- export declare const write: (buf: Buffer | Uint8Array, value: string, offset: number, noCompression: boolean, lang?: LangCode) => number;
2
+ export declare const write: (buf: Uint8Array, value: string, offset: number, noCompression: boolean, lang?: LangCode) => number;
3
3
  export declare const compress: (str: string) => Uint8Array;
4
4
  export declare const decompress: (val: Uint8Array) => string;
5
5
  export declare const read: (val: Uint8Array, offset: number, len: number) => string;
@@ -1,8 +1,7 @@
1
1
  import native from '../native.js';
2
2
  import { readUint32 } from './bitWise.js';
3
3
  import makeTmpBuffer from './tmpBuffer.js';
4
- const DECODER = new TextDecoder('utf-8');
5
- const ENCODER = new TextEncoder();
4
+ import { DECODER, ENCODER } from '../utils.js';
6
5
  const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
7
6
  // type 0 = no compression; 1 = deflate
8
7
  // [lang] [type] [uncompressed size 4] [compressed string] [crc32]
@@ -1 +1 @@
1
- export declare const xxHash64: (buf: Buffer | Uint8Array, target?: Uint8Array | Buffer, index?: number) => Uint8Array | Buffer;
1
+ export declare const xxHash64: (buf: Uint8Array, target?: Uint8Array, index?: number) => Uint8Array;
@@ -1,5 +1,5 @@
1
1
  import { compress, decompress } from './client/string.js';
2
- import { ModifyCtx } from './client/operations.js';
2
+ import { ModifyCtx } from './client/flushModify.js';
3
3
  import { DbServer } from './server/index.js';
4
4
  import { DbClient } from './client/index.js';
5
5
  export * from './client/modify/modify.js';
@@ -27,14 +27,15 @@ export declare class BasedDb {
27
27
  upsert: DbClient['upsert'];
28
28
  delete: DbClient['delete'];
29
29
  expire: DbClient['expire'];
30
- remove: DbClient['delete'];
31
30
  query: DbClient['query'];
32
- putSchema: DbClient['putSchema'];
31
+ setSchema: DbClient['setSchema'];
32
+ putSchema: DbClient['setSchema'];
33
33
  drain: DbClient['drain'];
34
34
  start: DbServer['start'];
35
35
  stop: DbServer['stop'];
36
36
  save: DbServer['save'];
37
37
  migrateSchema: DbServer['migrateSchema'];
38
+ isReady: DbClient['isModified'];
38
39
  destroy(): Promise<void>;
39
40
  wipe(): Promise<void>;
40
41
  }
package/dist/src/index.js CHANGED
@@ -1,8 +1,9 @@
1
1
  import { compress, decompress } from './client/string.js';
2
- import { ModifyCtx } from './client/operations.js';
2
+ import { ModifyCtx } from './client/flushModify.js';
3
3
  import { DbServer } from './server/index.js';
4
4
  import { DbClient } from './client/index.js';
5
5
  import picocolors from 'picocolors';
6
+ import { wait } from '@saulx/utils';
6
7
  export * from './client/modify/modify.js';
7
8
  export { compress, decompress };
8
9
  export { ModifyCtx }; // TODO move this somewhere
@@ -45,8 +46,9 @@ export class BasedDb {
45
46
  const client = new DbClient({
46
47
  maxModifySize,
47
48
  hooks: {
48
- putSchema(schema, fromStart) {
49
- return Promise.resolve(server.putSchema(schema, fromStart));
49
+ flushTime: 0,
50
+ setSchema(schema, fromStart) {
51
+ return Promise.resolve(server.setSchema(schema, fromStart));
50
52
  },
51
53
  flushModify(buf) {
52
54
  const offsets = server.modify(buf);
@@ -54,6 +56,8 @@ export class BasedDb {
54
56
  offsets,
55
57
  });
56
58
  },
59
+ flushReady: () => { },
60
+ flushIsReady: new Promise(() => { }),
57
61
  getQueryBuf(buf) {
58
62
  return Promise.resolve(server.getQueryBuf(buf));
59
63
  },
@@ -81,15 +85,15 @@ export class BasedDb {
81
85
  expire = function () {
82
86
  return this.client.expire.apply(this.client, arguments);
83
87
  };
84
- remove = function () {
85
- console.warn('WARNING: db.remove() is deprecated. Use db.delete() instead');
86
- return this.client.delete.apply(this.client, arguments);
87
- };
88
88
  query = function () {
89
89
  return this.client.query.apply(this.client, arguments);
90
90
  };
91
+ setSchema = function () {
92
+ return this.client.setSchema.apply(this.client, arguments);
93
+ };
91
94
  putSchema = function () {
92
- return this.client.putSchema.apply(this.client, arguments);
95
+ console.warn('URGENT: putSchema will be removed in next release. Use setSchema instead!');
96
+ return this.setSchema.apply(this, arguments);
93
97
  };
94
98
  drain = function () {
95
99
  return this.client.drain.apply(this.client, arguments);
@@ -108,10 +112,15 @@ export class BasedDb {
108
112
  migrateSchema = function () {
109
113
  return this.server.migrateSchema.apply(this.server, arguments);
110
114
  };
111
- // both
112
- destroy() {
115
+ isReady = function () {
116
+ return this.client.isReady.apply(this.client, arguments);
117
+ };
118
+ async destroy() {
119
+ // Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
120
+ // Todo: clear all active queries, queues ETC
121
+ await wait(Math.max(this.client.hooks.flushTime + 10, 10));
113
122
  this.client.destroy();
114
- return this.server.destroy();
123
+ await this.server.destroy();
115
124
  }
116
125
  async wipe() {
117
126
  const opts = {
@@ -4,8 +4,8 @@ declare const _default: {
4
4
  workerCtxInit: () => void;
5
5
  externalFromInt(address: BigInt): any;
6
6
  intFromExternal(external: any): BigInt;
7
- modify: (data: Buffer, types: Buffer, dbCtx: any) => any;
8
- getQueryBuf: (q: Buffer, dbCtx: any) => ArrayBuffer | null;
7
+ modify: (data: Uint8Array, types: Uint8Array, dbCtx: any) => any;
8
+ getQueryBuf: (q: Uint8Array, dbCtx: any) => ArrayBuffer | null;
9
9
  start: (id: number) => any;
10
10
  stop: (dbCtx: any) => any;
11
11
  saveCommon: (path: string, dbCtx: any) => number;
@@ -15,17 +15,12 @@ declare const _default: {
15
15
  updateSchemaType: (prefix: number, buf: Uint8Array, dbCtx: any) => any;
16
16
  getTypeInfo: (typeId: number, dbCtx: any) => any;
17
17
  getNodeRangeHash: (typeId: number, start: number, end: number, bufOut: Uint8Array, dbCtx: any) => any;
18
- createHash: () => {
19
- update: (buf: Buffer | Uint8Array) => any;
20
- digest: (encoding?: "hex") => Uint8Array | string;
21
- reset: () => void;
22
- };
23
- compress: (buf: Buffer | Uint8Array, offset: number, stringSize: number) => any;
24
- decompress: (input: Buffer | Uint8Array, output: Buffer | Uint8Array, offset: number, len: number) => any;
25
- crc32: (buf: Buffer | Uint8Array) => any;
26
- createSortIndex: (buf: Buffer, dbCtx: any) => any;
27
- destroySortIndex: (buf: Buffer, dbCtx: any) => any;
28
- xxHash64: (buf: Buffer | Uint8Array, target: Buffer | Uint8Array, index: number) => any;
18
+ compress: (buf: Uint8Array, offset: number, stringSize: number) => any;
19
+ decompress: (input: Uint8Array, output: Uint8Array, offset: number, len: number) => any;
20
+ crc32: (buf: Uint8Array) => any;
21
+ createSortIndex: (buf: Uint8Array, dbCtx: any) => any;
22
+ destroySortIndex: (buf: Uint8Array, dbCtx: any) => any;
23
+ xxHash64: (buf: Uint8Array, target: Uint8Array, index: number) => any;
29
24
  base64encode: (dst: Uint8Array, src: Uint8Array, lineMax: number) => Uint8Array;
30
25
  equals: (a: Uint8Array, b: Uint8Array) => boolean;
31
26
  };