@based/db 0.0.31 → 0.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/README.md +565 -3
  2. package/dist/lib/darwin_aarch64/include/selva/db.h +1 -1
  3. package/dist/lib/darwin_aarch64/include/selva/fields.h +0 -2
  4. package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
  5. package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
  6. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  7. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  8. package/dist/src/client/flushModify.d.ts +1 -1
  9. package/dist/src/client/flushModify.js +5 -4
  10. package/dist/src/client/index.d.ts +3 -2
  11. package/dist/src/client/modify/binary.js +1 -1
  12. package/dist/src/client/modify/cardinality.js +1 -1
  13. package/dist/src/client/modify/references/appendEdgeRefs.js +3 -0
  14. package/dist/src/client/modify/references/edge.js +6 -0
  15. package/dist/src/client/modify/references/getEdgeSize.js +1 -1
  16. package/dist/src/client/modify/string.js +10 -4
  17. package/dist/src/client/modify/text.js +1 -9
  18. package/dist/src/client/modify/types.d.ts +1 -0
  19. package/dist/src/client/modify/types.js +1 -0
  20. package/dist/src/client/modify/upsert.js +33 -21
  21. package/dist/src/client/query/BasedDbQuery.js +1 -1
  22. package/dist/src/client/query/BasedIterable.d.ts +2 -2
  23. package/dist/src/client/query/BasedIterable.js +7 -1
  24. package/dist/src/client/query/aggregates/aggregation.d.ts +4 -0
  25. package/dist/src/client/query/aggregates/aggregation.js +12 -0
  26. package/dist/src/client/query/aggregation.d.ts +1 -1
  27. package/dist/src/client/query/debug.js +3 -2
  28. package/dist/src/client/query/display.js +1 -1
  29. package/dist/src/client/query/filter/createFixedFilterBuffer.js +1 -1
  30. package/dist/src/client/query/filter/createVariableFilterBuffer.js +1 -1
  31. package/dist/src/client/query/filter/parseFilterValue.js +1 -2
  32. package/dist/src/client/query/queryDef.js +2 -2
  33. package/dist/src/client/query/read/read.js +4 -14
  34. package/dist/src/client/query/registerQuery.js +1 -1
  35. package/dist/src/client/query/search/index.d.ts +1 -1
  36. package/dist/src/client/query/search/index.js +1 -1
  37. package/dist/src/client/query/sort.d.ts +1 -1
  38. package/dist/src/client/query/toBuffer.js +11 -15
  39. package/dist/src/client/query/types.d.ts +7 -0
  40. package/dist/src/client/query/types.js +8 -0
  41. package/dist/src/client/query/validation.d.ts +1 -1
  42. package/dist/src/client/query/validation.js +4 -5
  43. package/dist/src/client/string.js +1 -3
  44. package/dist/src/client/xxHash64.d.ts +1 -1
  45. package/dist/src/index.d.ts +1 -0
  46. package/dist/src/index.js +5 -1
  47. package/dist/src/native.d.ts +1 -2
  48. package/dist/src/native.js +2 -5
  49. package/dist/src/server/csmt/draw-dot.d.ts +3 -1
  50. package/dist/src/server/csmt/draw-dot.js +7 -2
  51. package/dist/src/server/csmt/match.d.ts +1 -1
  52. package/dist/src/server/csmt/memebership-proof.d.ts +1 -1
  53. package/dist/src/server/csmt/tree-utils.d.ts +4 -4
  54. package/dist/src/server/csmt/tree.d.ts +1 -1
  55. package/dist/src/server/csmt/tree.js +1 -1
  56. package/dist/src/server/csmt/types.d.ts +10 -10
  57. package/dist/src/server/dbHash.d.ts +1 -1
  58. package/dist/src/server/dbHash.js +1 -1
  59. package/dist/src/server/index.d.ts +8 -4
  60. package/dist/src/server/index.js +28 -13
  61. package/dist/src/server/migrate/worker.js +11 -0
  62. package/dist/src/server/save.js +3 -2
  63. package/dist/src/server/start.js +9 -5
  64. package/dist/src/utils.d.ts +0 -10
  65. package/dist/src/utils.js +0 -152
  66. package/package.json +3 -3
  67. package/dist/lib/darwin_aarch64/include/selva/xxhash64.h +0 -23
  68. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  69. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  70. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  71. package/dist/lib/linux_aarch64/include/cdefs.h +0 -317
  72. package/dist/lib/linux_aarch64/include/jemalloc.h +0 -468
  73. package/dist/lib/linux_aarch64/include/libdeflate.h +0 -322
  74. package/dist/lib/linux_aarch64/include/libdeflate_strings.h +0 -35
  75. package/dist/lib/linux_aarch64/include/linker_set.h +0 -109
  76. package/dist/lib/linux_aarch64/include/queue.h +0 -627
  77. package/dist/lib/linux_aarch64/include/selva/_export.h +0 -7
  78. package/dist/lib/linux_aarch64/include/selva/align.h +0 -9
  79. package/dist/lib/linux_aarch64/include/selva/backoff_timeout.h +0 -29
  80. package/dist/lib/linux_aarch64/include/selva/bitmap.h +0 -95
  81. package/dist/lib/linux_aarch64/include/selva/crc32c.h +0 -17
  82. package/dist/lib/linux_aarch64/include/selva/ctime.h +0 -135
  83. package/dist/lib/linux_aarch64/include/selva/db.h +0 -418
  84. package/dist/lib/linux_aarch64/include/selva/endian.h +0 -301
  85. package/dist/lib/linux_aarch64/include/selva/fast_linear_search.h +0 -23
  86. package/dist/lib/linux_aarch64/include/selva/fast_memcmp.h +0 -18
  87. package/dist/lib/linux_aarch64/include/selva/fast_memmem.h +0 -11
  88. package/dist/lib/linux_aarch64/include/selva/fast_parsei.h +0 -36
  89. package/dist/lib/linux_aarch64/include/selva/fields.h +0 -383
  90. package/dist/lib/linux_aarch64/include/selva/find.h +0 -47
  91. package/dist/lib/linux_aarch64/include/selva/history.h +0 -64
  92. package/dist/lib/linux_aarch64/include/selva/hll.h +0 -81
  93. package/dist/lib/linux_aarch64/include/selva/lpf.h +0 -28
  94. package/dist/lib/linux_aarch64/include/selva/node_id_set.h +0 -43
  95. package/dist/lib/linux_aarch64/include/selva/poptop.h +0 -114
  96. package/dist/lib/linux_aarch64/include/selva/queue_r.h +0 -190
  97. package/dist/lib/linux_aarch64/include/selva/selva_hash128.h +0 -49
  98. package/dist/lib/linux_aarch64/include/selva/selva_lang.h +0 -105
  99. package/dist/lib/linux_aarch64/include/selva/selva_math.h +0 -37
  100. package/dist/lib/linux_aarch64/include/selva/selva_string.h +0 -674
  101. package/dist/lib/linux_aarch64/include/selva/sort.h +0 -140
  102. package/dist/lib/linux_aarch64/include/selva/strsearch.h +0 -43
  103. package/dist/lib/linux_aarch64/include/selva/timestamp.h +0 -25
  104. package/dist/lib/linux_aarch64/include/selva/traverse.h +0 -65
  105. package/dist/lib/linux_aarch64/include/selva/types.h +0 -106
  106. package/dist/lib/linux_aarch64/include/selva/vector.h +0 -35
  107. package/dist/lib/linux_aarch64/include/selva/worker_ctx.h +0 -13
  108. package/dist/lib/linux_aarch64/include/selva/xxhash64.h +0 -23
  109. package/dist/lib/linux_aarch64/include/selva_error.h +0 -137
  110. package/dist/lib/linux_aarch64/include/selva_lang_code.h +0 -157
  111. package/dist/lib/linux_aarch64/include/tree.h +0 -852
  112. package/dist/lib/linux_aarch64/libdeflate.so +0 -0
  113. package/dist/lib/linux_aarch64/libjemalloc_selva.so.2 +0 -0
  114. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  115. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  116. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  117. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  118. package/dist/lib/linux_aarch64/libselva.so +0 -0
  119. package/dist/lib/linux_aarch64/libxxhash.so.0 +0 -0
  120. package/dist/lib/linux_x86_64/include/cdefs.h +0 -317
  121. package/dist/lib/linux_x86_64/include/jemalloc.h +0 -468
  122. package/dist/lib/linux_x86_64/include/libdeflate.h +0 -322
  123. package/dist/lib/linux_x86_64/include/libdeflate_strings.h +0 -35
  124. package/dist/lib/linux_x86_64/include/linker_set.h +0 -109
  125. package/dist/lib/linux_x86_64/include/queue.h +0 -627
  126. package/dist/lib/linux_x86_64/include/selva/_export.h +0 -7
  127. package/dist/lib/linux_x86_64/include/selva/align.h +0 -9
  128. package/dist/lib/linux_x86_64/include/selva/backoff_timeout.h +0 -29
  129. package/dist/lib/linux_x86_64/include/selva/bitmap.h +0 -95
  130. package/dist/lib/linux_x86_64/include/selva/crc32c.h +0 -17
  131. package/dist/lib/linux_x86_64/include/selva/ctime.h +0 -135
  132. package/dist/lib/linux_x86_64/include/selva/db.h +0 -418
  133. package/dist/lib/linux_x86_64/include/selva/endian.h +0 -301
  134. package/dist/lib/linux_x86_64/include/selva/fast_linear_search.h +0 -23
  135. package/dist/lib/linux_x86_64/include/selva/fast_memcmp.h +0 -18
  136. package/dist/lib/linux_x86_64/include/selva/fast_memmem.h +0 -11
  137. package/dist/lib/linux_x86_64/include/selva/fast_parsei.h +0 -36
  138. package/dist/lib/linux_x86_64/include/selva/fields.h +0 -383
  139. package/dist/lib/linux_x86_64/include/selva/find.h +0 -47
  140. package/dist/lib/linux_x86_64/include/selva/history.h +0 -64
  141. package/dist/lib/linux_x86_64/include/selva/hll.h +0 -81
  142. package/dist/lib/linux_x86_64/include/selva/lpf.h +0 -28
  143. package/dist/lib/linux_x86_64/include/selva/node_id_set.h +0 -43
  144. package/dist/lib/linux_x86_64/include/selva/poptop.h +0 -114
  145. package/dist/lib/linux_x86_64/include/selva/queue_r.h +0 -190
  146. package/dist/lib/linux_x86_64/include/selva/selva_hash128.h +0 -49
  147. package/dist/lib/linux_x86_64/include/selva/selva_lang.h +0 -105
  148. package/dist/lib/linux_x86_64/include/selva/selva_math.h +0 -37
  149. package/dist/lib/linux_x86_64/include/selva/selva_string.h +0 -674
  150. package/dist/lib/linux_x86_64/include/selva/sort.h +0 -140
  151. package/dist/lib/linux_x86_64/include/selva/strsearch.h +0 -43
  152. package/dist/lib/linux_x86_64/include/selva/timestamp.h +0 -25
  153. package/dist/lib/linux_x86_64/include/selva/traverse.h +0 -65
  154. package/dist/lib/linux_x86_64/include/selva/types.h +0 -106
  155. package/dist/lib/linux_x86_64/include/selva/vector.h +0 -35
  156. package/dist/lib/linux_x86_64/include/selva/worker_ctx.h +0 -13
  157. package/dist/lib/linux_x86_64/include/selva/xxhash64.h +0 -23
  158. package/dist/lib/linux_x86_64/include/selva_error.h +0 -137
  159. package/dist/lib/linux_x86_64/include/selva_lang_code.h +0 -157
  160. package/dist/lib/linux_x86_64/include/tree.h +0 -852
  161. package/dist/lib/linux_x86_64/libdeflate.so +0 -0
  162. package/dist/lib/linux_x86_64/libjemalloc_selva.so.2 +0 -0
  163. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  164. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  165. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  166. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  167. package/dist/lib/linux_x86_64/libselva.so +0 -0
  168. package/dist/lib/linux_x86_64/libxxhash.so.0 +0 -0
  169. package/dist/src/client/query/subscription/markers.d.ts +0 -10
  170. package/dist/src/client/query/subscription/markers.js +0 -213
  171. package/dist/src/client/query/subscription/run.d.ts +0 -5
  172. package/dist/src/client/query/subscription/run.js +0 -76
package/dist/src/index.js CHANGED
@@ -32,12 +32,13 @@ export class BasedDb {
32
32
  }
33
33
  }
34
34
  }
35
- #init({ path, maxModifySize }) {
35
+ #init({ path, maxModifySize, saveIntervalInSeconds, }) {
36
36
  this.fileSystemPath = path;
37
37
  this.maxModifySize = maxModifySize;
38
38
  const server = new DbServer({
39
39
  path,
40
40
  maxModifySize,
41
+ saveIntervalInSeconds,
41
42
  onSchemaChange(schema) {
42
43
  client.putLocalSchema(schema);
43
44
  },
@@ -76,9 +77,12 @@ export class BasedDb {
76
77
  return Promise.resolve(server.setSchema(schema, fromStart));
77
78
  },
78
79
  flushModify(buf) {
80
+ const d = performance.now();
79
81
  const offsets = server.modify(buf);
82
+ const dbWriteTime = performance.now() - d;
80
83
  return Promise.resolve({
81
84
  offsets,
85
+ dbWriteTime,
82
86
  });
83
87
  },
84
88
  getQueryBuf(buf) {
@@ -6,7 +6,7 @@ declare const native: {
6
6
  intFromExternal(external: any): BigInt;
7
7
  modify: (data: Uint8Array, types: Uint8Array, dbCtx: any, dirtyBlocksOut: Float64Array) => any;
8
8
  getQueryBuf: (q: Uint8Array, dbCtx: any) => ArrayBuffer | null;
9
- start: (id: number) => any;
9
+ start: () => any;
10
10
  stop: (dbCtx: any) => any;
11
11
  saveCommon: (path: string, dbCtx: any) => number;
12
12
  saveRange: (path: string, typeCode: number, start: number, end: number, dbCtx: any, hashOut: Uint8Array) => number;
@@ -22,6 +22,5 @@ declare const native: {
22
22
  destroySortIndex: (buf: Uint8Array, dbCtx: any) => any;
23
23
  xxHash64: (buf: Uint8Array, target: Uint8Array, index: number) => any;
24
24
  equals: (a: Uint8Array, b: Uint8Array) => boolean;
25
- expire: (dbCtx: any) => void;
26
25
  };
27
26
  export default native;
@@ -35,8 +35,8 @@ const native = {
35
35
  const x = db.getQueryBuf(dbCtx, q);
36
36
  return x;
37
37
  },
38
- start: (id) => {
39
- return db.start(id);
38
+ start: () => {
39
+ return db.start();
40
40
  },
41
41
  stop: (dbCtx) => {
42
42
  return db.stop(dbCtx);
@@ -93,9 +93,6 @@ const native = {
93
93
  equals: (a, b) => {
94
94
  return !!db.equals(a, b);
95
95
  },
96
- expire: (dbCtx) => {
97
- db.expire(dbCtx);
98
- },
99
96
  };
100
97
  global.__basedDb__native__ = native;
101
98
  export default native;
@@ -1,2 +1,4 @@
1
1
  import { Csmt } from './index.js';
2
- export default function draw(csmt: Csmt): string;
2
+ type DataFormatter<T> = (data: T) => string;
3
+ export default function draw<T = any>(csmt: Csmt<T>, dataFormatter?: DataFormatter<T>): string;
4
+ export {};
@@ -2,7 +2,12 @@ import { encodeBase64 } from '@saulx/utils';
2
2
  function makeLabel(node) {
3
3
  return `${node.key}\n${encodeBase64(node.hash).substring(0, 5)}`;
4
4
  }
5
- export default function draw(csmt) {
5
+ function wrapFormatter(dataFormatter, node) {
6
+ return `\n${dataFormatter(node.data).replace('"', '\\"')}`;
7
+ }
8
+ // This can be visualized with Graphviz dot.
9
+ // Online: https://dreampuf.github.io/GraphvizOnline/?engine=dot
10
+ export default function draw(csmt, dataFormatter) {
6
11
  const root = csmt.getRoot();
7
12
  const lines = [];
8
13
  const nodes = [];
@@ -12,7 +17,7 @@ export default function draw(csmt) {
12
17
  const left = node.left;
13
18
  const right = node.right;
14
19
  const isLeaf = !left && !right;
15
- nodes.push(`n${cur} [label="${makeLabel(node)}"${isLeaf ? ' shape=box' : ''}];`);
20
+ nodes.push(`n${cur} [label="${makeLabel(node) + ((dataFormatter && node.data) ? wrapFormatter(dataFormatter, node) : '')}"${isLeaf ? ' shape=box' : ''}];`);
16
21
  if (cur > 0) {
17
22
  lines.push(`n${prev} -- n${cur}`);
18
23
  }
@@ -1,7 +1,7 @@
1
1
  type Predicate = (x: any) => boolean;
2
2
  type Func = (x: any) => any;
3
3
  declare const match: (x: any) => {
4
- on: (pred: Predicate, fn: Func) => any;
4
+ on: (pred: Predicate, fn: Func) => /*elided*/ any;
5
5
  otherwise: (fn: Func) => any;
6
6
  };
7
7
  export default match;
@@ -4,4 +4,4 @@ export declare enum Direction {
4
4
  Right = "R"
5
5
  }
6
6
  export type Proof = [TreeKey | null, TreeKey | null] | [TreeKey | Uint8Array, TreeKey | Direction][];
7
- export default function membershipProof(root: TreeNode | null, k: TreeKey): Proof;
7
+ export default function membershipProof<T = any>(root: TreeNode<T> | null, k: TreeKey): Proof;
@@ -1,6 +1,6 @@
1
1
  import { TreeKey, TreeNode } from './types.js';
2
2
  export declare function distance(x: TreeKey, y: TreeKey): TreeKey;
3
- export declare function min(x: TreeNode | null, y: TreeNode | null): number;
4
- export declare function max(x: TreeNode | null, y: TreeNode | null): number;
5
- export declare function minInSubtree(node: TreeNode): TreeKey;
6
- export declare function maxInSubtree(node: TreeNode): TreeKey;
3
+ export declare function min(x: TreeNode<any> | null, y: TreeNode<any> | null): TreeKey;
4
+ export declare function max(x: TreeNode<any> | null, y: TreeNode<any> | null): TreeKey;
5
+ export declare function minInSubtree(node: TreeNode<any>): TreeKey;
6
+ export declare function maxInSubtree(node: TreeNode<any>): TreeKey;
@@ -1,3 +1,3 @@
1
1
  import { Hash, Csmt } from './types.js';
2
2
  export declare function hashEq(a: Hash, b: Hash): boolean;
3
- export declare function createTree(createHash: () => any): Csmt;
3
+ export declare function createTree<T>(createHash: () => any): Csmt<T>;
@@ -1,7 +1,7 @@
1
1
  import { TreeKeyNil } from './types.js';
2
2
  import { distance, min, max } from './tree-utils.js';
3
3
  import membershipProof from './memebership-proof.js';
4
- import { equals } from '../../utils.js';
4
+ import { equals } from '@saulx/utils';
5
5
  export function hashEq(a, b) {
6
6
  return equals(a, b);
7
7
  }
@@ -2,24 +2,24 @@ import { Proof } from './memebership-proof.js';
2
2
  export type TreeKey = number;
3
3
  export declare const TreeKeyNil = 0;
4
4
  export type Hash = Uint8Array;
5
- export interface TreeNode {
5
+ export interface TreeNode<T> {
6
6
  hash: Hash;
7
- key: TreeKey;
8
- data?: any;
9
- left: TreeNode | null;
10
- right: TreeNode | null;
7
+ key: TreeKey | null;
8
+ data?: T | null;
9
+ left: TreeNode<T> | null;
10
+ right: TreeNode<T> | null;
11
11
  }
12
12
  export type KeyHashPair = [TreeKey, Hash];
13
13
  export interface TreeDiff {
14
14
  left: KeyHashPair[];
15
15
  right: KeyHashPair[];
16
16
  }
17
- export interface Csmt {
17
+ export interface Csmt<T> {
18
18
  emptyHash: Uint8Array;
19
19
  /**
20
20
  * Get the root node.
21
21
  */
22
- getRoot: () => TreeNode | null;
22
+ getRoot: () => TreeNode<T> | null;
23
23
  /**
24
24
  * Insert a new key-hash pair.
25
25
  */
@@ -35,12 +35,12 @@ export interface Csmt {
35
35
  /**
36
36
  * Compute the diff between this and a given tree.
37
37
  */
38
- diff: (tree: Csmt) => TreeDiff;
38
+ diff: (tree: Csmt<T>) => TreeDiff;
39
39
  /**
40
40
  * Provide a proof of membership if a key exist in the three;
41
41
  * Otherwise a proof of non-membership is returned.
42
42
  */
43
43
  membershipProof: (k: TreeKey) => Proof;
44
- visitLeafNodes: (cb: (leaf: TreeNode) => void) => void;
45
- search: (k: TreeKey) => TreeNode;
44
+ visitLeafNodes: (cb: (leaf: TreeNode<T>) => void) => void;
45
+ search: (k: TreeKey) => TreeNode<T>;
46
46
  }
@@ -1,5 +1,5 @@
1
1
  export default function createHash(): {
2
- update: (buf: Uint8Array) => any;
2
+ update: (buf: Uint8Array) => /*elided*/ any;
3
3
  digest: (encoding?: "hex") => Uint8Array | string;
4
4
  reset: () => void;
5
5
  };
@@ -1,6 +1,6 @@
1
1
  // @ts-ignore
2
2
  import db from '../../../basedDbNative.cjs';
3
- import { bufToHex } from '../utils.js';
3
+ import { bufToHex } from '@saulx/utils';
4
4
  export default function createHash() {
5
5
  const state = db.hashCreate();
6
6
  const hash = {
@@ -1,6 +1,7 @@
1
1
  import { LangName, StrictSchema } from '@based/schema';
2
2
  import { SchemaTypesParsed, SchemaTypesParsedById } from '@based/schema/def';
3
3
  import { createTree } from './csmt/index.js';
4
+ import { CsmtNodeRange } from './tree.js';
4
5
  import { Worker, MessagePort } from 'node:worker_threads';
5
6
  import { TransformFns } from './migrate/index.js';
6
7
  import exitHook from 'exit-hook';
@@ -35,10 +36,10 @@ export declare class DbServer {
35
36
  schemaTypesParsedById: SchemaTypesParsedById;
36
37
  fileSystemPath: string;
37
38
  maxModifySize: number;
38
- merkleTree: ReturnType<typeof createTree>;
39
+ merkleTree: ReturnType<typeof createTree<CsmtNodeRange>>;
39
40
  dirtyRanges: Set<number>;
40
41
  csmtHashFun: {
41
- update: (buf: Uint8Array) => any;
42
+ update: (buf: Uint8Array) => /*elided*/ any;
42
43
  digest: (encoding?: "hex") => Uint8Array | string;
43
44
  reset: () => void;
44
45
  };
@@ -50,11 +51,14 @@ export declare class DbServer {
50
51
  stopped: boolean;
51
52
  onSchemaChange: OnSchemaChange;
52
53
  unlistenExit: ReturnType<typeof exitHook>;
53
- constructor({ path, maxModifySize, onSchemaChange, debug, }: {
54
+ saveIntervalInSeconds?: number;
55
+ saveInterval?: NodeJS.Timeout;
56
+ constructor({ path, maxModifySize, onSchemaChange, debug, saveIntervalInSeconds, }: {
54
57
  path: string;
55
58
  maxModifySize?: number;
56
59
  onSchemaChange?: OnSchemaChange;
57
60
  debug?: boolean;
61
+ saveIntervalInSeconds?: number;
58
62
  });
59
63
  start(opts?: {
60
64
  clean?: boolean;
@@ -64,7 +68,7 @@ export declare class DbServer {
64
68
  forceFullDump?: boolean;
65
69
  }): Promise<void>;
66
70
  createCsmtHashFun: () => {
67
- update: (buf: Uint8Array) => any;
71
+ update: (buf: Uint8Array) => /*elided*/ any;
68
72
  digest: (encoding?: "hex") => Uint8Array | string;
69
73
  reset: () => void;
70
74
  };
@@ -5,7 +5,7 @@ import { dirname, join } from 'node:path';
5
5
  import { getPropType, langCodesMap, } from '@based/schema';
6
6
  import { updateTypeDefs, schemaToSelvaBuffer, } from '@based/schema/def';
7
7
  import { start } from './start.js';
8
- import { initCsmt, makeCsmtKey, makeCsmtKeyFromNodeId } from './tree.js';
8
+ import { initCsmt, makeCsmtKey, makeCsmtKeyFromNodeId, } from './tree.js';
9
9
  import { save } from './save.js';
10
10
  import { Worker, MessageChannel } from 'node:worker_threads';
11
11
  import { fileURLToPath } from 'node:url';
@@ -17,6 +17,7 @@ export const WRITELOG_FILE = 'writelog.json';
17
17
  const __filename = fileURLToPath(import.meta.url);
18
18
  const __dirname = dirname(__filename);
19
19
  const workerPath = join(__dirname, 'worker.js');
20
+ const emptyUint8Array = new Uint8Array(0);
20
21
  class SortIndex {
21
22
  constructor(buf, dbCtxExternal) {
22
23
  this.buf = buf;
@@ -89,11 +90,14 @@ export class DbServer {
89
90
  stopped;
90
91
  onSchemaChange;
91
92
  unlistenExit;
92
- constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, debug, }) {
93
+ saveIntervalInSeconds;
94
+ saveInterval;
95
+ constructor({ path, maxModifySize = 100 * 1e3 * 1e3, onSchemaChange, debug, saveIntervalInSeconds, }) {
93
96
  this.maxModifySize = maxModifySize;
94
97
  this.fileSystemPath = path;
95
98
  this.sortIndexes = {};
96
99
  this.onSchemaChange = onSchemaChange;
100
+ this.saveIntervalInSeconds = saveIntervalInSeconds;
97
101
  if (debug) {
98
102
  debugServer(this);
99
103
  }
@@ -380,12 +384,6 @@ export class DbServer {
380
384
  const def = this.schemaTypesParsedById[typeId];
381
385
  let offset = def.lastId - startId;
382
386
  if (offset < 0) {
383
- console.log('-----------------');
384
- console.log(def.type, {
385
- offset,
386
- serverId: def.lastId,
387
- clientId: startId,
388
- });
389
387
  offset = 0;
390
388
  }
391
389
  buf[i++] = offset;
@@ -397,7 +395,6 @@ export class DbServer {
397
395
  def.lastId = lastId + offset;
398
396
  offsets[typeId] = offset;
399
397
  }
400
- // console.log('modify', this.processingQueries)
401
398
  if (this.processingQueries) {
402
399
  this.modifyQueue.push(new Uint8Array(buf));
403
400
  }
@@ -424,7 +421,8 @@ export class DbServer {
424
421
  const view = new DataView(buf.buffer, buf.byteOffset);
425
422
  while (i < end) {
426
423
  const key = view.getFloat64(i, true);
427
- this.dirtyRanges.add(key);
424
+ // These node ranges may not actually exist
425
+ //this.dirtyRanges.add(key)
428
426
  i += 8;
429
427
  }
430
428
  this.#resizeModifyDirtyRanges();
@@ -436,6 +434,16 @@ export class DbServer {
436
434
  this.dirtyRanges.add(key);
437
435
  }
438
436
  }
437
+ #expire() {
438
+ this.#resizeModifyDirtyRanges();
439
+ native.modify(emptyUint8Array, emptyUint8Array, this.dbCtxExternal, this.modifyDirtyRanges);
440
+ for (let key of this.modifyDirtyRanges) {
441
+ if (key === 0) {
442
+ break;
443
+ }
444
+ this.dirtyRanges.add(key);
445
+ }
446
+ }
439
447
  addToQueryQueue(resolve, buf) {
440
448
  if (this.queryQueue.size === 16777216) {
441
449
  resolve(new Error('Query queue exceeded'));
@@ -477,7 +485,7 @@ export class DbServer {
477
485
  // This will be more advanced - sometimes has indexes / sometimes not
478
486
  }
479
487
  if (!fromQueue) {
480
- native.expire(this.dbCtxExternal);
488
+ this.#expire();
481
489
  }
482
490
  this.availableWorkerIndex =
483
491
  (this.availableWorkerIndex + 1) % this.workers.length;
@@ -497,7 +505,7 @@ export class DbServer {
497
505
  if (this.queryQueue.size) {
498
506
  const queryQueue = this.queryQueue;
499
507
  this.queryQueue = new Map();
500
- native.expire(this.dbCtxExternal);
508
+ this.#expire();
501
509
  for (const [resolve, buf] of queryQueue) {
502
510
  resolve(this.getQueryBuf(buf, true));
503
511
  }
@@ -509,8 +517,15 @@ export class DbServer {
509
517
  return;
510
518
  }
511
519
  this.stopped = true;
512
- clearTimeout(this.cleanupTimer);
513
520
  this.unlistenExit();
521
+ if (this.cleanupTimer) {
522
+ clearTimeout(this.cleanupTimer);
523
+ this.cleanupTimer = null;
524
+ }
525
+ if (this.saveInterval) {
526
+ clearInterval(this.saveInterval);
527
+ this.saveInterval = null;
528
+ }
514
529
  try {
515
530
  if (!noSave) {
516
531
  await this.save();
@@ -48,6 +48,17 @@ else {
48
48
  if (props[path].typeIndex === REFERENCE ||
49
49
  props[path].typeIndex === REFERENCES) {
50
50
  include[i] = `${path}.id`;
51
+ if (props[path].edges) {
52
+ for (const key in props[path].edges) {
53
+ const prop = props[path].edges[key];
54
+ if (prop.typeIndex === REFERENCE || prop.typeIndex === REFERENCES) {
55
+ include.push(`${path}.${key}.id`);
56
+ }
57
+ else {
58
+ include.push(`${path}.${key}`);
59
+ }
60
+ }
61
+ }
51
62
  }
52
63
  }
53
64
  map[id] = { type, include };
@@ -5,14 +5,15 @@ import { join } from 'node:path';
5
5
  import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, initCsmt, makeCsmtKey, specialBlock, } from './tree.js';
6
6
  import { WRITELOG_FILE } from './index.js';
7
7
  import { writeFileSync } from 'node:fs';
8
- import { bufToHex } from '../utils.js';
8
+ import { bufToHex } from '@saulx/utils';
9
9
  const COMMON_SDB_FILE = 'common.sdb';
10
10
  const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
11
11
  function saveRange(db, typeId, start, end, hashOut) {
12
12
  const file = block_sdb_file(typeId, start, end);
13
13
  const path = join(db.fileSystemPath, file);
14
14
  const err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hashOut);
15
- if (err == -8) { // TODO ENOENT
15
+ if (err == -8) {
16
+ // TODO ENOENT
16
17
  return ''; // empty range
17
18
  }
18
19
  else if (err) {
@@ -1,4 +1,3 @@
1
- import { stringHash } from '@saulx/hash';
2
1
  import { DbWorker, SCHEMA_FILE, WRITELOG_FILE } from './index.js';
3
2
  import native from '../native.js';
4
3
  import { rm, mkdir, readFile } from 'node:fs/promises';
@@ -10,16 +9,15 @@ import exitHook from 'exit-hook';
10
9
  import './worker.js';
11
10
  import { save } from './save.js';
12
11
  import { DEFAULT_BLOCK_CAPACITY } from '@based/schema/def';
13
- import { bufToHex, hexToBuf } from '../utils.js';
12
+ import { bufToHex, hexToBuf } from '@saulx/utils';
14
13
  export async function start(db, opts) {
15
14
  const path = db.fileSystemPath;
16
- const id = stringHash(path) >>> 0;
17
15
  const noop = () => { };
18
16
  if (opts?.clean) {
19
17
  await rm(path, { recursive: true, force: true }).catch(noop);
20
18
  }
21
19
  await mkdir(path, { recursive: true }).catch(noop);
22
- db.dbCtxExternal = native.start(id);
20
+ db.dbCtxExternal = native.start();
23
21
  let writelog = null;
24
22
  try {
25
23
  writelog = JSON.parse((await readFile(join(path, WRITELOG_FILE))).toString());
@@ -65,7 +63,8 @@ export async function start(db, opts) {
65
63
  writelog?.types[def.id]?.blockCapacity || DEFAULT_BLOCK_CAPACITY;
66
64
  foreachBlock(db, def, (start, end, hash) => {
67
65
  const mtKey = makeCsmtKey(def.id, start);
68
- const file = writelog.rangeDumps[def.id]?.find((v) => v.start === start)?.file || '';
66
+ const file = writelog.rangeDumps[def.id]?.find((v) => v.start === start)?.file ||
67
+ '';
69
68
  const data = {
70
69
  file,
71
70
  typeId: def.id,
@@ -103,5 +102,10 @@ export async function start(db, opts) {
103
102
  signals.forEach((sig) => process.off(sig, blockSig));
104
103
  });
105
104
  }
105
+ if (db.saveIntervalInSeconds > 0) {
106
+ db.saveInterval ??= setInterval(() => {
107
+ save(db);
108
+ }, db.saveIntervalInSeconds * 1e3);
109
+ }
106
110
  }
107
111
  //# sourceMappingURL=start.js.map
@@ -3,13 +3,3 @@ export declare const DECODER: TextDecoder;
3
3
  export declare const ENCODER: TextEncoder;
4
4
  export declare const debugMode: (target: any, getInfo?: any) => void;
5
5
  export declare const debugServer: (server: DbServer) => void;
6
- export declare const equals: (aB: Uint8Array, bB: Uint8Array) => boolean;
7
- export declare function concatUint8Arr(bufs: Uint8Array[], totalByteLength?: number): Uint8Array;
8
- export declare const bufToHex: (a: Uint8Array) => string;
9
- export declare const hexToBuf: (s: string) => Uint8Array;
10
- export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
11
- export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
12
- export declare const readUint32: (val: Uint8Array, offset: number) => number;
13
- export declare const readInt32: (val: Uint8Array, offset: number) => number;
14
- export declare const readInt16: (val: Uint8Array, offset: number) => number;
15
- export declare const readUint16: (val: Uint8Array, offset: number) => number;
package/dist/src/utils.js CHANGED
@@ -1,6 +1,5 @@
1
1
  import { inspect } from 'node:util';
2
2
  import picocolors from 'picocolors';
3
- const native = typeof window === 'undefined' ? (await import('./native.js')).default : null;
4
3
  export const DECODER = new TextDecoder('utf-8');
5
4
  export const ENCODER = new TextEncoder();
6
5
  export const debugMode = (target, getInfo = null) => {
@@ -44,155 +43,4 @@ export const debugMode = (target, getInfo = null) => {
44
43
  }
45
44
  };
46
45
  export const debugServer = (server) => debugMode(server, () => `p: ${server.processingQueries} m: ${server.modifyQueue.length} q: ${server.queryQueue.size}`);
47
- export const equals = (aB, bB) => {
48
- const len = aB.byteLength;
49
- if (len != bB.byteLength) {
50
- return false;
51
- }
52
- let i = 0;
53
- if (len < 50 || !native) {
54
- while (i < len) {
55
- if (aB[i] != bB[i]) {
56
- return false;
57
- }
58
- i++;
59
- }
60
- return true;
61
- }
62
- else {
63
- while (i < 4) {
64
- if (aB[i] != bB[i]) {
65
- return false;
66
- }
67
- i++;
68
- }
69
- return native.equals(aB, bB);
70
- }
71
- };
72
- export function concatUint8Arr(bufs, totalByteLength) {
73
- totalByteLength =
74
- totalByteLength ?? bufs.reduce((acc, cur) => acc + cur.byteLength, 0);
75
- const res = new Uint8Array(totalByteLength);
76
- let off = 0;
77
- for (let i = 0; i < bufs.length; i++) {
78
- const buf = bufs[i];
79
- res.set(buf, off);
80
- off += buf.byteLength;
81
- }
82
- return res;
83
- }
84
- const charMap = ENCODER.encode('0123456789abcdef');
85
- // Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
86
- // https://issues.chromium.org/issues/42204568
87
- export const bufToHex = (a) => {
88
- const tmp = new Uint8Array(2 * a.byteLength);
89
- const n = charMap.length;
90
- let j = 0;
91
- for (let i = 0; i < a.byteLength; i++) {
92
- tmp[j++] = charMap[a[i] >>> 4 % n];
93
- tmp[j++] = charMap[a[i] & 0x0f % n];
94
- }
95
- return DECODER.decode(tmp);
96
- };
97
- const intMap = {
98
- '0': 0,
99
- '1': 0x1,
100
- '2': 0x2,
101
- '3': 0x3,
102
- '4': 0x4,
103
- '5': 0x5,
104
- '6': 0x6,
105
- '7': 0x7,
106
- '8': 0x8,
107
- '9': 0x9,
108
- a: 0xa,
109
- b: 0xb,
110
- c: 0xc,
111
- d: 0xd,
112
- e: 0xe,
113
- f: 0xf,
114
- };
115
- // Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
116
- // https://issues.chromium.org/issues/42204568
117
- export const hexToBuf = (s) => {
118
- const len = (s.length >>> 1) << 1;
119
- const buf = new Uint8Array(len / 2);
120
- for (let i = 0; i < buf.byteLength; i++) {
121
- const x = s.charAt(2 * i);
122
- const y = s.charAt(2 * i + 1);
123
- buf[i] = (intMap[x] << 4) + intMap[y];
124
- }
125
- return buf;
126
- };
127
- export const readDoubleLE = (val, offset) => {
128
- const low = (val[offset] |
129
- (val[offset + 1] << 8) |
130
- (val[offset + 2] << 16) |
131
- (val[offset + 3] << 24)) >>>
132
- 0;
133
- const high = (val[offset + 4] |
134
- (val[offset + 5] << 8) |
135
- (val[offset + 6] << 16) |
136
- (val[offset + 7] << 24)) >>>
137
- 0;
138
- const sign = high >>> 31 ? -1 : 1;
139
- let exponent = (high >>> 20) & 0x7ff;
140
- let fraction = (high & 0xfffff) * 2 ** 32 + low;
141
- if (exponent === 0x7ff) {
142
- if (fraction === 0)
143
- return sign * Infinity;
144
- return NaN;
145
- }
146
- if (exponent === 0) {
147
- if (fraction === 0)
148
- return sign * 0;
149
- exponent = 1;
150
- }
151
- else {
152
- fraction += 2 ** 52;
153
- }
154
- return sign * fraction * 2 ** (exponent - 1075);
155
- };
156
- export const readFloatLE = (val, offset) => {
157
- const bits = val[offset] |
158
- (val[offset + 1] << 8) |
159
- (val[offset + 2] << 16) |
160
- (val[offset + 3] << 24);
161
- const sign = bits >>> 31 ? -1 : 1;
162
- let exponent = (bits >>> 23) & 0xff;
163
- let fraction = bits & 0x7fffff;
164
- if (exponent === 0xff) {
165
- if (fraction === 0)
166
- return sign * Infinity;
167
- return NaN;
168
- }
169
- if (exponent === 0) {
170
- if (fraction === 0)
171
- return sign * 0;
172
- exponent = 1;
173
- }
174
- else {
175
- fraction |= 0x800000;
176
- }
177
- return sign * fraction * 2 ** (exponent - 150);
178
- };
179
- export const readUint32 = (val, offset) => {
180
- return ((val[offset] |
181
- (val[offset + 1] << 8) |
182
- (val[offset + 2] << 16) |
183
- (val[offset + 3] << 24)) >>>
184
- 0);
185
- };
186
- export const readInt32 = (val, offset) => {
187
- return (val[offset] |
188
- (val[offset + 1] << 8) |
189
- (val[offset + 2] << 16) |
190
- (val[offset + 3] << 24));
191
- };
192
- export const readInt16 = (val, offset) => {
193
- return ((val[offset] | (val[offset + 1] << 8)) << 16) >> 16;
194
- };
195
- export const readUint16 = (val, offset) => {
196
- return (val[offset] | (val[offset + 1] << 8)) >>> 0;
197
- };
198
46
  //# sourceMappingURL=utils.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@based/db",
3
- "version": "0.0.31",
3
+ "version": "0.0.32",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "main": "./dist/src/index.js",
@@ -38,9 +38,9 @@
38
38
  "basedDbNative.cjs"
39
39
  ],
40
40
  "dependencies": {
41
- "@based/schema": "5.0.0-alpha.10",
41
+ "@based/schema": "5.0.0-alpha.12",
42
42
  "@saulx/hash": "^3.0.0",
43
- "@saulx/utils": "^6.4.0",
43
+ "@saulx/utils": "^6.5.0",
44
44
  "exit-hook": "^4.0.0",
45
45
  "picocolors": "^1.1.0",
46
46
  "@based/crc32c": "^1.0.0"
@@ -1,23 +0,0 @@
1
- #pragma once
2
- #ifndef XXHASH64_H
3
- #define XXHASH64_H
4
-
5
- #include <stdlib.h>
6
- #include <stdint.h>
7
- #include "selva/_export.h"
8
- #include "cdefs.h"
9
-
10
-
11
- /**
12
- * @brief Wrapper function that computes the 64-bit hash of a given input string using the xxHash algorithm.
13
- *
14
- * Takes a string as input and returns a 64-bit hash value.
15
- *
16
- * @param s The input string to be hashed.
17
- * @return A 64-bit hash value of the input string.
18
- */
19
-
20
- SELVA_EXPORT
21
- uint64_t xxHash64(const char *s, size_t len);
22
-
23
- #endif