@based/db 0.0.25 → 0.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/dist/lib/darwin_aarch64/include/cdefs.h +1 -1
  2. package/dist/lib/darwin_aarch64/include/selva/db.h +45 -11
  3. package/dist/lib/darwin_aarch64/include/selva/fields.h +48 -13
  4. package/dist/lib/darwin_aarch64/include/selva/hll.h +59 -0
  5. package/dist/lib/darwin_aarch64/include/selva/types.h +2 -0
  6. package/dist/lib/darwin_aarch64/include/tree.h +69 -69
  7. package/dist/lib/darwin_aarch64/libnode-v20.node +0 -0
  8. package/dist/lib/darwin_aarch64/libnode-v21.node +0 -0
  9. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  10. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  11. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  12. package/dist/lib/linux_aarch64/include/cdefs.h +1 -1
  13. package/dist/lib/linux_aarch64/include/selva/db.h +45 -11
  14. package/dist/lib/linux_aarch64/include/selva/fields.h +48 -13
  15. package/dist/lib/linux_aarch64/include/selva/hll.h +59 -0
  16. package/dist/lib/linux_aarch64/include/selva/types.h +2 -0
  17. package/dist/lib/linux_aarch64/include/tree.h +69 -69
  18. package/dist/lib/linux_aarch64/libdeflate.so +0 -0
  19. package/dist/lib/linux_aarch64/libnode-v20.node +0 -0
  20. package/dist/lib/linux_aarch64/libnode-v21.node +0 -0
  21. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  22. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  23. package/dist/lib/linux_aarch64/libselva.so +0 -0
  24. package/dist/lib/linux_x86_64/include/cdefs.h +1 -1
  25. package/dist/lib/linux_x86_64/include/selva/db.h +45 -11
  26. package/dist/lib/linux_x86_64/include/selva/fields.h +48 -13
  27. package/dist/lib/linux_x86_64/include/selva/hll.h +59 -0
  28. package/dist/lib/linux_x86_64/include/selva/types.h +2 -0
  29. package/dist/lib/linux_x86_64/include/tree.h +69 -69
  30. package/dist/lib/linux_x86_64/libdeflate.so +0 -0
  31. package/dist/lib/linux_x86_64/libnode-v20.node +0 -0
  32. package/dist/lib/linux_x86_64/libnode-v21.node +0 -0
  33. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  34. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  35. package/dist/lib/linux_x86_64/libselva.so +0 -0
  36. package/dist/src/client/flushModify.js +5 -1
  37. package/dist/src/client/index.d.ts +12 -6
  38. package/dist/src/client/index.js +33 -1
  39. package/dist/src/client/modify/alias.js +3 -0
  40. package/dist/src/client/modify/binary.js +1 -1
  41. package/dist/src/client/modify/cardinality.d.ts +2 -2
  42. package/dist/src/client/modify/cardinality.js +17 -6
  43. package/dist/src/client/modify/create.js +17 -1
  44. package/dist/src/client/modify/fixed.js +27 -23
  45. package/dist/src/client/modify/json.js +15 -1
  46. package/dist/src/client/modify/modify.js +3 -1
  47. package/dist/src/client/modify/references/edge.js +4 -2
  48. package/dist/src/client/modify/references/references.js +21 -6
  49. package/dist/src/client/modify/string.js +5 -6
  50. package/dist/src/client/modify/text.js +0 -11
  51. package/dist/src/client/modify/upsert.js +1 -1
  52. package/dist/src/client/modify/vector.js +3 -3
  53. package/dist/src/client/query/BasedDbQuery.js +9 -1
  54. package/dist/src/client/query/BasedIterable.js +10 -3
  55. package/dist/src/client/query/aggregation.d.ts +3 -0
  56. package/dist/src/client/query/aggregation.js +9 -0
  57. package/dist/src/client/query/display.js +12 -2
  58. package/dist/src/client/query/filter/parseFilterValue.js +2 -4
  59. package/dist/src/client/query/filter/toBuffer.js +2 -2
  60. package/dist/src/client/query/include/walk.js +1 -0
  61. package/dist/src/client/query/query.d.ts +1 -1
  62. package/dist/src/client/query/query.js +1 -1
  63. package/dist/src/client/query/queryDef.js +0 -1
  64. package/dist/src/client/query/read/read.js +17 -5
  65. package/dist/src/client/query/toBuffer.js +2 -2
  66. package/dist/src/client/query/types.d.ts +4 -3
  67. package/dist/src/client/query/validation.js +8 -1
  68. package/dist/src/client/string.js +1 -1
  69. package/dist/src/index.d.ts +7 -1
  70. package/dist/src/index.js +19 -4
  71. package/dist/src/native.d.ts +1 -1
  72. package/dist/src/native.js +2 -2
  73. package/dist/src/server/csmt/tree.js +2 -2
  74. package/dist/src/server/index.d.ts +1 -1
  75. package/dist/src/server/index.js +36 -4
  76. package/dist/src/server/migrate/index.js +9 -5
  77. package/dist/src/server/migrate/worker.js +26 -1
  78. package/dist/src/server/save.js +20 -12
  79. package/dist/src/server/start.js +0 -2
  80. package/dist/src/utils.d.ts +6 -0
  81. package/dist/src/utils.js +81 -9
  82. package/package.json +4 -3
  83. package/dist/src/client/bitWise.d.ts +0 -6
  84. package/dist/src/client/bitWise.js +0 -72
  85. package/dist/src/client/query/aggregationFn.d.ts +0 -3
  86. package/dist/src/client/query/aggregationFn.js +0 -9
  87. package/dist/src/client/tree.d.ts +0 -1
  88. package/dist/src/client/tree.js +0 -5
@@ -1,5 +1,5 @@
1
1
  import picocolors from 'picocolors';
2
- import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, } from '@based/schema/def';
2
+ import { ALIAS, BINARY, BOOLEAN, REFERENCE, REFERENCES, REVERSE_TYPE_INDEX_MAP, STRING, TEXT, TIMESTAMP, VECTOR, propIsNumerical, createEmptyDef, DEFAULT_MAP, } from '@based/schema/def';
3
3
  import { EQUAL, EXISTS, HAS, isNumerical, LIKE, operatorReverseMap, VECTOR_FNS, } from './filter/types.js';
4
4
  import { MAX_ID, MAX_ID_VALUE, MAX_IDS_PER_QUERY, MIN_ID_VALUE, } from './thresholds.js';
5
5
  import { displayTarget, safeStringify } from './display.js';
@@ -418,8 +418,10 @@ export const EMPTY_ALIAS_PROP_DEF = {
418
418
  typeIndex: ALIAS,
419
419
  __isPropDef: true,
420
420
  separate: true,
421
+ validation: () => true,
421
422
  len: 0,
422
423
  start: 0,
424
+ default: DEFAULT_MAP[ALIAS],
423
425
  path: ['ERROR_ALIAS'],
424
426
  };
425
427
  export const ERROR_STRING = {
@@ -427,8 +429,10 @@ export const ERROR_STRING = {
427
429
  typeIndex: STRING,
428
430
  __isPropDef: true,
429
431
  separate: true,
432
+ validation: () => true,
430
433
  len: 0,
431
434
  start: 0,
435
+ default: DEFAULT_MAP[STRING],
432
436
  path: ['ERROR_STRING'],
433
437
  };
434
438
  export const ERROR_VECTOR = {
@@ -436,8 +440,10 @@ export const ERROR_VECTOR = {
436
440
  typeIndex: VECTOR,
437
441
  __isPropDef: true,
438
442
  separate: true,
443
+ validation: () => true,
439
444
  len: 0,
440
445
  start: 0,
446
+ default: DEFAULT_MAP[VECTOR],
441
447
  path: ['ERROR_VECTOR'],
442
448
  };
443
449
  export const EMPTY_SCHEMA_DEF = {
@@ -446,5 +452,6 @@ export const EMPTY_SCHEMA_DEF = {
446
452
  propNames: new Uint8Array([]),
447
453
  packed: new Uint8Array([]),
448
454
  idUint8: new Uint8Array([0, 0]),
455
+ mainEmptyAllZeroes: true,
449
456
  };
450
457
  //# sourceMappingURL=validation.js.map
@@ -1,5 +1,5 @@
1
1
  import native from '../native.js';
2
- import { readUint32 } from './bitWise.js';
2
+ import { readUint32 } from './../utils.js';
3
3
  import makeTmpBuffer from './tmpBuffer.js';
4
4
  import { DECODER, ENCODER } from '../utils.js';
5
5
  const { getUint8Array: getTmpBuffer } = makeTmpBuffer(4096); // the usual page size?
@@ -10,6 +10,9 @@ export { xxHash64 } from './client/xxHash64.js';
10
10
  export { crc32 } from './client/crc32.js';
11
11
  export * from './client/query/serialize.js';
12
12
  export * from './utils.js';
13
+ export * from './client/query/query.js';
14
+ export * from './client/query/BasedDbQuery.js';
15
+ export * from './client/query/BasedIterable.js';
13
16
  export declare class BasedDb {
14
17
  #private;
15
18
  client: DbClient;
@@ -35,7 +38,10 @@ export declare class BasedDb {
35
38
  stop: DbServer['stop'];
36
39
  save: DbServer['save'];
37
40
  migrateSchema: DbServer['migrateSchema'];
38
- isReady: DbClient['isModified'];
41
+ isModified: DbClient['isModified'];
42
+ schemaIsSet: DbClient['schemaIsSet'];
39
43
  destroy(): Promise<void>;
40
44
  wipe(): Promise<void>;
45
+ on: DbClient['on'];
46
+ off: DbClient['off'];
41
47
  }
package/dist/src/index.js CHANGED
@@ -12,6 +12,9 @@ export { xxHash64 } from './client/xxHash64.js';
12
12
  export { crc32 } from './client/crc32.js';
13
13
  export * from './client/query/serialize.js';
14
14
  export * from './utils.js';
15
+ export * from './client/query/query.js';
16
+ export * from './client/query/BasedDbQuery.js';
17
+ export * from './client/query/BasedIterable.js';
15
18
  export class BasedDb {
16
19
  client;
17
20
  server;
@@ -102,20 +105,26 @@ export class BasedDb {
102
105
  start = function () {
103
106
  return this.server.start.apply(this.server, arguments);
104
107
  };
105
- stop = function () {
108
+ stop = async function () {
109
+ await this.isModified();
106
110
  this.client.stop();
107
111
  return this.server.stop.apply(this.server, arguments);
108
112
  };
109
- save = function () {
113
+ save = async function () {
114
+ await this.isModified();
110
115
  return this.server.save.apply(this.server, arguments);
111
116
  };
112
117
  migrateSchema = function () {
113
118
  return this.server.migrateSchema.apply(this.server, arguments);
114
119
  };
115
- isReady = function () {
116
- return this.client.isReady.apply(this.client, arguments);
120
+ isModified = function () {
121
+ return this.client.isModified.apply(this.client, arguments);
122
+ };
123
+ schemaIsSet = function () {
124
+ return this.client.schemaIsSet.apply(this.client, arguments);
117
125
  };
118
126
  async destroy() {
127
+ await this.isModified();
119
128
  // Tmp fix: Gives node time to GC existing buffers else it can incorrectly re-asign to mem
120
129
  // Todo: clear all active queries, queues ETC
121
130
  await wait(Math.max(this.client.hooks.flushTime + 10, 10));
@@ -131,5 +140,11 @@ export class BasedDb {
131
140
  this.#init(opts);
132
141
  await this.start({ clean: true });
133
142
  }
143
+ on = function () {
144
+ return this.client.on.apply(this.client, arguments);
145
+ };
146
+ off = function () {
147
+ return this.client.on.apply(this.client, arguments);
148
+ };
134
149
  }
135
150
  //# sourceMappingURL=index.js.map
@@ -4,7 +4,7 @@ declare const _default: {
4
4
  workerCtxInit: () => void;
5
5
  externalFromInt(address: BigInt): any;
6
6
  intFromExternal(external: any): BigInt;
7
- modify: (data: Uint8Array, types: Uint8Array, dbCtx: any) => any;
7
+ modify: (data: Uint8Array, types: Uint8Array, dbCtx: any, dirtyBlocksOut: Float64Array) => any;
8
8
  getQueryBuf: (q: Uint8Array, dbCtx: any) => ArrayBuffer | null;
9
9
  start: (id: number) => any;
10
10
  stop: (dbCtx: any) => any;
@@ -28,8 +28,8 @@ export default {
28
28
  intFromExternal(external) {
29
29
  return db.intFromExternal(external);
30
30
  },
31
- modify: (data, types, dbCtx) => {
32
- db.modify(data, types, dbCtx);
31
+ modify: (data, types, dbCtx, dirtyBlocksOut) => {
32
+ db.modify(data, types, dbCtx, dirtyBlocksOut);
33
33
  },
34
34
  getQueryBuf: (q, dbCtx) => {
35
35
  const x = db.getQueryBuf(dbCtx, q);
@@ -199,8 +199,8 @@ export function createTree(createHash) {
199
199
  return {
200
200
  getRoot: () => root,
201
201
  insert: (k, h, data = null) => {
202
- if (!(h instanceof Uint8Array)) { // TODO can we extract the name somehow from Hash?
203
- throw new TypeError('`h` must be a Uint8Array'); // TODO can we extract the name somehow from Hash?
202
+ if (!(h instanceof Uint8Array)) {
203
+ throw new TypeError('`h` must be a Uint8Array');
204
204
  }
205
205
  const newLeaf = createLeaf(k, h, data);
206
206
  root = root ? insert(root, newLeaf) : newLeaf;
@@ -25,7 +25,7 @@ export declare class DbWorker {
25
25
  type OnSchemaChange = (schema: StrictSchema) => void;
26
26
  export declare class DbServer {
27
27
  #private;
28
- modifyBuf: SharedArrayBuffer;
28
+ modifyDirtyRanges: Float64Array;
29
29
  dbCtxExternal: any;
30
30
  schema: StrictSchema & {
31
31
  lastId: number;
@@ -46,6 +46,9 @@ export class DbWorker {
46
46
  });
47
47
  port1.on('message', (buf) => {
48
48
  // TODO FIX TYPES CHECK IF THIS MAKES A COPY
49
+ // It's a copy, if you don't want a copy you'd need to make it an explicit view
50
+ // to the underlying buffer:
51
+ // new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
49
52
  this.resolvers.shift()(new Uint8Array(buf));
50
53
  this.db.onQueryEnd();
51
54
  });
@@ -68,7 +71,7 @@ export class DbWorker {
68
71
  }
69
72
  }
70
73
  export class DbServer {
71
- modifyBuf;
74
+ modifyDirtyRanges;
72
75
  dbCtxExternal; // pointer to zig dbCtx
73
76
  schema = {
74
77
  lastId: 1, // we reserve one for root props
@@ -96,6 +99,22 @@ export class DbServer {
96
99
  this.sortIndexes = {};
97
100
  this.onSchemaChange = onSchemaChange;
98
101
  }
102
+ #resizeModifyDirtyRanges() {
103
+ let maxNrChanges = 0;
104
+ for (const typeId in this.schemaTypesParsedById) {
105
+ const def = this.schemaTypesParsedById[typeId];
106
+ const lastId = def.lastId;
107
+ const blockCapacity = def.blockCapacity;
108
+ const tmp = lastId - +!(lastId % def.blockCapacity);
109
+ const lastBlock = Math.ceil((((tmp / blockCapacity) | 0) * blockCapacity + 1) / blockCapacity);
110
+ maxNrChanges += lastBlock;
111
+ }
112
+ if (!this.modifyDirtyRanges ||
113
+ this.modifyDirtyRanges.length < maxNrChanges) {
114
+ const min = Math.max(maxNrChanges * 1.2, 1024) | 0;
115
+ this.modifyDirtyRanges = new Float64Array(min);
116
+ }
117
+ }
99
118
  start(opts) {
100
119
  return start(this, opts);
101
120
  }
@@ -338,7 +357,7 @@ export class DbServer {
338
357
  }
339
358
  updateTypeDefs(this.schema, this.schemaTypesParsed, this.schemaTypesParsedById);
340
359
  if (!fromStart) {
341
- writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error(SCHEMA_FILE, err));
360
+ writeFile(join(this.fileSystemPath, SCHEMA_FILE), JSON.stringify(this.schema)).catch((err) => console.error('!!!', SCHEMA_FILE, err));
342
361
  let types = Object.keys(this.schemaTypesParsed);
343
362
  const s = schemaToSelvaBuffer(this.schemaTypesParsed);
344
363
  for (let i = 0; i < s.length; i++) {
@@ -421,7 +440,14 @@ export class DbServer {
421
440
  this.dirtyRanges.add(key);
422
441
  i += 8;
423
442
  }
424
- native.modify(data, types, this.dbCtxExternal);
443
+ this.#resizeModifyDirtyRanges();
444
+ native.modify(data, types, this.dbCtxExternal, this.modifyDirtyRanges);
445
+ for (let key of this.modifyDirtyRanges) {
446
+ if (key === 0) {
447
+ break;
448
+ }
449
+ this.dirtyRanges.add(key);
450
+ }
425
451
  }
426
452
  getQueryBuf(buf) {
427
453
  if (this.modifyQueue.length) {
@@ -501,7 +527,13 @@ export class DbServer {
501
527
  }
502
528
  async destroy() {
503
529
  await this.stop(true);
504
- await rm(this.fileSystemPath, { recursive: true }).catch((err) => console.warn('Error removing dump folder', this.fileSystemPath, err.message));
530
+ await rm(this.fileSystemPath, { recursive: true }).catch((err) => {
531
+ // console.warn(
532
+ // 'Error removing dump folder',
533
+ // this.fileSystemPath,
534
+ // err.message,
535
+ // ),
536
+ });
505
537
  }
506
538
  }
507
539
  //# sourceMappingURL=index.js.map
@@ -5,7 +5,10 @@ import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threa
5
5
  import native from '../../native.js';
6
6
  import './worker.js';
7
7
  import { foreachDirtyBlock } from '../tree.js';
8
+ import { SCHEMA_FILE } from '../index.js';
8
9
  import { fileURLToPath } from 'url';
10
+ import { deepMerge } from '@saulx/utils';
11
+ import { writeFile } from 'fs/promises';
9
12
  const __filename = fileURLToPath(import.meta.url);
10
13
  const __dirname = dirname(__filename);
11
14
  const workerPath = join(__dirname, 'worker.js');
@@ -64,8 +67,9 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
64
67
  worker.on('error', console.error);
65
68
  let i = 0;
66
69
  let ranges = [];
67
- fromDbServer.updateMerkleTree();
68
- fromDbServer.dirtyRanges.clear();
70
+ // fromDbServer.updateMerkleTree()
71
+ // fromDbServer.dirtyRanges.clear()
72
+ await fromDbServer.save();
69
73
  fromDbServer.merkleTree.visitLeafNodes((leaf) => {
70
74
  ranges.push(leaf.data);
71
75
  });
@@ -106,13 +110,13 @@ export const migrate = async (fromDbServer, toSchema, transform) => {
106
110
  ;
107
111
  [schema, schemaTypesParsed] = msg.message;
108
112
  }
109
- fromDbServer.schema = schema;
110
- fromDbServer.schemaTypesParsed = schemaTypesParsed;
113
+ fromDbServer.schema = deepMerge(toDb.server.schema, schema);
114
+ fromDbServer.schemaTypesParsed = deepMerge(toDb.server.schemaTypesParsed, schemaTypesParsed);
111
115
  fromDbServer.dbCtxExternal = toCtx;
112
116
  toDb.server.dbCtxExternal = fromCtx;
113
117
  }
114
118
  const promises = fromDbServer.workers.map((worker) => worker.updateCtx(toAddress));
115
- promises.push(toDb.destroy(), worker.terminate());
119
+ promises.push(toDb.destroy(), worker.terminate(), fromDbServer.save({ forceFullDump: true }), writeFile(join(fromDbServer.fileSystemPath, SCHEMA_FILE), JSON.stringify(fromDbServer.schema)));
116
120
  await Promise.all(promises);
117
121
  fromDbServer.onSchemaChange?.(fromDbServer.schema);
118
122
  return fromDbServer.schema;
@@ -2,6 +2,7 @@ import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_thr
2
2
  import native from '../../native.js';
3
3
  import { BasedDb } from '../../index.js';
4
4
  import { REFERENCE, REFERENCES } from '@based/schema/def';
5
+ import { isTypedArray } from 'node:util/types';
5
6
  if (isMainThread) {
6
7
  console.warn('running worker.ts in mainthread');
7
8
  }
@@ -12,6 +13,27 @@ else {
12
13
  const path = null;
13
14
  const fromDb = new BasedDb({ path });
14
15
  const toDb = new BasedDb({ path });
16
+ const cp = (obj) => {
17
+ let copy;
18
+ for (const key in obj) {
19
+ const val = obj[key];
20
+ if (typeof val === 'number') {
21
+ // only copy numbers
22
+ copy ??= Array.isArray(obj) ? [] : {};
23
+ copy[key] = val;
24
+ }
25
+ else if (typeof val === 'object' &&
26
+ val !== null &&
27
+ !isTypedArray(val)) {
28
+ const res = cp(val);
29
+ if (res) {
30
+ copy ??= Array.isArray(obj) ? [] : {};
31
+ copy[key] = cp(val);
32
+ }
33
+ }
34
+ }
35
+ return copy;
36
+ };
15
37
  fromDb.server.dbCtxExternal = fromCtx;
16
38
  toDb.server.dbCtxExternal = toCtx;
17
39
  await fromDb.setSchema(fromSchema, true);
@@ -71,7 +93,10 @@ else {
71
93
  }
72
94
  }
73
95
  await toDb.drain();
74
- channel.postMessage([toDb.server.schema, toDb.server.schemaTypesParsed]);
96
+ channel.postMessage([
97
+ cp(toDb.server.schema),
98
+ cp(toDb.server.schemaTypesParsed),
99
+ ]);
75
100
  // put it to sleep
76
101
  atomics[0] = 0;
77
102
  Atomics.notify(atomics, 0);
@@ -2,13 +2,24 @@ import native from '../native.js';
2
2
  import { isMainThread } from 'node:worker_threads';
3
3
  import { writeFile } from 'node:fs/promises';
4
4
  import { join } from 'node:path';
5
- import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey } from './tree.js';
5
+ import { destructureCsmtKey, foreachBlock, foreachDirtyBlock, makeCsmtKey, } from './tree.js';
6
6
  import { WRITELOG_FILE } from './index.js';
7
7
  import { writeFileSync } from 'node:fs';
8
8
  import { bufToHex } from '../utils.js';
9
9
  import { createTree } from './csmt/tree.js';
10
10
  const COMMON_SDB_FILE = 'common.sdb';
11
11
  const block_sdb_file = (typeId, start, end) => `${typeId}_${start}_${end}.sdb`;
12
+ function saveRange(db, typeId, start, end, hashOut) {
13
+ const file = block_sdb_file(typeId, start, end);
14
+ const path = join(db.fileSystemPath, file);
15
+ const err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hashOut);
16
+ if (err) {
17
+ // TODO print the error string
18
+ console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
19
+ return null;
20
+ }
21
+ return file;
22
+ }
12
23
  export function save(db, sync = false, forceFullDump = false) {
13
24
  if (!(isMainThread && (db.dirtyRanges.size || forceFullDump))) {
14
25
  return;
@@ -26,12 +37,12 @@ export function save(db, sync = false, forceFullDump = false) {
26
37
  const def = db.schemaTypesParsed[key];
27
38
  foreachBlock(db, def, (start, end, _hash) => {
28
39
  const typeId = def.id;
29
- const file = block_sdb_file(typeId, start, end);
30
- const hash = new Uint8Array(16); // TODO One is unnecessary, probably the arg of this cb
31
- err = native.saveRange(join(db.fileSystemPath, file), typeId, start, end, db.dbCtxExternal, hash);
32
- if (err) {
33
- console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
34
- return; // TODO What to do with the merkle tree in db situation?
40
+ const hash = new Uint8Array(16);
41
+ const file = saveRange(db, typeId, start, end, hash);
42
+ if (!file) {
43
+ // The previous state should remain in the merkle tree for
44
+ // load and sync purposes.
45
+ return;
35
46
  }
36
47
  const mtKey = makeCsmtKey(typeId, start);
37
48
  const data = {
@@ -46,12 +57,9 @@ export function save(db, sync = false, forceFullDump = false) {
46
57
  }
47
58
  else {
48
59
  foreachDirtyBlock(db, (mtKey, typeId, start, end) => {
49
- const file = block_sdb_file(typeId, start, end);
50
- const path = join(db.fileSystemPath, file);
51
60
  const hash = new Uint8Array(16);
52
- err = native.saveRange(path, typeId, start, end, db.dbCtxExternal, hash);
53
- if (err) {
54
- console.error(`Save ${typeId}:${start}-${end} failed: ${err}`);
61
+ const file = saveRange(db, typeId, start, end, hash);
62
+ if (!file) {
55
63
  // The previous state should remain in the merkle tree for
56
64
  // load and sync purposes.
57
65
  return;
@@ -19,8 +19,6 @@ export async function start(db, opts) {
19
19
  await rm(path, { recursive: true, force: true }).catch(noop);
20
20
  }
21
21
  await mkdir(path, { recursive: true }).catch(noop);
22
- // not doing db yet
23
- // db.modifyBuf = new SharedArrayBuffer(db.maxModifySize)
24
22
  db.dbCtxExternal = native.start(id);
25
23
  let writelog = null;
26
24
  try {
@@ -5,3 +5,9 @@ export declare function concatUint8Arr(bufs: Uint8Array[], totalByteLength?: num
5
5
  export declare const bufToHex: (a: Uint8Array) => string;
6
6
  export declare const hexToBuf: (s: string) => Uint8Array;
7
7
  export declare const base64encode: (a: Uint8Array, lineMax?: number) => string;
8
+ export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
9
+ export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
10
+ export declare const readUint32: (val: Uint8Array, offset: number) => number;
11
+ export declare const readInt32: (val: Uint8Array, offset: number) => number;
12
+ export declare const readInt16: (val: Uint8Array, offset: number) => number;
13
+ export declare const readUint16: (val: Uint8Array, offset: number) => number;
package/dist/src/utils.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { encodeBase64 } from '@saulx/utils';
2
- const native = (typeof window === 'undefined') ? (await import('./native.js')).default : null;
2
+ const native = typeof window === 'undefined' ? (await import('./native.js')).default : null;
3
3
  export const DECODER = new TextDecoder('utf-8');
4
4
  export const ENCODER = new TextEncoder();
5
5
  export const equals = (aB, bB) => {
@@ -28,7 +28,8 @@ export const equals = (aB, bB) => {
28
28
  }
29
29
  };
30
30
  export function concatUint8Arr(bufs, totalByteLength) {
31
- totalByteLength = totalByteLength ?? bufs.reduce((acc, cur) => acc + cur.byteLength, 0);
31
+ totalByteLength =
32
+ totalByteLength ?? bufs.reduce((acc, cur) => acc + cur.byteLength, 0);
32
33
  const res = new Uint8Array(totalByteLength);
33
34
  let off = 0;
34
35
  for (let i = 0; i < bufs.length; i++) {
@@ -62,12 +63,12 @@ const intMap = {
62
63
  '7': 0x7,
63
64
  '8': 0x8,
64
65
  '9': 0x9,
65
- 'a': 0xa,
66
- 'b': 0xb,
67
- 'c': 0xc,
68
- 'd': 0xd,
69
- 'e': 0xe,
70
- 'f': 0xf,
66
+ a: 0xa,
67
+ b: 0xb,
68
+ c: 0xc,
69
+ d: 0xd,
70
+ e: 0xe,
71
+ f: 0xf,
71
72
  };
72
73
  // Uint8Array.fromHex() and Uint8Array.toHex() are not available in V8
73
74
  // https://issues.chromium.org/issues/42204568
@@ -85,7 +86,7 @@ function base64OutLen(n, lineMax) {
85
86
  let olen;
86
87
  /* This version would be with padding but we don't pad */
87
88
  //olen = n * 4 / 3 + 4; /* 3-byte blocks to 4-byte */
88
- olen = ((4 * n / 3) + 3) & ~3;
89
+ olen = ((4 * n) / 3 + 3) & ~3;
89
90
  olen += lineMax > 0 ? olen / lineMax : 0; // line feeds
90
91
  return olen;
91
92
  }
@@ -99,4 +100,75 @@ export const base64encode = (a, lineMax = 72) => {
99
100
  return DECODER.decode(native.base64encode(tmp, a, lineMax));
100
101
  }
101
102
  };
103
+ export const readDoubleLE = (val, offset) => {
104
+ const low = (val[offset] |
105
+ (val[offset + 1] << 8) |
106
+ (val[offset + 2] << 16) |
107
+ (val[offset + 3] << 24)) >>>
108
+ 0;
109
+ const high = (val[offset + 4] |
110
+ (val[offset + 5] << 8) |
111
+ (val[offset + 6] << 16) |
112
+ (val[offset + 7] << 24)) >>>
113
+ 0;
114
+ const sign = high >>> 31 ? -1 : 1;
115
+ let exponent = (high >>> 20) & 0x7ff;
116
+ let fraction = (high & 0xfffff) * 2 ** 32 + low;
117
+ if (exponent === 0x7ff) {
118
+ if (fraction === 0)
119
+ return sign * Infinity;
120
+ return NaN;
121
+ }
122
+ if (exponent === 0) {
123
+ if (fraction === 0)
124
+ return sign * 0;
125
+ exponent = 1;
126
+ }
127
+ else {
128
+ fraction += 2 ** 52;
129
+ }
130
+ return sign * fraction * 2 ** (exponent - 1075);
131
+ };
132
+ export const readFloatLE = (val, offset) => {
133
+ const bits = val[offset] |
134
+ (val[offset + 1] << 8) |
135
+ (val[offset + 2] << 16) |
136
+ (val[offset + 3] << 24);
137
+ const sign = bits >>> 31 ? -1 : 1;
138
+ let exponent = (bits >>> 23) & 0xff;
139
+ let fraction = bits & 0x7fffff;
140
+ if (exponent === 0xff) {
141
+ if (fraction === 0)
142
+ return sign * Infinity;
143
+ return NaN;
144
+ }
145
+ if (exponent === 0) {
146
+ if (fraction === 0)
147
+ return sign * 0;
148
+ exponent = 1;
149
+ }
150
+ else {
151
+ fraction |= 0x800000;
152
+ }
153
+ return sign * fraction * 2 ** (exponent - 150);
154
+ };
155
+ export const readUint32 = (val, offset) => {
156
+ return ((val[offset] |
157
+ (val[offset + 1] << 8) |
158
+ (val[offset + 2] << 16) |
159
+ (val[offset + 3] << 24)) >>>
160
+ 0);
161
+ };
162
+ export const readInt32 = (val, offset) => {
163
+ return (val[offset] |
164
+ (val[offset + 1] << 8) |
165
+ (val[offset + 2] << 16) |
166
+ (val[offset + 3] << 24));
167
+ };
168
+ export const readInt16 = (val, offset) => {
169
+ return ((val[offset] | (val[offset + 1] << 8)) << 16) >> 16;
170
+ };
171
+ export const readUint16 = (val, offset) => {
172
+ return (val[offset] | (val[offset + 1] << 8)) >>> 0;
173
+ };
102
174
  //# sourceMappingURL=utils.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@based/db",
3
- "version": "0.0.25",
3
+ "version": "0.0.27",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "main": "./dist/src/index.js",
@@ -32,9 +32,9 @@
32
32
  "basedDbNative.cjs"
33
33
  ],
34
34
  "dependencies": {
35
- "@based/schema": "5.0.0-alpha.7",
35
+ "@based/schema": "5.0.0-alpha.8",
36
36
  "@saulx/hash": "^3.0.0",
37
- "@saulx/utils": "^4.3.2",
37
+ "@saulx/utils": "^6.1.1",
38
38
  "exit-hook": "^4.0.0",
39
39
  "picocolors": "^1.1.0",
40
40
  "@based/crc32c": "^1.0.0"
@@ -43,6 +43,7 @@
43
43
  "@based/locale-x86-64-gnu": "*"
44
44
  },
45
45
  "devDependencies": {
46
+ "jsondiffpatch": "^0.7.3",
46
47
  "@based/crc32c": "^1.0.0",
47
48
  "@types/node": "^22.5.3",
48
49
  "axios": "^1.7.9",
@@ -1,6 +0,0 @@
1
- export declare const readDoubleLE: (val: Uint8Array, offset: number) => number;
2
- export declare const readFloatLE: (val: Uint8Array, offset: number) => number;
3
- export declare const readUint32: (val: Uint8Array, offset: number) => number;
4
- export declare const readInt32: (val: Uint8Array, offset: number) => number;
5
- export declare const readInt16: (val: Uint8Array, offset: number) => number;
6
- export declare const readUint16: (val: Uint8Array, offset: number) => number;
@@ -1,72 +0,0 @@
1
- export const readDoubleLE = (val, offset) => {
2
- const low = (val[offset] |
3
- (val[offset + 1] << 8) |
4
- (val[offset + 2] << 16) |
5
- (val[offset + 3] << 24)) >>>
6
- 0;
7
- const high = (val[offset + 4] |
8
- (val[offset + 5] << 8) |
9
- (val[offset + 6] << 16) |
10
- (val[offset + 7] << 24)) >>>
11
- 0;
12
- const sign = high >>> 31 ? -1 : 1;
13
- let exponent = (high >>> 20) & 0x7ff;
14
- let fraction = (high & 0xfffff) * 2 ** 32 + low;
15
- if (exponent === 0x7ff) {
16
- if (fraction === 0)
17
- return sign * Infinity;
18
- return NaN;
19
- }
20
- if (exponent === 0) {
21
- if (fraction === 0)
22
- return sign * 0;
23
- exponent = 1;
24
- }
25
- else {
26
- fraction += 2 ** 52;
27
- }
28
- return sign * fraction * 2 ** (exponent - 1075);
29
- };
30
- export const readFloatLE = (val, offset) => {
31
- const bits = val[offset] |
32
- (val[offset + 1] << 8) |
33
- (val[offset + 2] << 16) |
34
- (val[offset + 3] << 24);
35
- const sign = bits >>> 31 ? -1 : 1;
36
- let exponent = (bits >>> 23) & 0xff;
37
- let fraction = bits & 0x7fffff;
38
- if (exponent === 0xff) {
39
- if (fraction === 0)
40
- return sign * Infinity;
41
- return NaN;
42
- }
43
- if (exponent === 0) {
44
- if (fraction === 0)
45
- return sign * 0;
46
- exponent = 1;
47
- }
48
- else {
49
- fraction |= 0x800000;
50
- }
51
- return sign * fraction * 2 ** (exponent - 150);
52
- };
53
- export const readUint32 = (val, offset) => {
54
- return ((val[offset] |
55
- (val[offset + 1] << 8) |
56
- (val[offset + 2] << 16) |
57
- (val[offset + 3] << 24)) >>>
58
- 0);
59
- };
60
- export const readInt32 = (val, offset) => {
61
- return (val[offset] |
62
- (val[offset + 1] << 8) |
63
- (val[offset + 2] << 16) |
64
- (val[offset + 3] << 24));
65
- };
66
- export const readInt16 = (val, offset) => {
67
- return ((val[offset] | (val[offset + 1] << 8)) << 16) >> 16;
68
- };
69
- export const readUint16 = (val, offset) => {
70
- return (val[offset] | (val[offset + 1] << 8)) >>> 0;
71
- };
72
- //# sourceMappingURL=bitWise.js.map
@@ -1,3 +0,0 @@
1
- import { QueryDef, AggFn } from './types.js';
2
- export declare const createAggFnBuffer: (aggregation: AggFn) => Uint8Array;
3
- export declare const count: (def: QueryDef) => void;
@@ -1,9 +0,0 @@
1
- export const createAggFnBuffer = (aggregation) => {
2
- const buf = new Uint8Array(1);
3
- buf[0] = aggregation;
4
- return buf;
5
- };
6
- export const count = (def) => {
7
- def.aggregation = 4 /* AggFn.COUNT */;
8
- };
9
- //# sourceMappingURL=aggregationFn.js.map