@based/db 0.0.70 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +105 -77
  3. package/dist/lib/darwin_aarch64/include/libdeflate.h +23 -0
  4. package/dist/lib/darwin_aarch64/include/selva/db.h +0 -41
  5. package/dist/lib/darwin_aarch64/include/selva/fast_linear_search.h +6 -2
  6. package/dist/lib/darwin_aarch64/include/selva/fields.h +52 -37
  7. package/dist/lib/darwin_aarch64/include/selva/gmtime.h +137 -0
  8. package/dist/lib/darwin_aarch64/include/selva/hll.h +1 -1
  9. package/dist/lib/darwin_aarch64/include/selva/mblen.h +40 -0
  10. package/dist/lib/darwin_aarch64/include/selva/selva_string.h +8 -5
  11. package/dist/lib/darwin_aarch64/include/selva/thread.h +37 -0
  12. package/dist/lib/darwin_aarch64/include/selva/types.h +18 -4
  13. package/dist/lib/darwin_aarch64/libdeflate.dylib +0 -0
  14. package/dist/lib/darwin_aarch64/libjemalloc_selva.2.dylib +0 -0
  15. package/dist/lib/darwin_aarch64/libnode-v22.node +0 -0
  16. package/dist/lib/darwin_aarch64/libnode-v23.node +0 -0
  17. package/dist/lib/darwin_aarch64/libnode-v24.node +0 -0
  18. package/dist/lib/darwin_aarch64/libselva.dylib +0 -0
  19. package/dist/lib/darwin_aarch64/libxxhash.dylib +0 -0
  20. package/dist/lib/linux_aarch64/include/libdeflate.h +23 -0
  21. package/dist/lib/linux_aarch64/include/selva/db.h +0 -41
  22. package/dist/lib/linux_aarch64/include/selva/fast_linear_search.h +6 -2
  23. package/dist/lib/linux_aarch64/include/selva/fields.h +52 -37
  24. package/dist/lib/linux_aarch64/include/selva/gmtime.h +137 -0
  25. package/dist/lib/linux_aarch64/include/selva/hll.h +1 -1
  26. package/dist/lib/linux_aarch64/include/selva/mblen.h +40 -0
  27. package/dist/lib/linux_aarch64/include/selva/selva_string.h +8 -5
  28. package/dist/lib/linux_aarch64/include/selva/thread.h +37 -0
  29. package/dist/lib/linux_aarch64/include/selva/types.h +18 -4
  30. package/dist/lib/linux_aarch64/libdeflate.so +0 -0
  31. package/dist/lib/linux_aarch64/libnode-v22.node +0 -0
  32. package/dist/lib/linux_aarch64/libnode-v23.node +0 -0
  33. package/dist/lib/linux_aarch64/libnode-v24.node +0 -0
  34. package/dist/lib/linux_aarch64/libselva.so +0 -0
  35. package/dist/lib/linux_x86_64/include/libdeflate.h +23 -0
  36. package/dist/lib/linux_x86_64/include/selva/db.h +0 -41
  37. package/dist/lib/linux_x86_64/include/selva/fast_linear_search.h +6 -2
  38. package/dist/lib/linux_x86_64/include/selva/fields.h +52 -37
  39. package/dist/lib/linux_x86_64/include/selva/gmtime.h +137 -0
  40. package/dist/lib/linux_x86_64/include/selva/hll.h +1 -1
  41. package/dist/lib/linux_x86_64/include/selva/mblen.h +40 -0
  42. package/dist/lib/linux_x86_64/include/selva/selva_string.h +8 -5
  43. package/dist/lib/linux_x86_64/include/selva/thread.h +37 -0
  44. package/dist/lib/linux_x86_64/include/selva/types.h +18 -4
  45. package/dist/lib/linux_x86_64/libdeflate.so +0 -0
  46. package/dist/lib/linux_x86_64/libnode-v22.node +0 -0
  47. package/dist/lib/linux_x86_64/libnode-v23.node +0 -0
  48. package/dist/lib/linux_x86_64/libnode-v24.node +0 -0
  49. package/dist/lib/linux_x86_64/libselva.so +0 -0
  50. package/dist/src/client/index.d.ts +16 -21
  51. package/dist/src/client/index.js +48 -59
  52. package/dist/src/client/modify/Ctx.d.ts +38 -0
  53. package/dist/src/client/modify/Ctx.js +30 -0
  54. package/dist/src/client/modify/Tmp.d.ts +19 -0
  55. package/dist/src/client/modify/Tmp.js +67 -0
  56. package/dist/src/client/modify/create/index.d.ts +6 -0
  57. package/dist/src/client/modify/create/index.js +191 -0
  58. package/dist/src/client/modify/create/mark.d.ts +7 -0
  59. package/dist/src/client/modify/create/mark.js +42 -0
  60. package/dist/src/client/modify/cursor.d.ts +10 -0
  61. package/dist/src/client/modify/cursor.js +48 -0
  62. package/dist/src/client/modify/delete/index.d.ts +2 -0
  63. package/dist/src/client/modify/delete/index.js +39 -0
  64. package/dist/src/client/modify/drain.d.ts +7 -0
  65. package/dist/src/client/modify/drain.js +77 -0
  66. package/dist/src/client/modify/edges/binary.d.ts +3 -0
  67. package/dist/src/client/modify/edges/binary.js +28 -0
  68. package/dist/src/client/modify/edges/cardinality.d.ts +3 -0
  69. package/dist/src/client/modify/edges/cardinality.js +15 -0
  70. package/dist/src/client/modify/edges/header.d.ts +5 -0
  71. package/dist/src/client/modify/edges/header.js +21 -0
  72. package/dist/src/client/modify/edges/index.d.ts +3 -0
  73. package/dist/src/client/modify/edges/index.js +128 -0
  74. package/dist/src/client/modify/edges/reference.d.ts +3 -0
  75. package/dist/src/client/modify/edges/reference.js +31 -0
  76. package/dist/src/client/modify/edges/references.d.ts +3 -0
  77. package/dist/src/client/modify/edges/references.js +38 -0
  78. package/dist/src/client/modify/edges/separate.d.ts +3 -0
  79. package/dist/src/client/modify/edges/separate.js +24 -0
  80. package/dist/src/client/modify/edges/string.d.ts +3 -0
  81. package/dist/src/client/modify/edges/string.js +30 -0
  82. package/dist/src/client/modify/error.d.ts +7 -0
  83. package/dist/src/client/modify/error.js +68 -0
  84. package/dist/src/client/modify/expire/index.d.ts +2 -0
  85. package/dist/src/client/modify/expire/index.js +31 -0
  86. package/dist/src/client/modify/props/alias.d.ts +3 -0
  87. package/dist/src/client/modify/props/alias.js +26 -0
  88. package/dist/src/client/modify/props/binary.d.ts +5 -0
  89. package/dist/src/client/modify/props/binary.js +52 -0
  90. package/dist/src/client/modify/props/cardinality.d.ts +4 -0
  91. package/dist/src/client/modify/props/cardinality.js +46 -0
  92. package/dist/src/client/modify/props/delete.d.ts +3 -0
  93. package/dist/src/client/modify/props/delete.js +13 -0
  94. package/dist/src/client/modify/props/fixed.d.ts +4 -0
  95. package/dist/src/client/modify/props/fixed.js +105 -0
  96. package/dist/src/client/modify/props/increment.d.ts +3 -0
  97. package/dist/src/client/modify/props/increment.js +28 -0
  98. package/dist/src/client/modify/props/json.d.ts +3 -0
  99. package/dist/src/client/modify/props/json.js +5 -0
  100. package/dist/src/client/modify/props/main.d.ts +4 -0
  101. package/dist/src/client/modify/props/main.js +22 -0
  102. package/dist/src/client/modify/props/object.d.ts +3 -0
  103. package/dist/src/client/modify/props/object.js +37 -0
  104. package/dist/src/client/modify/props/reference.d.ts +6 -0
  105. package/dist/src/client/modify/props/reference.js +92 -0
  106. package/dist/src/client/modify/props/references.d.ts +3 -0
  107. package/dist/src/client/modify/props/references.js +231 -0
  108. package/dist/src/client/modify/props/separate.d.ts +3 -0
  109. package/dist/src/client/modify/props/separate.js +43 -0
  110. package/dist/src/client/modify/props/string.d.ts +5 -0
  111. package/dist/src/client/modify/props/string.js +51 -0
  112. package/dist/src/client/modify/props/text.d.ts +3 -0
  113. package/dist/src/client/modify/props/text.js +35 -0
  114. package/dist/src/client/modify/props/vector.d.ts +3 -0
  115. package/dist/src/client/modify/props/vector.js +30 -0
  116. package/dist/src/client/modify/resize.d.ts +3 -0
  117. package/dist/src/client/modify/resize.js +20 -0
  118. package/dist/src/client/modify/types.d.ts +3 -4
  119. package/dist/src/client/modify/types.js +2 -1
  120. package/dist/src/client/modify/uint.d.ts +7 -0
  121. package/dist/src/client/modify/uint.js +27 -0
  122. package/dist/src/client/modify/update/index.d.ts +3 -0
  123. package/dist/src/client/modify/update/index.js +75 -0
  124. package/dist/src/client/modify/upsert/index.d.ts +3 -0
  125. package/dist/src/client/modify/{upsert.js → upsert/index.js} +4 -4
  126. package/dist/src/client/modify/validate.d.ts +6 -0
  127. package/dist/src/client/modify/validate.js +24 -0
  128. package/dist/src/client/query/BasedDbQuery.d.ts +18 -15
  129. package/dist/src/client/query/BasedDbQuery.js +93 -96
  130. package/dist/src/client/query/{BasedIterable.d.ts → BasedQueryResponse.d.ts} +5 -4
  131. package/dist/src/client/query/{BasedIterable.js → BasedQueryResponse.js} +15 -10
  132. package/dist/src/client/query/aggregates/aggregation.d.ts +5 -3
  133. package/dist/src/client/query/aggregates/aggregation.js +105 -48
  134. package/dist/src/client/query/aggregates/types.d.ts +34 -12
  135. package/dist/src/client/query/aggregates/types.js +30 -12
  136. package/dist/src/client/query/debug.js +1 -1
  137. package/dist/src/client/query/display.d.ts +1 -1
  138. package/dist/src/client/query/display.js +21 -4
  139. package/dist/src/client/query/filter/FilterBranch.js +2 -2
  140. package/dist/src/client/query/filter/convertFilter.d.ts +3 -2
  141. package/dist/src/client/query/filter/convertFilter.js +13 -1
  142. package/dist/src/client/query/filter/createFixedFilterBuffer.js +16 -30
  143. package/dist/src/client/query/filter/createReferenceFilter.js +6 -20
  144. package/dist/src/client/query/filter/createVariableFilterBuffer.js +15 -32
  145. package/dist/src/client/query/filter/parseFilterValue.js +3 -11
  146. package/dist/src/client/query/filter/primitiveFilter.js +1 -1
  147. package/dist/src/client/query/filter/toBuffer.js +9 -15
  148. package/dist/src/client/query/filter/types.d.ts +6 -6
  149. package/dist/src/client/query/filter/types.js +8 -8
  150. package/dist/src/client/query/include/include.d.ts +3 -0
  151. package/dist/src/client/query/include/include.js +58 -0
  152. package/dist/src/client/query/include/props.d.ts +7 -7
  153. package/dist/src/client/query/include/props.js +26 -21
  154. package/dist/src/client/query/include/toByteCode.js +188 -0
  155. package/dist/src/client/query/include/utils.d.ts +3 -1
  156. package/dist/src/client/query/include/utils.js +13 -0
  157. package/dist/src/client/query/include/walk.d.ts +2 -2
  158. package/dist/src/client/query/include/walk.js +38 -23
  159. package/dist/src/client/query/query.d.ts +2 -3
  160. package/dist/src/client/query/query.js +2 -3
  161. package/dist/src/client/query/queryDef.d.ts +4 -2
  162. package/dist/src/client/query/queryDef.js +3 -5
  163. package/dist/src/client/query/queryDefToReadSchema.d.ts +3 -0
  164. package/dist/src/client/query/queryDefToReadSchema.js +126 -0
  165. package/dist/src/client/query/registerQuery.js +15 -4
  166. package/dist/src/client/query/search/index.d.ts +2 -1
  167. package/dist/src/client/query/search/index.js +9 -2
  168. package/dist/src/client/query/sort.js +3 -0
  169. package/dist/src/client/query/subscription/index.d.ts +1 -1
  170. package/dist/src/client/query/subscription/index.js +2 -2
  171. package/dist/src/client/query/subscription/types.d.ts +1 -1
  172. package/dist/src/client/query/thresholds.d.ts +1 -1
  173. package/dist/src/client/query/thresholds.js +1 -1
  174. package/dist/src/client/query/toByteCode/{toBuffer.js → toByteCode.js} +8 -8
  175. package/dist/src/client/query/types.d.ts +40 -25
  176. package/dist/src/client/query/types.js +2 -6
  177. package/dist/src/client/query/validation.d.ts +6 -0
  178. package/dist/src/client/query/validation.js +32 -8
  179. package/dist/src/client/setLocalClientSchema.js +6 -10
  180. package/dist/src/client/string.d.ts +3 -7
  181. package/dist/src/client/string.js +18 -47
  182. package/dist/src/hooks.d.ts +2 -3
  183. package/dist/src/hooks.js +14 -9
  184. package/dist/src/index.d.ts +3 -5
  185. package/dist/src/index.js +4 -6
  186. package/dist/src/native.d.ts +3 -2
  187. package/dist/src/native.js +9 -8
  188. package/dist/src/server/IoWorker.js +2 -3
  189. package/dist/src/server/QueryWorker.js +1 -1
  190. package/dist/src/server/blocks.js +5 -4
  191. package/dist/src/server/dbHash.js +1 -1
  192. package/dist/src/server/index.d.ts +4 -6
  193. package/dist/src/server/index.js +63 -96
  194. package/dist/src/server/migrate/index.d.ts +2 -4
  195. package/dist/src/server/migrate/index.js +31 -16
  196. package/dist/src/server/migrate/worker.js +2 -27
  197. package/dist/src/server/save.js +10 -8
  198. package/dist/src/server/schema.js +17 -25
  199. package/dist/src/server/start.d.ts +1 -0
  200. package/dist/src/server/start.js +18 -10
  201. package/dist/src/server/tree.d.ts +20 -0
  202. package/dist/src/server/tree.js +17 -2
  203. package/dist/src/server/workers/DbWorker.d.ts +1 -0
  204. package/dist/src/server/workers/DbWorker.js +8 -1
  205. package/dist/src/server/workers/io_worker.js +1 -1
  206. package/dist/src/server/workers/worker.js +9 -4
  207. package/dist/src/table.d.ts +4 -0
  208. package/dist/src/table.js +97 -0
  209. package/dist/src/types.d.ts +1 -1
  210. package/package.json +18 -17
  211. package/dist/lib/darwin_aarch64/include/selva/worker_ctx.h +0 -29
  212. package/dist/lib/linux_aarch64/include/selva/worker_ctx.h +0 -29
  213. package/dist/lib/linux_x86_64/include/selva/worker_ctx.h +0 -29
  214. package/dist/src/client/flushModify.d.ts +0 -36
  215. package/dist/src/client/flushModify.js +0 -197
  216. package/dist/src/client/modify/ModifyRes.d.ts +0 -23
  217. package/dist/src/client/modify/ModifyRes.js +0 -101
  218. package/dist/src/client/modify/alias.d.ts +0 -4
  219. package/dist/src/client/modify/alias.js +0 -57
  220. package/dist/src/client/modify/binary.d.ts +0 -6
  221. package/dist/src/client/modify/binary.js +0 -80
  222. package/dist/src/client/modify/cardinality.d.ts +0 -6
  223. package/dist/src/client/modify/cardinality.js +0 -63
  224. package/dist/src/client/modify/create.d.ts +0 -5
  225. package/dist/src/client/modify/create.js +0 -228
  226. package/dist/src/client/modify/delete.d.ts +0 -4
  227. package/dist/src/client/modify/delete.js +0 -47
  228. package/dist/src/client/modify/expire.d.ts +0 -3
  229. package/dist/src/client/modify/expire.js +0 -25
  230. package/dist/src/client/modify/fixed.d.ts +0 -5
  231. package/dist/src/client/modify/fixed.js +0 -235
  232. package/dist/src/client/modify/index.d.ts +0 -4
  233. package/dist/src/client/modify/index.js +0 -4
  234. package/dist/src/client/modify/json.d.ts +0 -4
  235. package/dist/src/client/modify/json.js +0 -26
  236. package/dist/src/client/modify/modify.d.ts +0 -5
  237. package/dist/src/client/modify/modify.js +0 -134
  238. package/dist/src/client/modify/references/appendEdgeRefs.d.ts +0 -4
  239. package/dist/src/client/modify/references/appendEdgeRefs.js +0 -33
  240. package/dist/src/client/modify/references/edge.d.ts +0 -5
  241. package/dist/src/client/modify/references/edge.js +0 -322
  242. package/dist/src/client/modify/references/getEdgeSize.d.ts +0 -3
  243. package/dist/src/client/modify/references/getEdgeSize.js +0 -27
  244. package/dist/src/client/modify/references/reference.d.ts +0 -11
  245. package/dist/src/client/modify/references/reference.js +0 -112
  246. package/dist/src/client/modify/references/references.d.ts +0 -16
  247. package/dist/src/client/modify/references/references.js +0 -308
  248. package/dist/src/client/modify/setCursor.d.ts +0 -5
  249. package/dist/src/client/modify/setCursor.js +0 -33
  250. package/dist/src/client/modify/string.d.ts +0 -5
  251. package/dist/src/client/modify/string.js +0 -71
  252. package/dist/src/client/modify/text.d.ts +0 -8
  253. package/dist/src/client/modify/text.js +0 -69
  254. package/dist/src/client/modify/update.d.ts +0 -6
  255. package/dist/src/client/modify/update.js +0 -90
  256. package/dist/src/client/modify/upsert.d.ts +0 -3
  257. package/dist/src/client/modify/vector.d.ts +0 -4
  258. package/dist/src/client/modify/vector.js +0 -53
  259. package/dist/src/client/query/include/toBuffer.js +0 -123
  260. package/dist/src/client/query/read/read.d.ts +0 -9
  261. package/dist/src/client/query/read/read.js +0 -483
  262. /package/dist/src/client/query/include/{toBuffer.d.ts → toByteCode.d.ts} +0 -0
  263. /package/dist/src/client/query/toByteCode/{toBuffer.d.ts → toByteCode.d.ts} +0 -0
@@ -1,6 +1,6 @@
1
1
  import { BasedDb, save } from '../../index.js';
2
2
  import { dirname, join } from 'path';
3
- import { Worker, MessageChannel, receiveMessageOnPort, } from 'node:worker_threads';
3
+ import { Worker, MessageChannel, } from 'node:worker_threads';
4
4
  import native from '../../native.js';
5
5
  import { destructureTreeKey } from '../tree.js';
6
6
  import { foreachDirtyBlock } from '../blocks.js';
@@ -8,6 +8,7 @@ import { fileURLToPath } from 'url';
8
8
  import { setNativeSchema, setSchemaOnServer, writeSchemaFile, } from '../schema.js';
9
9
  import { setToAwake, waitUntilSleeping } from './utils.js';
10
10
  import { serialize } from '@based/schema';
11
+ import { parse, parseRange, satisfies } from '@std/semver';
11
12
  const __filename = fileURLToPath(import.meta.url);
12
13
  const __dirname = dirname(__filename);
13
14
  const workerPath = join(__dirname, 'worker.js');
@@ -28,10 +29,39 @@ const parseTransform = (transform) => {
28
29
  }
29
30
  return res;
30
31
  };
32
+ const stripHooks = (schema) => {
33
+ const res = {};
34
+ for (const i in schema) {
35
+ if (i === 'types') {
36
+ res[i] = {};
37
+ for (const type in schema.types) {
38
+ const { hooks: _, ...rest } = schema.types[type];
39
+ res[i][type] = rest;
40
+ }
41
+ }
42
+ else {
43
+ res[i] = schema[i];
44
+ }
45
+ }
46
+ return res;
47
+ };
31
48
  export const migrate = async (server, fromSchema, toSchema, transform) => {
32
49
  const migrationId = toSchema.hash;
33
50
  server.migrating = migrationId;
34
51
  server.emit('info', `migrating schema ${migrationId}`);
52
+ fromSchema = stripHooks(fromSchema);
53
+ toSchema = stripHooks(toSchema);
54
+ if (!transform && toSchema.migrations?.length) {
55
+ const fromVersion = fromSchema.version || '0.0.0';
56
+ transform = {};
57
+ for (const { migrate, version } of toSchema.migrations) {
58
+ if (satisfies(parse(fromVersion), parseRange(version))) {
59
+ for (const type in migrate) {
60
+ transform[type] = migrate[type];
61
+ }
62
+ }
63
+ }
64
+ }
35
65
  let killed = false;
36
66
  const abort = () => {
37
67
  if (killed) {
@@ -77,11 +107,9 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
77
107
  to: toAddress,
78
108
  fromSchema: serialize(fromSchema, {
79
109
  stripMetaInformation: true,
80
- stripTransform: true,
81
110
  }),
82
111
  toSchema: serialize(toSchema, {
83
112
  stripMetaInformation: true,
84
- stripTransform: true,
85
113
  }),
86
114
  channel: port2,
87
115
  workerState,
@@ -141,22 +169,9 @@ export const migrate = async (server, fromSchema, toSchema, transform) => {
141
169
  await Promise.all([tmpDb.destroy(), worker.terminate()]);
142
170
  return;
143
171
  }
144
- let msg;
145
- let schemaTypesParsed;
146
- while ((msg = receiveMessageOnPort(port1))) {
147
- schemaTypesParsed = msg.message;
148
- }
149
172
  server.dbCtxExternal = toCtx;
150
173
  server.sortIndexes = {};
151
- // ----------------MAKE NICE THIS------------------
152
- // pass last node IDS { type: lastId }
153
174
  setSchemaOnServer(server, toSchema);
154
- for (const key in schemaTypesParsed) {
155
- // maybe only send the lastId
156
- const def = server.schemaTypesParsed[key];
157
- def.lastId = schemaTypesParsed[key].lastId;
158
- }
159
- // -----------------------------------------
160
175
  tmpDb.server.dbCtxExternal = fromCtx;
161
176
  // TODO makes this SYNC
162
177
  const promises = [server.ioWorker, ...server.workers].map((worker) => worker.updateCtx(toAddress));
@@ -2,7 +2,6 @@ import { isMainThread, receiveMessageOnPort, workerData, } from 'node:worker_thr
2
2
  import native from '../../native.js';
3
3
  import { BasedDb } from '../../index.js';
4
4
  import { REFERENCE, REFERENCES } from '@based/schema/def';
5
- import { isTypedArray } from 'node:util/types';
6
5
  import { setSchemaOnServer } from '../schema.js';
7
6
  import { setToSleep } from './utils.js';
8
7
  import { setLocalClientSchema } from '../../client/setLocalClientSchema.js';
@@ -14,32 +13,10 @@ else if (workerData?.isDbMigrateWorker) {
14
13
  const { from, to, fromSchema, toSchema, channel, workerState, transformFns } = workerData;
15
14
  const fromCtx = native.externalFromInt(from);
16
15
  const toCtx = native.externalFromInt(to);
17
- // worker ctx init - maybe instead of this just add it on native
18
- // instead of here just do this in native.js
19
- native.workerCtxInit();
16
+ native.createThreadCtx(fromCtx, native.getThreadId());
17
+ native.createThreadCtx(toCtx, native.getThreadId());
20
18
  const fromDb = new BasedDb({ path: null });
21
19
  const toDb = new BasedDb({ path: null });
22
- const cp = (obj) => {
23
- let copy;
24
- for (const key in obj) {
25
- const val = obj[key];
26
- if (typeof val === 'number') {
27
- // only copy numbers
28
- copy ??= Array.isArray(obj) ? [] : {};
29
- copy[key] = val;
30
- }
31
- else if (typeof val === 'object' &&
32
- val !== null &&
33
- !isTypedArray(val)) {
34
- const res = cp(val);
35
- if (res) {
36
- copy ??= Array.isArray(obj) ? [] : {};
37
- copy[key] = cp(val);
38
- }
39
- }
40
- }
41
- return copy;
42
- };
43
20
  fromDb.server.dbCtxExternal = fromCtx;
44
21
  toDb.server.dbCtxExternal = toCtx;
45
22
  setSchemaOnServer(fromDb.server, deSerialize(fromSchema));
@@ -115,8 +92,6 @@ else if (workerData?.isDbMigrateWorker) {
115
92
  }
116
93
  await toDb.drain();
117
94
  native.membarSyncWrite();
118
- // WE ARE ONLY GOING TO SEND { type: lastNodeId }
119
- channel.postMessage(cp(toDb.server.schemaTypesParsed));
120
95
  setToSleep(workerState);
121
96
  }
122
97
  }
@@ -2,17 +2,18 @@ import native from '../native.js';
2
2
  import { isMainThread } from 'node:worker_threads';
3
3
  import { writeFile } from 'node:fs/promises';
4
4
  import { join } from 'node:path';
5
- import { VerifTree, destructureTreeKey, } from './tree.js';
5
+ import { VerifTree, destructureTreeKey } from './tree.js';
6
6
  import { saveBlock, foreachBlock, foreachDirtyBlock, saveBlocks, } from './blocks.js';
7
7
  import { writeFileSync } from 'node:fs';
8
- import { bufToHex } from '@saulx/utils';
8
+ import { bufToHex } from '@based/utils';
9
9
  import { COMMON_SDB_FILE, WRITELOG_FILE } from '../types.js';
10
10
  function hasPartialTypes(db) {
11
- let res = false;
12
- for (let id in db.schemaTypesParsedById) {
13
- res = res || db.schemaTypesParsedById[id].partial;
11
+ for (const id in db.schemaTypesParsedById) {
12
+ if (db.schemaTypesParsedById[id].partial) {
13
+ return true;
14
+ }
14
15
  }
15
- return res;
16
+ return false;
16
17
  }
17
18
  function inhibitSave(db, { skipDirtyCheck, forceFullDump, skipMigrationCheck }) {
18
19
  // RFE isMainThread needed??
@@ -100,8 +101,9 @@ export function saveSync(db, opts = {}) {
100
101
  }
101
102
  }
102
103
  export async function save(db, opts = {}) {
103
- if (inhibitSave(db, opts))
104
+ if (inhibitSave(db, opts)) {
104
105
  return;
106
+ }
105
107
  let ts = Date.now();
106
108
  db.saveInProgress = true;
107
109
  try {
@@ -138,7 +140,7 @@ export async function save(db, opts = {}) {
138
140
  blocks.push({
139
141
  filepath,
140
142
  typeId,
141
- start
143
+ start,
142
144
  });
143
145
  });
144
146
  }
@@ -1,13 +1,15 @@
1
1
  import { schemaToSelvaBuffer, updateTypeDefs } from '@based/schema/def';
2
- import { deepCopy, writeUint64 } from '@saulx/utils';
2
+ import { deepCopy } from '@based/utils';
3
3
  import { getPropType, serialize } from '@based/schema';
4
4
  import { join } from 'node:path';
5
5
  import { writeFile } from 'node:fs/promises';
6
6
  import native from '../native.js';
7
- import { makeTreeKey } from './tree.js';
8
7
  import { SCHEMA_FILE } from '../types.js';
9
8
  import { saveSync } from './save.js';
10
- import { hash } from '@saulx/hash';
9
+ import { hash } from '@based/hash';
10
+ import { writeCreate } from '../client/modify/create/index.js';
11
+ import { Ctx } from '../client/modify/Ctx.js';
12
+ import { consume } from '../client/modify/drain.js';
11
13
  export const setSchemaOnServer = (server, schema) => {
12
14
  const { schemaTypesParsed, schemaTypesParsedById } = updateTypeDefs(schema);
13
15
  server.schema = schema;
@@ -15,12 +17,14 @@ export const setSchemaOnServer = (server, schema) => {
15
17
  server.schemaTypesParsedById = schemaTypesParsedById;
16
18
  };
17
19
  export const writeSchemaFile = async (server, schema) => {
18
- const schemaFilePath = join(server.fileSystemPath, SCHEMA_FILE);
19
- try {
20
- await writeFile(schemaFilePath, serialize(schema));
21
- }
22
- catch (err) {
23
- throw new Error(`Error writing schema to a file path ${schemaFilePath}}`);
20
+ if (server.fileSystemPath) {
21
+ const schemaFilePath = join(server.fileSystemPath, SCHEMA_FILE);
22
+ try {
23
+ await writeFile(schemaFilePath, serialize(schema));
24
+ }
25
+ catch (err) {
26
+ throw new Error(`Error writing schema to a file path ${schemaFilePath}}`);
27
+ }
24
28
  }
25
29
  };
26
30
  export const setNativeSchema = (server, schema) => {
@@ -37,23 +41,11 @@ export const setNativeSchema = (server, schema) => {
37
41
  }
38
42
  // Insert a root node
39
43
  if (schema.types._root) {
40
- // TODO fix server add it in schema at least
41
- const data = [2, 1, 0, 0, 0, 1, 9, 1, 0, 0, 0, 7, 1, 0, 1];
42
- const blockKey = makeTreeKey(1, 1);
43
- const buf = new Uint8Array(8 + data.length + 2 + 8 + 4);
44
- const view = new DataView(buf.buffer, 0, buf.byteLength);
45
- // set schema hash
46
- writeUint64(buf, server.schema.hash, 0);
47
- // add content
48
- buf.set(data, 8);
49
- // add typesLen
50
- view.setFloat64(8 + data.length, 0, true);
51
- // add dirty key
52
- view.setFloat64(8 + data.length + 2, blockKey, true);
53
- // add dataLen
54
- view.setUint32(buf.length - 4, data.length, true);
44
+ const tmpArr = new Uint8Array(new ArrayBuffer(1e3, { maxByteLength: 10e3 }));
45
+ const tmpCtx = new Ctx(schema.hash, tmpArr);
46
+ writeCreate(tmpCtx, server.schemaTypesParsed._root, {}, null);
47
+ const buf = consume(tmpCtx);
55
48
  server.modify(buf);
56
- //server.verifTree = new VerifTree(.schemaTypesParsed)
57
49
  }
58
50
  server.verifTree.updateTypes(server.schemaTypesParsed);
59
51
  if (server.fileSystemPath) {
@@ -2,6 +2,7 @@ import { DbServer } from './index.js';
2
2
  export type StartOpts = {
3
3
  clean?: boolean;
4
4
  hosted?: boolean;
5
+ noLoadDumps?: boolean;
5
6
  delayInMs?: number;
6
7
  queryThreads?: number;
7
8
  };
@@ -10,7 +10,7 @@ import exitHook from 'exit-hook';
10
10
  import { save, saveSync } from './save.js';
11
11
  import { deSerialize } from '@based/schema';
12
12
  import { BLOCK_CAPACITY_DEFAULT } from '@based/schema/def';
13
- import { bufToHex, equals, hexToBuf, wait } from '@saulx/utils';
13
+ import { bufToHex, equals, hexToBuf, wait } from '@based/utils';
14
14
  import { SCHEMA_FILE, WRITELOG_FILE, SCHEMA_FILE_DEPRECATED } from '../types.js';
15
15
  import { setSchemaOnServer } from './schema.js';
16
16
  function startWorkers(db, opts) {
@@ -34,6 +34,11 @@ export async function start(db, opts) {
34
34
  let partials = []; // Blocks that exists but were not loaded [key, hash]
35
35
  try {
36
36
  writelog = JSON.parse((await readFile(join(path, WRITELOG_FILE))).toString());
37
+ }
38
+ catch (err) {
39
+ // No dump
40
+ }
41
+ if (writelog) {
37
42
  // Load the common dump
38
43
  try {
39
44
  native.loadCommon(join(path, writelog.commonDump), db.dbCtxExternal);
@@ -42,6 +47,7 @@ export async function start(db, opts) {
42
47
  console.error(e.message);
43
48
  throw e;
44
49
  }
50
+ // Load schema
45
51
  const schema = await readFile(join(path, SCHEMA_FILE)).catch(noop);
46
52
  if (schema) {
47
53
  const s = deSerialize(schema);
@@ -53,14 +59,14 @@ export async function start(db, opts) {
53
59
  setSchemaOnServer(db, JSON.parse(schemaJson.toString()));
54
60
  }
55
61
  }
56
- // Load all range dumps
62
+ // Load block dumps
57
63
  for (const typeId in writelog.rangeDumps) {
58
64
  const dumps = writelog.rangeDumps[typeId];
59
65
  const def = db.schemaTypesParsedById[typeId];
60
- for (const dump of dumps) {
61
- const fname = dump.file;
62
- if (fname?.length > 0) {
63
- if (!def.partial) {
66
+ if (!def.partial && !opts?.noLoadDumps) {
67
+ for (const dump of dumps) {
68
+ const fname = dump.file;
69
+ if (fname?.length > 0) {
64
70
  try {
65
71
  // Can't use loadBlock() yet because verifTree is not avail
66
72
  native.loadBlock(join(path, fname), db.dbCtxExternal);
@@ -69,7 +75,12 @@ export async function start(db, opts) {
69
75
  console.error(e.message);
70
76
  }
71
77
  }
72
- else {
78
+ }
79
+ }
80
+ else {
81
+ for (const dump of dumps) {
82
+ const fname = dump.file;
83
+ if (fname?.length > 0) {
73
84
  partials.push([
74
85
  makeTreeKey(def.id, dump.start),
75
86
  hexToBuf(dump.hash),
@@ -79,9 +90,6 @@ export async function start(db, opts) {
79
90
  }
80
91
  }
81
92
  }
82
- catch (err) {
83
- // TODO In some cases we really should give up!
84
- }
85
93
  db.verifTree = new VerifTree(db.schemaTypesParsed);
86
94
  for (const { typeId } of db.verifTree.types()) {
87
95
  const def = db.schemaTypesParsedById[typeId];
@@ -4,7 +4,15 @@ export declare const makeTreeKey: (typeId: number, start: number) => number;
4
4
  export declare const nodeId2Start: (blockCapacity: number, nodeId: number) => number;
5
5
  export declare const makeTreeKeyFromNodeId: (typeId: number, blockCapacity: number, nodeId: number) => number;
6
6
  type Hash = Uint8Array;
7
+ /**
8
+ * Block state.
9
+ * Type and a node id range.
10
+ */
7
11
  export type VerifBlock = {
12
+ /**
13
+ * key = typeId + startNodeId
14
+ * Made with makeTreeKey(t, i) and can be destructured with destructureTreeKey(k).
15
+ */
8
16
  key: number;
9
17
  /**
10
18
  * Last acquired hash of the block.
@@ -21,10 +29,22 @@ export type VerifBlock = {
21
29
  */
22
30
  loadPromise: null | Promise<void>;
23
31
  };
32
+ /**
33
+ * Container for a whole type.
34
+ */
35
+ type VerifType = {
36
+ typeId: number; /*!< typeId as in the schema. */
37
+ blockCapacity: number; /*!< Max number of nodes per block. */
38
+ blocks: VerifBlock[];
39
+ };
40
+ /**
41
+ * An object that keeps track of all blocks of nodes existing in the database.
42
+ */
24
43
  export declare class VerifTree {
25
44
  #private;
26
45
  constructor(schemaTypesParsed: Record<string, SchemaTypeDef>);
27
46
  types(): Generator<any, void, unknown>;
47
+ blocks(type: VerifType): Generator<VerifBlock, void, unknown>;
28
48
  foreachBlock(cb: (block: VerifBlock) => void): void;
29
49
  get hash(): Uint8Array;
30
50
  update(key: number, hash: Hash, inmem?: boolean): void;
@@ -5,12 +5,15 @@ export const destructureTreeKey = (key) => [
5
5
  ];
6
6
  export const makeTreeKey = (typeId, start) => typeId * 4294967296 + start;
7
7
  export const nodeId2Start = (blockCapacity, nodeId) => ((nodeId - +!(nodeId % blockCapacity)) / blockCapacity) | 0;
8
- const nodeId2BlockI = (nodeId, blockCapacity) => ((nodeId - 1) - ((nodeId - 1) % blockCapacity)) / blockCapacity;
8
+ const nodeId2BlockI = (nodeId, blockCapacity) => (nodeId - 1 - ((nodeId - 1) % blockCapacity)) / blockCapacity;
9
9
  export const makeTreeKeyFromNodeId = (typeId, blockCapacity, nodeId) => {
10
10
  const tmp = nodeId - +!(nodeId % blockCapacity);
11
11
  return typeId * 4294967296 + ((tmp / blockCapacity) | 0) * blockCapacity + 1;
12
12
  };
13
13
  const HASH_SIZE = 16;
14
+ /**
15
+ * An object that keeps track of all blocks of nodes existing in the database.
16
+ */
14
17
  export class VerifTree {
15
18
  #types;
16
19
  #h = createDbHash();
@@ -36,6 +39,12 @@ export class VerifTree {
36
39
  yield this.#types[k];
37
40
  }
38
41
  }
42
+ *blocks(type) {
43
+ const { blocks } = type;
44
+ for (const block of blocks) {
45
+ yield block;
46
+ }
47
+ }
39
48
  foreachBlock(cb) {
40
49
  for (const k of Object.keys(this.#types)) {
41
50
  const { blocks } = this.#types[k];
@@ -57,7 +66,13 @@ export class VerifTree {
57
66
  throw new Error(`type ${typeId} not found`);
58
67
  }
59
68
  const blockI = nodeId2BlockI(start, type.blockCapacity);
60
- const block = type.blocks[blockI] ?? (type.blocks[blockI] = Object.preventExtensions({ key, hash, inmem, loadPromise: null }));
69
+ const block = type.blocks[blockI] ??
70
+ (type.blocks[blockI] = Object.preventExtensions({
71
+ key,
72
+ hash,
73
+ inmem,
74
+ loadPromise: null,
75
+ }));
61
76
  block.hash = hash;
62
77
  block.inmem = inmem;
63
78
  }
@@ -1,6 +1,7 @@
1
1
  import { MessagePort } from 'node:worker_threads';
2
2
  import { DbServer } from '../index.js';
3
3
  export declare abstract class DbWorker {
4
+ #private;
4
5
  constructor(address: BigInt, db: DbServer, onExit: (code: number) => void, workerName: string);
5
6
  protected db: DbServer;
6
7
  protected channel: MessagePort;
@@ -1,9 +1,11 @@
1
1
  import { MessageChannel, Worker } from 'node:worker_threads';
2
2
  import { fileURLToPath } from 'node:url';
3
3
  import { dirname, join } from 'node:path';
4
+ import native from '../../native.js';
4
5
  const __filename = fileURLToPath(import.meta.url);
5
6
  const __dirname = dirname(__filename);
6
7
  export class DbWorker {
8
+ #threadId;
7
9
  constructor(address, db, onExit, workerName) {
8
10
  const { port1, port2 } = new MessageChannel();
9
11
  this.db = db;
@@ -18,7 +20,10 @@ export class DbWorker {
18
20
  });
19
21
  this.readyPromise = new Promise((resolve) => {
20
22
  const onReady = (msg) => {
21
- if (msg === 'READY') {
23
+ if (msg?.status === 'READY') {
24
+ // TODO Also call native.destroyThreadCtx() somewhere
25
+ this.#threadId = msg.threadId;
26
+ native.createThreadCtx(db.dbCtxExternal, this.#threadId);
22
27
  this.worker.off('message', onReady);
23
28
  resolve(true);
24
29
  }
@@ -67,6 +72,8 @@ export class DbWorker {
67
72
  return new Promise(this.callback);
68
73
  }
69
74
  updateCtx(address) {
75
+ const ctx = native.externalFromInt(address);
76
+ native.createThreadCtx(ctx, this.#threadId);
70
77
  return this.call(address);
71
78
  }
72
79
  }
@@ -1,5 +1,5 @@
1
1
  import { registerMsgHandler } from './worker.js';
2
- import { writeInt32 } from '@saulx/utils';
2
+ import { writeInt32 } from '@based/utils';
3
3
  import native from '../../native.js';
4
4
  function loadBlock(dbCtx, filepath) {
5
5
  native.loadBlock(filepath, dbCtx);
@@ -2,14 +2,12 @@ import { isMainThread, parentPort, workerData } from 'node:worker_threads';
2
2
  import { nextTick } from 'node:process';
3
3
  import native from '../../native.js';
4
4
  let dbCtx;
5
- let wCtx; // This must be held until the worker exits otherwise the ctx will be autofreed instantly
6
5
  if (isMainThread) {
7
6
  console.warn(`running a worker in the mainthread - incorrect`);
8
7
  }
9
8
  else if (workerData?.isDbWorker) {
10
9
  let { address } = workerData;
11
10
  dbCtx = native.externalFromInt(address);
12
- wCtx = native.workerCtxInit();
13
11
  }
14
12
  else {
15
13
  console.info('incorrect worker db query');
@@ -24,7 +22,14 @@ export function registerMsgHandler(onMsg) {
24
22
  if (typeof msg === 'bigint') {
25
23
  if (msg === 0n) {
26
24
  // terminate
27
- nextTick(() => typeof self === 'undefined' ? process.exit() : self.close());
25
+ nextTick(() => {
26
+ if (typeof self === 'undefined') {
27
+ process.exit();
28
+ }
29
+ else {
30
+ self.close();
31
+ }
32
+ });
28
33
  channel.postMessage(null);
29
34
  }
30
35
  else {
@@ -44,6 +49,6 @@ export function registerMsgHandler(onMsg) {
44
49
  }
45
50
  };
46
51
  channel.on('message', handleMsg);
47
- parentPort.postMessage('READY');
52
+ parentPort.postMessage({ status: 'READY', threadId: native.getThreadId() });
48
53
  }
49
54
  //# sourceMappingURL=worker.js.map
@@ -0,0 +1,4 @@
1
+ export declare function formatTable(header: string[], align: Array<'l' | 'r' | 'c' | '.'>, blocks: {
2
+ name: string;
3
+ rows: (string | number)[][];
4
+ }[], hsep?: string): string;
@@ -0,0 +1,97 @@
1
+ import { styleText } from 'node:util';
2
+ function dotindex(c) {
3
+ const m = /\.[^.]*$/.exec(c);
4
+ return m ? m.index + 1 : c.length;
5
+ }
6
+ function strlen(str) {
7
+ return str.replace(/\u001b[^m]*m/g, '').length;
8
+ }
9
+ function table(rows_, opts) {
10
+ if (!opts)
11
+ opts = {};
12
+ const hsep = opts.hsep;
13
+ const align = opts.align || [];
14
+ const dotsizes = rows_.reduce((acc, row) => {
15
+ row.forEach((c, ix) => {
16
+ const n = align[ix] == '.' ? dotindex(c) : 0;
17
+ if (!acc[ix] || n > acc[ix])
18
+ acc[ix] = n;
19
+ });
20
+ return acc;
21
+ }, []);
22
+ const rows = rows_.map((row) => {
23
+ return row.map((c_, ix) => {
24
+ const c = String(c_);
25
+ if (align[ix] === '.') {
26
+ const index = dotindex(c);
27
+ const size = dotsizes[ix] + (/\./.test(c) ? 1 : 2) - (strlen(c) - index);
28
+ return c + Array(size).join(' ');
29
+ }
30
+ else
31
+ return c;
32
+ });
33
+ });
34
+ const sizes = rows.reduce((acc, row) => {
35
+ row.forEach((c, ix) => {
36
+ const n = strlen(c);
37
+ if (!acc[ix] || n > acc[ix])
38
+ acc[ix] = n;
39
+ });
40
+ return acc;
41
+ }, []);
42
+ return rows
43
+ .map((row) => {
44
+ return row
45
+ .map((c, ix) => {
46
+ const n = sizes[ix] - strlen(c) || 0;
47
+ const s = Array(Math.max(n + 1, 1)).join(' ');
48
+ if (align[ix] === 'r' || align[ix] === '.') {
49
+ return s + c;
50
+ }
51
+ if (align[ix] === 'c') {
52
+ return (Array(Math.ceil(n / 2 + 1)).join(' ') +
53
+ c +
54
+ Array(Math.floor(n / 2 + 1)).join(' '));
55
+ }
56
+ return c + s;
57
+ })
58
+ .join(hsep)
59
+ .replace(/\s+$/, '');
60
+ })
61
+ .join('\n');
62
+ }
63
+ export function formatTable(header, align, blocks, hsep = ' ') {
64
+ const nrCols = header.length;
65
+ const padding = [];
66
+ let out = '\n';
67
+ for (let i = 0; i < nrCols; i++) {
68
+ padding[i] = blocks.reduce((acc, block) => {
69
+ const maxLen = Math.max(...block.rows.map((row) => strlen(`${row[i]}`)));
70
+ return Math.max(acc, Math.ceil(maxLen / 8));
71
+ }, 1);
72
+ }
73
+ for (const block of blocks) {
74
+ if (block.name) {
75
+ out += `${styleText('bold', block.name)}\n`;
76
+ }
77
+ // @ts-ignore
78
+ const rows = [header.map((s) => styleText('dim', s))].concat(block.rows);
79
+ if (rows.length > 0) {
80
+ rows[0][0] = ` ${rows[0][0]}`;
81
+ for (let i = 1; i < rows.length; i++) {
82
+ const row = rows[i].slice(0);
83
+ row[0] = ` ${row[0]}`;
84
+ for (let j = 0; j < nrCols; j++) {
85
+ const col = `${row[j]}`;
86
+ const al = align[j] || 'l';
87
+ const pad = padding[j] > 1 ? ' '.repeat(padding[j] * 8 - strlen(col)) : '';
88
+ rows[i][j] = al === 'l' ? col + pad : pad + col;
89
+ }
90
+ }
91
+ out += table(rows, { align, hsep });
92
+ }
93
+ out += '\n\n';
94
+ }
95
+ return out.slice(0, -1);
96
+ }
97
+ //# sourceMappingURL=table.js.map
@@ -3,7 +3,7 @@ export declare const SCHEMA_FILE = "schema.bin";
3
3
  export declare const WRITELOG_FILE = "writelog.json";
4
4
  export declare const COMMON_SDB_FILE = "common.sdb";
5
5
  export type BasedDbOpts = {
6
- path: string;
6
+ path: string | null;
7
7
  maxModifySize?: number;
8
8
  debug?: boolean | 'server' | 'client';
9
9
  saveIntervalInSeconds?: number;