@ohos-ports/lmdb 2.5.3 → 2.8.5-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/LICENSE +20 -20
  2. package/README.md +72 -28
  3. package/SECURITY.md +12 -12
  4. package/bin/download-prebuilds.js +15 -0
  5. package/binding.gyp +115 -111
  6. package/build/Release/lmdb.node +0 -0
  7. package/caching.js +212 -156
  8. package/dependencies/lmdb/libraries/liblmdb/COPYRIGHT +20 -20
  9. package/dependencies/lmdb/libraries/liblmdb/Doxyfile +1631 -1631
  10. package/dependencies/lmdb/libraries/liblmdb/LICENSE +47 -47
  11. package/dependencies/lmdb/libraries/liblmdb/Makefile +136 -136
  12. package/dependencies/lmdb/libraries/liblmdb/chacha8.c +183 -183
  13. package/dependencies/lmdb/libraries/liblmdb/chacha8.h +14 -14
  14. package/dependencies/lmdb/libraries/liblmdb/crypto.c +121 -121
  15. package/dependencies/lmdb/libraries/liblmdb/intro.doc +192 -192
  16. package/dependencies/lmdb/libraries/liblmdb/lmdb.h +28 -3
  17. package/dependencies/lmdb/libraries/liblmdb/mdb.c +157 -31
  18. package/dependencies/lmdb/libraries/liblmdb/mdb_copy.1 +74 -74
  19. package/dependencies/lmdb/libraries/liblmdb/mdb_copy.c +106 -106
  20. package/dependencies/lmdb/libraries/liblmdb/mdb_drop.1 +53 -53
  21. package/dependencies/lmdb/libraries/liblmdb/mdb_drop.c +154 -154
  22. package/dependencies/lmdb/libraries/liblmdb/mdb_dump.1 +94 -94
  23. package/dependencies/lmdb/libraries/liblmdb/mdb_dump.c +333 -333
  24. package/dependencies/lmdb/libraries/liblmdb/mdb_load.1 +97 -97
  25. package/dependencies/lmdb/libraries/liblmdb/mdb_load.c +530 -530
  26. package/dependencies/lmdb/libraries/liblmdb/mdb_stat.1 +83 -83
  27. package/dependencies/lmdb/libraries/liblmdb/mdb_stat.c +276 -276
  28. package/dependencies/lmdb/libraries/liblmdb/midl-sparse.c +452 -0
  29. package/dependencies/lmdb/libraries/liblmdb/midl.c +452 -452
  30. package/dependencies/lmdb/libraries/liblmdb/midl.h +208 -208
  31. package/dependencies/lmdb/libraries/liblmdb/module.c +101 -101
  32. package/dependencies/lmdb/libraries/liblmdb/module.h +16 -16
  33. package/dependencies/lmdb/libraries/liblmdb/mtest.c +178 -178
  34. package/dependencies/lmdb/libraries/liblmdb/mtest2.c +124 -124
  35. package/dependencies/lmdb/libraries/liblmdb/mtest3.c +133 -133
  36. package/dependencies/lmdb/libraries/liblmdb/mtest4.c +168 -168
  37. package/dependencies/lmdb/libraries/liblmdb/mtest5.c +135 -135
  38. package/dependencies/lmdb/libraries/liblmdb/mtest6.c +141 -141
  39. package/dependencies/lmdb/libraries/liblmdb/mtest_enc.c +190 -190
  40. package/dependencies/lmdb/libraries/liblmdb/mtest_enc2.c +189 -189
  41. package/dependencies/lmdb/libraries/liblmdb/mtest_remap.c +177 -177
  42. package/dependencies/lmdb/libraries/liblmdb/sample-bdb.txt +73 -73
  43. package/dependencies/lmdb/libraries/liblmdb/sample-mdb.txt +62 -62
  44. package/dependencies/lmdb/libraries/liblmdb/tooltag +27 -27
  45. package/dependencies/lmdb-data-v1/libraries/liblmdb/CHANGES +266 -0
  46. package/dependencies/lmdb-data-v1/libraries/liblmdb/COPYRIGHT +20 -20
  47. package/dependencies/lmdb-data-v1/libraries/liblmdb/Doxyfile +1631 -1631
  48. package/dependencies/lmdb-data-v1/libraries/liblmdb/LICENSE +47 -47
  49. package/dependencies/lmdb-data-v1/libraries/liblmdb/intro.doc +192 -192
  50. package/dependencies/lmdb-data-v1/libraries/liblmdb/lmdb.h +1608 -1653
  51. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb.c +10320 -11359
  52. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_copy.1 +55 -61
  53. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_copy.c +82 -84
  54. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_dump.1 +75 -81
  55. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_dump.c +319 -319
  56. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_load.1 +84 -84
  57. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_load.c +496 -492
  58. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_stat.1 +64 -70
  59. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_stat.c +263 -264
  60. package/dependencies/lmdb-data-v1/libraries/liblmdb/midl.c +359 -421
  61. package/dependencies/lmdb-data-v1/libraries/liblmdb/midl.h +186 -200
  62. package/dependencies/lmdb-data-v1/libraries/liblmdb/mtest.c +177 -177
  63. package/dependencies/lmdb-data-v1/libraries/liblmdb/mtest2.c +124 -124
  64. package/dependencies/lmdb-data-v1/libraries/liblmdb/mtest3.c +133 -133
  65. package/dependencies/lmdb-data-v1/libraries/liblmdb/mtest4.c +168 -168
  66. package/dependencies/lmdb-data-v1/libraries/liblmdb/mtest5.c +135 -135
  67. package/dependencies/lmdb-data-v1/libraries/liblmdb/mtest6.c +141 -141
  68. package/dependencies/lmdb-data-v1/libraries/liblmdb/sample-bdb.txt +73 -73
  69. package/dependencies/lmdb-data-v1/libraries/liblmdb/sample-mdb.txt +62 -62
  70. package/dependencies/lmdb-data-v1/libraries/liblmdb/tooltag +22 -27
  71. package/dependencies/lz4/LICENSE +11 -11
  72. package/dependencies/lz4/lib/LICENSE +24 -24
  73. package/dependencies/lz4/lib/Makefile +225 -0
  74. package/dependencies/lz4/lib/README.md +169 -137
  75. package/dependencies/lz4/lib/dll/example/Makefile +63 -0
  76. package/dependencies/lz4/lib/dll/example/README.md +69 -69
  77. package/dependencies/lz4/lib/dll/liblz4.def +62 -62
  78. package/dependencies/lz4/lib/liblz4-dll.rc.in +35 -35
  79. package/dependencies/lz4/lib/liblz4.pc.in +14 -14
  80. package/dependencies/lz4/lib/lz4.c +2722 -2495
  81. package/dependencies/lz4/lib/lz4.h +842 -774
  82. package/dependencies/lz4/lib/lz4file.c +311 -0
  83. package/dependencies/lz4/lib/lz4file.h +93 -0
  84. package/dependencies/lz4/lib/lz4frame.c +2078 -1899
  85. package/dependencies/lz4/lib/lz4frame.h +692 -623
  86. package/dependencies/lz4/lib/lz4frame_static.h +47 -47
  87. package/dependencies/lz4/lib/lz4hc.c +1631 -1615
  88. package/dependencies/lz4/lib/lz4hc.h +413 -413
  89. package/dependencies/lz4/lib/xxhash.c +1030 -1030
  90. package/dependencies/lz4/lib/xxhash.h +328 -328
  91. package/dependencies/v8/v8-fast-api-calls-v16.h +790 -790
  92. package/dependencies/v8/v8-fast-api-calls.h +784 -888
  93. package/dist/index.cjs +1050 -584
  94. package/dist/index.cjs.map +1 -1
  95. package/index.d.cts +420 -0
  96. package/index.d.ts +450 -385
  97. package/index.js +38 -37
  98. package/keys.js +116 -102
  99. package/level.js +26 -26
  100. package/native.js +8 -5
  101. package/node-index.js +21 -17
  102. package/open.js +51 -17
  103. package/package.json +63 -21
  104. package/read.js +338 -95
  105. package/src/compression.cpp +12 -9
  106. package/src/cursor.cpp +82 -31
  107. package/src/dbi.cpp +78 -40
  108. package/src/env.cpp +336 -68
  109. package/src/lmdb-js.cpp +64 -64
  110. package/src/lmdb-js.h +83 -49
  111. package/src/misc.cpp +297 -28
  112. package/src/ordered-binary.cpp +46 -46
  113. package/src/txn.cpp +210 -257
  114. package/src/v8-functions.cpp +170 -166
  115. package/src/windows.c +34 -34
  116. package/src/writer.cpp +169 -84
  117. package/util/RangeIterable.js +112 -17
  118. package/util/set-optional-deps.cjs +14 -10
  119. package/util/when.js +8 -8
  120. package/write.js +145 -121
  121. package/dependencies/lmdb-data-v1/libraries/liblmdb/Makefile +0 -118
  122. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_drop.1 +0 -40
  123. package/dependencies/lmdb-data-v1/libraries/liblmdb/mdb_drop.c +0 -135
  124. package/rollup.config.js +0 -12
@@ -1276,7 +1276,7 @@ typedef struct MDB_db {
1276
1276
  #define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID))
1277
1277
  /** #mdb_dbi_open() flags */
1278
1278
  #define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\
1279
- MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE)
1279
+ MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE|0x100)
1280
1280
 
1281
1281
  /** Handle for the DB used to track free pages. */
1282
1282
  #define FREE_DBI 0
@@ -1378,6 +1378,9 @@ struct MDB_txn {
1378
1378
  * #mt_workid, then grows as it is copied from children who commit.
1379
1379
  */
1380
1380
  txnid_t mt_last_workid;
1381
+ /** Callback function for when the transaction is visible */
1382
+ MDB_txn_visible* mt_callback;
1383
+ void* mt_ctx;
1381
1384
 
1382
1385
  MDB_env *mt_env; /**< the DB environment */
1383
1386
  /** The list of pages that became unused during this transaction.
@@ -1689,7 +1692,7 @@ struct MDB_env {
1689
1692
  #endif
1690
1693
  void *me_userctx; /**< User-settable context */
1691
1694
  MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
1692
- MDB_check_fd *me_check_fd; /**< Callback for assertion failures */
1695
+ void *me_callback; /**< General callback */
1693
1696
  int64_t boot_id;
1694
1697
  };
1695
1698
 
@@ -1861,6 +1864,8 @@ static char *const mdb_errstr[] = {
1861
1864
  "MDB_BAD_CHECKSUM: Page checksum mismatch",
1862
1865
  "MDB_CRYPTO_FAIL: Page encryption or decryption failed",
1863
1866
  "MDB_ENV_ENCRYPTION: Environment encryption mismatch",
1867
+ "MDB_LOCK_FAILURE: Failed to get robust semaphore, increase max semaphore count by running 'sudo sysctl kern.sysv.semume=200' or build with robust mutexes disabled (see docs)",
1868
+ "MDB_EMPTY_TXN: Transaction was empty",
1864
1869
  };
1865
1870
  //<lmdb-js>
1866
1871
  static char* last_error = NULL;
@@ -1924,7 +1929,7 @@ mdb_strerror(int err)
1924
1929
  buf[0] = 0;
1925
1930
  FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM |
1926
1931
  FORMAT_MESSAGE_IGNORE_INSERTS,
1927
- NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE);
1932
+ NULL, err, 0, ptr, MSGSIZE, (va_list *)NULL);
1928
1933
  return ptr;
1929
1934
  #else
1930
1935
  if (last_error) {
@@ -3066,12 +3071,29 @@ fail:
3066
3071
  return rc;
3067
3072
  }
3068
3073
 
3074
+ #ifdef _WIN32
3075
+ uint64_t get_time64() {
3076
+ return GetTickCount64();
3077
+ }
3078
+ #else
3079
+ uint64_t get_time64() {
3080
+ struct timespec time;
3081
+ clock_gettime(CLOCK_MONOTONIC, &time);
3082
+ return time.tv_sec * 1000000000ll + time.tv_nsec;
3083
+ }
3084
+ #endif
3085
+
3069
3086
  int
3070
3087
  mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs)
3071
3088
  {
3072
3089
  int rc = 0;
3073
3090
  if (env->me_flags & MDB_RDONLY)
3074
3091
  return EACCES;
3092
+ uint64_t start = 0;
3093
+ if (env->me_flags & MDB_TRACK_METRICS) {
3094
+ start = get_time64();
3095
+ }
3096
+
3075
3097
  if (force || !(env->me_flags & MDB_NOSYNC)
3076
3098
  #ifdef _WIN32 /* Sync is normally achieved in Windows by doing WRITE_THROUGH writes */
3077
3099
  && (env->me_flags & MDB_WRITEMAP)
@@ -3097,6 +3119,9 @@ mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs)
3097
3119
  rc = ErrCode();
3098
3120
  }
3099
3121
  }
3122
+ if (env->me_flags & MDB_TRACK_METRICS) {
3123
+ ((MDB_metrics*) env->me_userctx)->time_sync += get_time64() - start;
3124
+ }
3100
3125
  return rc;
3101
3126
  }
3102
3127
 
@@ -3364,6 +3389,9 @@ mdb_txn_renew0(MDB_txn *txn)
3364
3389
 
3365
3390
  } else {
3366
3391
  /* Not yet touching txn == env->me_txn0, it may be active */
3392
+ if (env->me_flags & MDB_TRACK_METRICS) {
3393
+ ((MDB_metrics*) env->me_userctx)->clock_txn = get_time64();
3394
+ }
3367
3395
  if (ti) {
3368
3396
  if (LOCK_MUTEX(rc, env, env->me_wmutex))
3369
3397
  return rc;
@@ -3373,6 +3401,11 @@ mdb_txn_renew0(MDB_txn *txn)
3373
3401
  meta = mdb_env_pick_meta(env);
3374
3402
  txn->mt_txnid = meta->mm_txnid;
3375
3403
  }
3404
+ if (env->me_flags & MDB_TRACK_METRICS) {
3405
+ uint64_t now = get_time64();
3406
+ ((MDB_metrics*) env->me_userctx)->time_start_txns += now - ((MDB_metrics*) env->me_userctx)->clock_txn;
3407
+ ((MDB_metrics*) env->me_userctx)->clock_txn = now;
3408
+ }
3376
3409
  txn->mt_txnid++;
3377
3410
  #if MDB_DEBUG
3378
3411
  if (txn->mt_txnid == mdb_debug_start)
@@ -3496,6 +3529,16 @@ mdb_workid_rewind(MDB_txn *txn)
3496
3529
  return workid;
3497
3530
  }
3498
3531
 
3532
+ int ESECT
3533
+ mdb_txn_set_callback(MDB_txn *txn, MDB_txn_visible *func, void* ctx)
3534
+ {
3535
+ if (!txn)
3536
+ return EINVAL;
3537
+ txn->mt_callback = func;
3538
+ txn->mt_ctx = ctx;
3539
+ return MDB_SUCCESS;
3540
+ }
3541
+
3499
3542
  int
3500
3543
  mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
3501
3544
  {
@@ -3605,6 +3648,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
3605
3648
  renew:
3606
3649
  rc = mdb_txn_renew0(txn);
3607
3650
  }
3651
+ txn->mt_callback = NULL;
3608
3652
  if (rc) {
3609
3653
  if (txn != env->me_txn0) {
3610
3654
  #if MDB_RPAGE_CACHE
@@ -3841,7 +3885,6 @@ mdb_freelist_save(MDB_txn *txn)
3841
3885
  unsigned x;
3842
3886
  if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0)
3843
3887
  return rc;
3844
- lost_loose = txn->mt_loose_count;
3845
3888
  for (; mp; mp = NEXT_LOOSE_PAGE(mp)) {
3846
3889
  mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno);
3847
3890
  /* must also remove from dirty list */
@@ -4068,10 +4111,14 @@ mdb_page_flush(MDB_txn *txn, int keep)
4068
4111
  MDB_OFF_T pos = 0;
4069
4112
  pgno_t pgno = 0;
4070
4113
  MDB_page *dp = NULL;
4114
+ uint64_t start = 0;
4115
+ int write_i = 0;
4116
+ if (env->me_flags & MDB_TRACK_METRICS) {
4117
+ start = get_time64();
4118
+ }
4071
4119
  #ifdef _WIN32
4072
4120
  OVERLAPPED *ov, *this_ov;
4073
4121
  MDB_page *wdp;
4074
- int async_i = 0;
4075
4122
  HANDLE fd = (txn->mt_flags & MDB_NOSYNC) ? env->me_fd : env->me_ovfd;
4076
4123
  #else
4077
4124
  struct iovec iov[MDB_COMMIT_PAGES];
@@ -4089,13 +4136,18 @@ mdb_page_flush(MDB_txn *txn, int keep)
4089
4136
 
4090
4137
  /* setup nump list, flag that it's in use */
4091
4138
  dl_nump = env->me_dirty_nump;
4139
+ /* <lmdb-js addition> */
4092
4140
  for (n=1; n<=pagecount; n++) {
4093
4141
  dp = dl[n].mptr;
4094
4142
  dl_nump[n] = IS_OVERFLOW(dp) ? dp->mp_pages : 1;
4143
+ pgno_t p = dl[n].mid + dl_nump[n];
4144
+ if (p > pgno)
4145
+ pgno = p;
4146
+ else
4147
+ fprintf(stderr, "Page writes are out of order\n");
4095
4148
  }
4096
4149
  txn->mt_flags |= MDB_TXN_DIRTYNUM;
4097
4150
  /* <lmdb-js addition> */
4098
- pgno = dl[pagecount].mid + dl_nump[pagecount];
4099
4151
  n = 0;
4100
4152
 
4101
4153
  #ifdef _WIN32
@@ -4112,7 +4164,8 @@ mdb_page_flush(MDB_txn *txn, int keep)
4112
4164
  rc = SetEndOfFile(fd);
4113
4165
  if (!rc) {
4114
4166
  rc = ErrCode();
4115
- fprintf(stderr, "SetEndOfFile error %s\n", strerror(rc));
4167
+ if (rc != 1224) // occassionaly see a ERROR_USER_MAPPED_FILE with no apparent ill-effect
4168
+ fprintf(stderr, "SetEndOfFile error, extending to %u %i %s\n", file_size, rc, strerror(rc));
4116
4169
  }
4117
4170
  }
4118
4171
  }
@@ -4191,7 +4244,7 @@ retry_write:
4191
4244
  rc = 0;
4192
4245
  /* Write previous page(s) */
4193
4246
  #ifdef _WIN32
4194
- this_ov = &ov[async_i];
4247
+ this_ov = &ov[write_i];
4195
4248
  /* Clear status, and keep hEvent, we reuse that */
4196
4249
  this_ov->Internal = 0;
4197
4250
  this_ov->Offset = wpos & 0xffffffff;
@@ -4213,7 +4266,6 @@ retry_write:
4213
4266
  }
4214
4267
  rc = 0;
4215
4268
  }
4216
- async_i++;
4217
4269
  #else /* _WIN32 */
4218
4270
  #ifdef MDB_USE_PWRITEV
4219
4271
  wres = pwritev(fd, iov, n, wpos);
@@ -4246,6 +4298,7 @@ retry_seek:
4246
4298
  }
4247
4299
  }
4248
4300
  #endif /* _WIN32 */
4301
+ write_i++;
4249
4302
  if (rc)
4250
4303
  return rc;
4251
4304
  n = 0;
@@ -4293,6 +4346,11 @@ retry_seek:
4293
4346
  * flushed.
4294
4347
  */
4295
4348
  CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE);
4349
+ if (env->me_flags & MDB_TRACK_METRICS) {
4350
+ ((MDB_metrics*) env->me_userctx)->writes += write_i;
4351
+ ((MDB_metrics*) env->me_userctx)->page_flushes++;
4352
+ ((MDB_metrics*) env->me_userctx)->pages_written += pagecount - keep;
4353
+ }
4296
4354
 
4297
4355
  #ifdef _WIN32
4298
4356
  if (!(txn->mt_flags & MDB_NOSYNC)) {
@@ -4301,9 +4359,9 @@ retry_seek:
4301
4359
  * we reduce thread suspend/resuming (in practice, typically about 99.5% of writes are
4302
4360
  * done after the last write is done) */
4303
4361
  rc = 0;
4304
- while (--async_i >= 0) {
4305
- if (ov[async_i].hEvent) {
4306
- if (!GetOverlappedResult(fd, &ov[async_i], &wres, TRUE)) {
4362
+ while (--write_i >= 0) {
4363
+ if (ov[write_i].hEvent) {
4364
+ if (!GetOverlappedResult(fd, &ov[write_i], &wres, TRUE)) {
4307
4365
  rc = ErrCode(); /* Continue on so that all the event signals are reset */
4308
4366
  }
4309
4367
  }
@@ -4334,7 +4392,9 @@ done:
4334
4392
  i--;
4335
4393
  txn->mt_dirty_room += i - j;
4336
4394
  dl[0].mid = j;
4337
-
4395
+ if (env->me_flags & MDB_TRACK_METRICS) {
4396
+ ((MDB_metrics*) env->me_userctx)->time_page_flushes += get_time64() - start;
4397
+ }
4338
4398
  return MDB_SUCCESS;
4339
4399
  }
4340
4400
 
@@ -4514,8 +4574,10 @@ mdb_txn_commit(MDB_txn *txn)
4514
4574
  mdb_cursors_close(txn, 0);
4515
4575
 
4516
4576
  if (!txn->mt_u.dirty_list[0].mid &&
4517
- !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS)))
4518
- goto done;
4577
+ !(txn->mt_flags & (MDB_TXN_DIRTY | MDB_TXN_SPILLS))) {
4578
+ mdb_txn_end(txn, end_mode);
4579
+ return MDB_EMPTY_TXN;
4580
+ }
4519
4581
 
4520
4582
  DPRINTF(("committing txn %"Yu" %p on mdbenv %p, root page %"Yu,
4521
4583
  txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root));
@@ -4554,7 +4616,7 @@ mdb_txn_commit(MDB_txn *txn)
4554
4616
  #if (MDB_DEBUG) > 2
4555
4617
  mdb_audit(txn);
4556
4618
  #endif
4557
-
4619
+ int dirty_pages = txn->mt_u.dirty_list[0].mid;
4558
4620
  if ((rc = mdb_page_flush(txn, 0)))
4559
4621
  goto fail;
4560
4622
  if ((unsigned)txn->mt_loose_count < txn->mt_u.dirty_list[0].mid) {
@@ -4579,6 +4641,11 @@ mdb_txn_commit(MDB_txn *txn)
4579
4641
  goto fail;
4580
4642
 
4581
4643
  //<lmdb-js>
4644
+ if (txn->mt_callback) {
4645
+ MDB_txn_visible* callback = txn->mt_callback;
4646
+ callback(txn->mt_ctx);
4647
+ txn->mt_callback = NULL;
4648
+ }
4582
4649
  if (!F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC))
4583
4650
  env->me_synced_txn_id = txn->mt_txnid;
4584
4651
  //</lmdb-js>
@@ -4594,7 +4661,43 @@ mdb_txn_commit(MDB_txn *txn)
4594
4661
  }
4595
4662
 
4596
4663
  done:
4597
- mdb_txn_end(txn, end_mode);
4664
+ //<lmdb-js>
4665
+ if (env->me_flags & MDB_TRACK_METRICS) {
4666
+ ((MDB_metrics*) env->me_userctx)->time_during_txns += get_time64() - ((MDB_metrics*) env->me_userctx)->clock_txn;
4667
+ ((MDB_metrics*) env->me_userctx)->txns++;
4668
+ }
4669
+ if ((txn->mt_flags & MDB_NOSYNC) && (env->me_flags & MDB_OVERLAPPINGSYNC)) {
4670
+ MDB_txn sync_txn;
4671
+ MDB_db dbs[2];
4672
+ sync_txn.mt_env = env;
4673
+ sync_txn.mt_flags = 2;
4674
+ sync_txn.mt_dbs = dbs;
4675
+ sync_txn.mt_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
4676
+ sync_txn.mt_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
4677
+ sync_txn.mt_dbs[FREE_DBI].md_flags &= ~MDB_OVERLAPPINGSYNC; // clear this to indicate it is flushed txn
4678
+ sync_txn.mt_txnid = txn->mt_txnid;
4679
+ sync_txn.mt_next_pgno = txn->mt_next_pgno;
4680
+ if (dirty_pages * (txn->mt_txnid - env->me_synced_txn_id) > 100) {
4681
+ // for bigger txns we wait for the flush before allowing next txn
4682
+ LOCK_MUTEX(rc, env, env->me_sync_mutex);
4683
+ mdb_txn_end(txn, end_mode);
4684
+ } else {
4685
+ mdb_txn_end(txn, end_mode);
4686
+ LOCK_MUTEX(rc, env, env->me_sync_mutex);
4687
+ }
4688
+ if (rc)
4689
+ return rc;
4690
+ rc = mdb_env_sync0(env, 0, sync_txn.mt_next_pgno);
4691
+ if (rc) {
4692
+ UNLOCK_MUTEX(env->me_sync_mutex);
4693
+ return rc;
4694
+ }
4695
+ rc = mdb_env_write_meta(&sync_txn);
4696
+ if (rc == 0)
4697
+ env->me_synced_txn_id = sync_txn.mt_txnid;
4698
+ UNLOCK_MUTEX(env->me_sync_mutex);
4699
+ } else
4700
+ mdb_txn_end(txn, end_mode);
4598
4701
  return MDB_SUCCESS;
4599
4702
 
4600
4703
  fail:
@@ -4725,7 +4828,7 @@ mdb_env_init_meta0(MDB_env *env, MDB_meta *meta)
4725
4828
  meta->mm_mapsize = env->me_mapsize;
4726
4829
  meta->mm_psize = env->me_psize;
4727
4830
  meta->mm_last_pg = NUM_METAS-1;
4728
- meta->mm_flags = env->me_flags & 0xffff;
4831
+ meta->mm_flags = env->me_flags & 0xffff & ~MDB_OVERLAPPINGSYNC;
4729
4832
  meta->mm_flags |= MDB_INTEGERKEY; /* this is mm_dbs[FREE_DBI].md_flags */
4730
4833
  meta->mm_dbs[FREE_DBI].md_root = P_INVALID;
4731
4834
  meta->mm_dbs[MAIN_DBI].md_root = P_INVALID;
@@ -5871,10 +5974,10 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
5871
5974
  #endif
5872
5975
  int rc;
5873
5976
  MDB_OFF_T size, rsize;
5874
- if (env->me_check_fd) {
5977
+ if (env->me_callback) {
5875
5978
  rc = mdb_fopen(env, fname, MDB_O_RDWR, mode, &env->me_lfd);
5876
5979
  if (!rc) {
5877
- rc = ((MDB_check_fd*) env->me_check_fd)(env->me_lfd, env);
5980
+ rc = ((MDB_check_fd*) env->me_callback)(env->me_lfd, env);
5878
5981
  if (rc)
5879
5982
  return rc;
5880
5983
  }
@@ -6011,11 +6114,11 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
6011
6114
  env->me_sync_mutex = sem_open(MUTEXNAME(env, 's'), O_CREAT|O_EXCL, mode, 1);
6012
6115
  if (env->me_sync_mutex == SEM_FAILED) goto fail_errno;
6013
6116
  #elif defined(MDB_USE_SYSV_SEM)
6014
- unsigned short vals[2] = {1, 1};
6117
+ unsigned short vals[3] = {1, 1, 1};
6015
6118
  key_t key = ftok(fname->mn_val, 'M'); /* fname is lockfile path now */
6016
6119
  if (key == -1)
6017
6120
  goto fail_errno;
6018
- semid = semget(key, 2, (mode & 0777) | IPC_CREAT);
6121
+ semid = semget(key, 3, (mode & 0777) | IPC_CREAT);
6019
6122
  if (semid < 0)
6020
6123
  goto fail_errno;
6021
6124
  semu.array = vals;
@@ -6040,7 +6143,7 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
6040
6143
  goto fail;
6041
6144
  rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
6042
6145
  #ifdef MDB_ROBUST_SUPPORTED
6043
- if (!rc) rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST);
6146
+ if (!rc) pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); // don't record rc, we don't care if it fails
6044
6147
  #endif
6045
6148
  if (!rc) rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr);
6046
6149
  if (!rc) rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr);
@@ -6187,7 +6290,7 @@ mdb_env_envflags(MDB_env *env)
6187
6290
  */
6188
6291
  #define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
6189
6292
  #define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \
6190
- MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVSNAPSHOT|MDB_REMAP_CHUNKS|MDB_OVERLAPPINGSYNC|MDB_SAFE_RESTORE)
6293
+ MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVSNAPSHOT|MDB_REMAP_CHUNKS|MDB_OVERLAPPINGSYNC|MDB_SAFE_RESTORE|MDB_TRACK_METRICS)
6191
6294
  #define EXPOSED (CHANGEABLE|CHANGELESS | MDB_ENCRYPT)
6192
6295
 
6193
6296
  #if VALID_FLAGS & PERSISTENT_FLAGS & EXPOSED
@@ -6412,6 +6515,8 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
6412
6515
  txn->mt_dbxs = env->me_dbxs;
6413
6516
  txn->mt_flags = MDB_TXN_FINISHED;
6414
6517
  env->me_txn0 = txn;
6518
+ MDB_meta* latest = mdb_env_pick_meta(env);
6519
+ txn->mt_env->me_synced_txn_id = latest->mm_txnid;
6415
6520
  } else {
6416
6521
  rc = ENOMEM;
6417
6522
  }
@@ -7769,6 +7874,12 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
7769
7874
  int
7770
7875
  mdb_get(MDB_txn *txn, MDB_dbi dbi,
7771
7876
  MDB_val *key, MDB_val *data)
7877
+ {
7878
+ return mdb_get_with_txn(txn, dbi, key, data, NULL);
7879
+ }
7880
+ int
7881
+ mdb_get_with_txn(MDB_txn *txn, MDB_dbi dbi,
7882
+ MDB_val *key, MDB_val *data, mdb_size_t *txn_id)
7772
7883
  {
7773
7884
  MDB_cursor mc;
7774
7885
  MDB_xcursor mx;
@@ -7805,6 +7916,9 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi,
7805
7916
  /* unref all the pages when MDB_REMAP_CHUNKS - caller must copy the data
7806
7917
  * before doing anything else
7807
7918
  */
7919
+ if (txn_id && !rc) {
7920
+ *txn_id = mc.mc_pg[mc.mc_top]->mp_txnid;
7921
+ }
7808
7922
  MDB_CURSOR_UNREF(&mc, 1);
7809
7923
  return rc;
7810
7924
  }
@@ -8893,6 +9007,7 @@ current:
8893
9007
  * is smaller than the overflow threshold.
8894
9008
  */
8895
9009
  if (!IS_WRITABLE(mc->mc_txn, omp)) {
9010
+
8896
9011
  if (!IS_DIRTY_NW(mc->mc_txn, omp)) {
8897
9012
  rc = mdb_page_unspill(mc->mc_txn, omp, &omp);
8898
9013
  if (rc)
@@ -10590,7 +10705,10 @@ mdb_del(MDB_txn *txn, MDB_dbi dbi,
10590
10705
  /* must ignore any data */
10591
10706
  data = NULL;
10592
10707
  }
10593
-
10708
+ MDB_env* env = txn->mt_env;
10709
+ if (env->me_flags & MDB_TRACK_METRICS) {
10710
+ ((MDB_metrics*) env->me_userctx)->deletes++;
10711
+ }
10594
10712
  return mdb_del0(txn, dbi, key, data, 0);
10595
10713
  }
10596
10714
 
@@ -11082,6 +11200,10 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
11082
11200
 
11083
11201
  if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
11084
11202
  return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
11203
+ MDB_env* env = txn->mt_env;
11204
+ if (env->me_flags & MDB_TRACK_METRICS) {
11205
+ ((MDB_metrics*) env->me_userctx)->puts++;
11206
+ }
11085
11207
 
11086
11208
  mdb_cursor_init(&mc, txn, dbi, &mx);
11087
11209
  mc.mc_next = txn->mt_cursors[dbi];
@@ -11477,7 +11599,7 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
11477
11599
  mm->mm_dbs[MAIN_DBI].md_flags = txn->mt_dbs[MAIN_DBI].md_flags;
11478
11600
  }
11479
11601
  if (root != P_INVALID || mm->mm_dbs[MAIN_DBI].md_flags) {
11480
- mm->mm_txnid = 1; /* use metapage 1 */
11602
+ mm->mm_txnid = txn->mt_txnid | 1; /* update to current txn id and make it odd*/
11481
11603
  }
11482
11604
 
11483
11605
  my.mc_wlen[0] = env->me_psize * NUM_METAS;
@@ -11676,7 +11798,8 @@ mdb_env_set_userctx(MDB_env *env, void *ctx)
11676
11798
  {
11677
11799
  if (!env)
11678
11800
  return EINVAL;
11679
- env->me_userctx = ctx;
11801
+ if (ctx)
11802
+ env->me_userctx = ctx;
11680
11803
  return MDB_SUCCESS;
11681
11804
  }
11682
11805
 
@@ -11698,11 +11821,11 @@ mdb_env_set_assert(MDB_env *env, MDB_assert_func *func)
11698
11821
  }
11699
11822
 
11700
11823
  int ESECT
11701
- mdb_env_set_check_fd(MDB_env *env, MDB_check_fd *func)
11824
+ mdb_env_set_callback(MDB_env *env, MDB_check_fd *func)
11702
11825
  {
11703
11826
  if (!env)
11704
11827
  return EINVAL;
11705
- env->me_check_fd = func;
11828
+ env->me_callback = func;
11706
11829
  return MDB_SUCCESS;
11707
11830
  }
11708
11831
 
@@ -11817,7 +11940,7 @@ mdb_env_info(MDB_env *env, MDB_envinfo *arg)
11817
11940
  return EINVAL;
11818
11941
 
11819
11942
  meta = mdb_env_pick_meta(env);
11820
- arg->me_mapaddr = meta->mm_address;
11943
+ arg->me_mapaddr = env->me_map;
11821
11944
  arg->me_last_pgno = meta->mm_last_pg;
11822
11945
  arg->me_last_txnid = meta->mm_txnid;
11823
11946
 
@@ -12394,7 +12517,10 @@ mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
12394
12517
  #endif
12395
12518
  DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc)));
12396
12519
  }
12397
-
12520
+ #ifdef __APPLE__
12521
+ if (rc == EINVAL)
12522
+ rc = MDB_LOCK_FAILURE;
12523
+ #endif
12398
12524
  return rc;
12399
12525
  }
12400
12526
  #endif /* MDB_ROBUST_SUPPORTED */
@@ -1,74 +1,74 @@
1
- .TH MDB_COPY 1 "2017/07/31" "LMDB 0.9.90"
2
- .\" Copyright 2012-2021 Howard Chu, Symas Corp. All Rights Reserved.
3
- .\" Copying restrictions apply. See COPYRIGHT/LICENSE.
4
- .SH NAME
5
- mdb_copy \- LMDB environment copy tool
6
- .SH SYNOPSIS
7
- .B mdb_copy
8
- [\c
9
- .BR \-V ]
10
- [\c
11
- .BR \-c ]
12
- [\c
13
- .BR \-n ]
14
- [\c
15
- .BR \-v ]
16
- [\c
17
- .BI \-m \ module
18
- [\c
19
- .BI \-w \ password\fR]]
20
- .B srcpath
21
- [\c
22
- .BR dstpath ]
23
- .SH DESCRIPTION
24
- The
25
- .B mdb_copy
26
- utility copies an LMDB environment. The environment can
27
- be copied regardless of whether it is currently in use.
28
- No lockfile is created, since it gets recreated at need.
29
-
30
- If
31
- .I dstpath
32
- is specified it must be the path of an empty directory
33
- for storing the backup. Otherwise, the backup will be
34
- written to stdout.
35
-
36
- .SH OPTIONS
37
- .TP
38
- .BR \-V
39
- Write the library version number to the standard output, and exit.
40
- .TP
41
- .BR \-c
42
- Compact while copying. Only current data pages will be copied; freed
43
- or unused pages will be omitted from the copy. This option will
44
- slow down the backup process as it is more CPU-intensive.
45
- Currently it fails if the environment has suffered a page leak.
46
- .TP
47
- .BR \-n
48
- Open LDMB environment(s) which do not use subdirectories.
49
- .TP
50
- .BR \-v
51
- Use the previous environment state instead of the latest state.
52
- This may be useful if the latest state has been corrupted.
53
- .TP
54
- .BI \-m \ module
55
- Load the specified dynamic module to utilize cryptographic functions.
56
- This is required to operate on environments that have been configured
57
- with page-level checksums or encryption.
58
- .TP
59
- .BI \-w \ password
60
- Specify the password for an encrypted environment. This is only
61
- used if a cryptography module has been loaded.
62
-
63
- .SH DIAGNOSTICS
64
- Exit status is zero if no errors occur.
65
- Errors result in a non-zero exit status and
66
- a diagnostic message being written to standard error.
67
- .SH CAVEATS
68
- This utility can trigger significant file size growth if run
69
- in parallel with write transactions, because pages which they
70
- free during copying cannot be reused until the copy is done.
71
- .SH "SEE ALSO"
72
- .BR mdb_stat (1)
73
- .SH AUTHOR
74
- Howard Chu of Symas Corporation <http://www.symas.com>
1
+ .TH MDB_COPY 1 "2017/07/31" "LMDB 0.9.90"
2
+ .\" Copyright 2012-2021 Howard Chu, Symas Corp. All Rights Reserved.
3
+ .\" Copying restrictions apply. See COPYRIGHT/LICENSE.
4
+ .SH NAME
5
+ mdb_copy \- LMDB environment copy tool
6
+ .SH SYNOPSIS
7
+ .B mdb_copy
8
+ [\c
9
+ .BR \-V ]
10
+ [\c
11
+ .BR \-c ]
12
+ [\c
13
+ .BR \-n ]
14
+ [\c
15
+ .BR \-v ]
16
+ [\c
17
+ .BI \-m \ module
18
+ [\c
19
+ .BI \-w \ password\fR]]
20
+ .B srcpath
21
+ [\c
22
+ .BR dstpath ]
23
+ .SH DESCRIPTION
24
+ The
25
+ .B mdb_copy
26
+ utility copies an LMDB environment. The environment can
27
+ be copied regardless of whether it is currently in use.
28
+ No lockfile is created, since it gets recreated at need.
29
+
30
+ If
31
+ .I dstpath
32
+ is specified it must be the path of an empty directory
33
+ for storing the backup. Otherwise, the backup will be
34
+ written to stdout.
35
+
36
+ .SH OPTIONS
37
+ .TP
38
+ .BR \-V
39
+ Write the library version number to the standard output, and exit.
40
+ .TP
41
+ .BR \-c
42
+ Compact while copying. Only current data pages will be copied; freed
43
+ or unused pages will be omitted from the copy. This option will
44
+ slow down the backup process as it is more CPU-intensive.
45
+ Currently it fails if the environment has suffered a page leak.
46
+ .TP
47
+ .BR \-n
48
+ Open LDMB environment(s) which do not use subdirectories.
49
+ .TP
50
+ .BR \-v
51
+ Use the previous environment state instead of the latest state.
52
+ This may be useful if the latest state has been corrupted.
53
+ .TP
54
+ .BI \-m \ module
55
+ Load the specified dynamic module to utilize cryptographic functions.
56
+ This is required to operate on environments that have been configured
57
+ with page-level checksums or encryption.
58
+ .TP
59
+ .BI \-w \ password
60
+ Specify the password for an encrypted environment. This is only
61
+ used if a cryptography module has been loaded.
62
+
63
+ .SH DIAGNOSTICS
64
+ Exit status is zero if no errors occur.
65
+ Errors result in a non-zero exit status and
66
+ a diagnostic message being written to standard error.
67
+ .SH CAVEATS
68
+ This utility can trigger significant file size growth if run
69
+ in parallel with write transactions, because pages which they
70
+ free during copying cannot be reused until the copy is done.
71
+ .SH "SEE ALSO"
72
+ .BR mdb_stat (1)
73
+ .SH AUTHOR
74
+ Howard Chu of Symas Corporation <http://www.symas.com>