mdbxmou 0.1.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/.github/workflows/ci.yml +32 -0
  2. package/.github/workflows/publish.yml +27 -0
  3. package/.gitmodules +3 -0
  4. package/CMakeLists.txt +53 -0
  5. package/LICENSE +201 -0
  6. package/README.md +639 -0
  7. package/build.js +11 -0
  8. package/deps/libmdbx/.clang-format +3 -0
  9. package/deps/libmdbx/.cmake-format.yaml +3 -0
  10. package/deps/libmdbx/.le.ini +40 -0
  11. package/deps/libmdbx/CMakeLists.txt +1269 -0
  12. package/deps/libmdbx/COPYRIGHT +159 -0
  13. package/deps/libmdbx/ChangeLog.md +2786 -0
  14. package/deps/libmdbx/GNUmakefile +950 -0
  15. package/deps/libmdbx/LICENSE +177 -0
  16. package/deps/libmdbx/Makefile +16 -0
  17. package/deps/libmdbx/NOTICE +39 -0
  18. package/deps/libmdbx/README.md +863 -0
  19. package/deps/libmdbx/TODO.md +43 -0
  20. package/deps/libmdbx/cmake/compiler.cmake +1221 -0
  21. package/deps/libmdbx/cmake/profile.cmake +58 -0
  22. package/deps/libmdbx/cmake/utils.cmake +524 -0
  23. package/deps/libmdbx/conanfile.py +323 -0
  24. package/deps/libmdbx/docs/Doxyfile.in +2734 -0
  25. package/deps/libmdbx/docs/_preface.md +47 -0
  26. package/deps/libmdbx/docs/_restrictions.md +248 -0
  27. package/deps/libmdbx/docs/_starting.md +245 -0
  28. package/deps/libmdbx/docs/_toc.md +34 -0
  29. package/deps/libmdbx/docs/header.html +96 -0
  30. package/deps/libmdbx/example/CMakeLists.txt +6 -0
  31. package/deps/libmdbx/example/README.md +1 -0
  32. package/deps/libmdbx/example/example-mdbx.c +154 -0
  33. package/deps/libmdbx/example/sample-bdb.txt +77 -0
  34. package/deps/libmdbx/mdbx.h +6655 -0
  35. package/deps/libmdbx/mdbx.h++ +6428 -0
  36. package/deps/libmdbx/packages/buildroot/0001-package-libmdbx-new-package-library-database.patch +173 -0
  37. package/deps/libmdbx/src/alloy.c +54 -0
  38. package/deps/libmdbx/src/api-cold.c +543 -0
  39. package/deps/libmdbx/src/api-copy.c +912 -0
  40. package/deps/libmdbx/src/api-cursor.c +754 -0
  41. package/deps/libmdbx/src/api-dbi.c +315 -0
  42. package/deps/libmdbx/src/api-env.c +1434 -0
  43. package/deps/libmdbx/src/api-extra.c +165 -0
  44. package/deps/libmdbx/src/api-key-transform.c +197 -0
  45. package/deps/libmdbx/src/api-misc.c +286 -0
  46. package/deps/libmdbx/src/api-opts.c +575 -0
  47. package/deps/libmdbx/src/api-range-estimate.c +365 -0
  48. package/deps/libmdbx/src/api-txn-data.c +454 -0
  49. package/deps/libmdbx/src/api-txn.c +921 -0
  50. package/deps/libmdbx/src/atomics-ops.h +364 -0
  51. package/deps/libmdbx/src/atomics-types.h +97 -0
  52. package/deps/libmdbx/src/audit.c +109 -0
  53. package/deps/libmdbx/src/bits.md +34 -0
  54. package/deps/libmdbx/src/chk.c +1796 -0
  55. package/deps/libmdbx/src/cogs.c +309 -0
  56. package/deps/libmdbx/src/cogs.h +506 -0
  57. package/deps/libmdbx/src/coherency.c +170 -0
  58. package/deps/libmdbx/src/config.h.in +88 -0
  59. package/deps/libmdbx/src/cursor.c +2396 -0
  60. package/deps/libmdbx/src/cursor.h +391 -0
  61. package/deps/libmdbx/src/dbi.c +717 -0
  62. package/deps/libmdbx/src/dbi.h +142 -0
  63. package/deps/libmdbx/src/debug_begin.h +36 -0
  64. package/deps/libmdbx/src/debug_end.h +15 -0
  65. package/deps/libmdbx/src/dpl.c +486 -0
  66. package/deps/libmdbx/src/dpl.h +134 -0
  67. package/deps/libmdbx/src/dxb.c +1335 -0
  68. package/deps/libmdbx/src/env.c +607 -0
  69. package/deps/libmdbx/src/essentials.h +125 -0
  70. package/deps/libmdbx/src/gc-get.c +1345 -0
  71. package/deps/libmdbx/src/gc-put.c +970 -0
  72. package/deps/libmdbx/src/gc.h +40 -0
  73. package/deps/libmdbx/src/global.c +474 -0
  74. package/deps/libmdbx/src/internals.h +585 -0
  75. package/deps/libmdbx/src/layout-dxb.h +288 -0
  76. package/deps/libmdbx/src/layout-lck.h +289 -0
  77. package/deps/libmdbx/src/lck-posix.c +859 -0
  78. package/deps/libmdbx/src/lck-windows.c +607 -0
  79. package/deps/libmdbx/src/lck.c +174 -0
  80. package/deps/libmdbx/src/lck.h +110 -0
  81. package/deps/libmdbx/src/logging_and_debug.c +250 -0
  82. package/deps/libmdbx/src/logging_and_debug.h +159 -0
  83. package/deps/libmdbx/src/man1/mdbx_chk.1 +106 -0
  84. package/deps/libmdbx/src/man1/mdbx_copy.1 +95 -0
  85. package/deps/libmdbx/src/man1/mdbx_drop.1 +48 -0
  86. package/deps/libmdbx/src/man1/mdbx_dump.1 +101 -0
  87. package/deps/libmdbx/src/man1/mdbx_load.1 +105 -0
  88. package/deps/libmdbx/src/man1/mdbx_stat.1 +86 -0
  89. package/deps/libmdbx/src/mdbx.c++ +1837 -0
  90. package/deps/libmdbx/src/meta.c +656 -0
  91. package/deps/libmdbx/src/meta.h +168 -0
  92. package/deps/libmdbx/src/mvcc-readers.c +414 -0
  93. package/deps/libmdbx/src/node.c +365 -0
  94. package/deps/libmdbx/src/node.h +102 -0
  95. package/deps/libmdbx/src/ntdll.def +1246 -0
  96. package/deps/libmdbx/src/options.h +534 -0
  97. package/deps/libmdbx/src/osal.c +3485 -0
  98. package/deps/libmdbx/src/osal.h +587 -0
  99. package/deps/libmdbx/src/page-get.c +483 -0
  100. package/deps/libmdbx/src/page-iov.c +185 -0
  101. package/deps/libmdbx/src/page-iov.h +34 -0
  102. package/deps/libmdbx/src/page-ops.c +744 -0
  103. package/deps/libmdbx/src/page-ops.h +142 -0
  104. package/deps/libmdbx/src/pnl.c +236 -0
  105. package/deps/libmdbx/src/pnl.h +146 -0
  106. package/deps/libmdbx/src/preface.h +990 -0
  107. package/deps/libmdbx/src/proto.h +105 -0
  108. package/deps/libmdbx/src/refund.c +212 -0
  109. package/deps/libmdbx/src/sort.h +484 -0
  110. package/deps/libmdbx/src/spill.c +431 -0
  111. package/deps/libmdbx/src/spill.h +74 -0
  112. package/deps/libmdbx/src/table.c +107 -0
  113. package/deps/libmdbx/src/tls.c +551 -0
  114. package/deps/libmdbx/src/tls.h +43 -0
  115. package/deps/libmdbx/src/tools/chk.c +673 -0
  116. package/deps/libmdbx/src/tools/copy.c +166 -0
  117. package/deps/libmdbx/src/tools/drop.c +199 -0
  118. package/deps/libmdbx/src/tools/dump.c +515 -0
  119. package/deps/libmdbx/src/tools/load.c +831 -0
  120. package/deps/libmdbx/src/tools/stat.c +516 -0
  121. package/deps/libmdbx/src/tools/wingetopt.c +87 -0
  122. package/deps/libmdbx/src/tools/wingetopt.h +30 -0
  123. package/deps/libmdbx/src/tree-ops.c +1554 -0
  124. package/deps/libmdbx/src/tree-search.c +140 -0
  125. package/deps/libmdbx/src/txl.c +99 -0
  126. package/deps/libmdbx/src/txl.h +26 -0
  127. package/deps/libmdbx/src/txn.c +1083 -0
  128. package/deps/libmdbx/src/unaligned.h +205 -0
  129. package/deps/libmdbx/src/utils.c +32 -0
  130. package/deps/libmdbx/src/utils.h +76 -0
  131. package/deps/libmdbx/src/version.c.in +44 -0
  132. package/deps/libmdbx/src/walk.c +290 -0
  133. package/deps/libmdbx/src/walk.h +20 -0
  134. package/deps/libmdbx/src/windows-import.c +152 -0
  135. package/deps/libmdbx/src/windows-import.h +128 -0
  136. package/deps/libmdbx/test/CMakeLists.txt +317 -0
  137. package/deps/libmdbx/test/append.c++ +237 -0
  138. package/deps/libmdbx/test/base.h++ +92 -0
  139. package/deps/libmdbx/test/battery-tmux.sh +64 -0
  140. package/deps/libmdbx/test/cases.c++ +118 -0
  141. package/deps/libmdbx/test/chrono.c++ +134 -0
  142. package/deps/libmdbx/test/chrono.h++ +85 -0
  143. package/deps/libmdbx/test/config.c++ +643 -0
  144. package/deps/libmdbx/test/config.h++ +334 -0
  145. package/deps/libmdbx/test/copy.c++ +62 -0
  146. package/deps/libmdbx/test/dead.c++ +39 -0
  147. package/deps/libmdbx/test/dump-load.sh +40 -0
  148. package/deps/libmdbx/test/extra/crunched_delete.c++ +409 -0
  149. package/deps/libmdbx/test/extra/cursor_closing.c++ +410 -0
  150. package/deps/libmdbx/test/extra/dbi.c++ +229 -0
  151. package/deps/libmdbx/test/extra/doubtless_positioning.c++ +253 -0
  152. package/deps/libmdbx/test/extra/dupfix_addodd.c +94 -0
  153. package/deps/libmdbx/test/extra/dupfix_multiple.c++ +311 -0
  154. package/deps/libmdbx/test/extra/early_close_dbi.c++ +137 -0
  155. package/deps/libmdbx/test/extra/hex_base64_base58.c++ +118 -0
  156. package/deps/libmdbx/test/extra/maindb_ordinal.c++ +61 -0
  157. package/deps/libmdbx/test/extra/open.c++ +96 -0
  158. package/deps/libmdbx/test/extra/pcrf/README.md +2 -0
  159. package/deps/libmdbx/test/extra/pcrf/pcrf_test.c +380 -0
  160. package/deps/libmdbx/test/extra/probe.c++ +10 -0
  161. package/deps/libmdbx/test/extra/txn.c++ +407 -0
  162. package/deps/libmdbx/test/extra/upsert_alldups.c +193 -0
  163. package/deps/libmdbx/test/fork.c++ +263 -0
  164. package/deps/libmdbx/test/hill.c++ +447 -0
  165. package/deps/libmdbx/test/jitter.c++ +197 -0
  166. package/deps/libmdbx/test/keygen.c++ +393 -0
  167. package/deps/libmdbx/test/keygen.h++ +130 -0
  168. package/deps/libmdbx/test/log.c++ +358 -0
  169. package/deps/libmdbx/test/log.h++ +91 -0
  170. package/deps/libmdbx/test/main.c++ +706 -0
  171. package/deps/libmdbx/test/nested.c++ +318 -0
  172. package/deps/libmdbx/test/osal-unix.c++ +647 -0
  173. package/deps/libmdbx/test/osal-windows.c++ +440 -0
  174. package/deps/libmdbx/test/osal.h++ +41 -0
  175. package/deps/libmdbx/test/stochastic.sh +690 -0
  176. package/deps/libmdbx/test/stub/LICENSE +24 -0
  177. package/deps/libmdbx/test/stub/README.md +8 -0
  178. package/deps/libmdbx/test/stub/pthread_barrier.c +104 -0
  179. package/deps/libmdbx/test/stub/pthread_barrier.h +77 -0
  180. package/deps/libmdbx/test/test.c++ +1551 -0
  181. package/deps/libmdbx/test/test.h++ +298 -0
  182. package/deps/libmdbx/test/tmux.conf +3 -0
  183. package/deps/libmdbx/test/try.c++ +30 -0
  184. package/deps/libmdbx/test/ttl.c++ +240 -0
  185. package/deps/libmdbx/test/utils.c++ +203 -0
  186. package/deps/libmdbx/test/utils.h++ +326 -0
  187. package/deps/libmdbx/test/valgrind_suppress.txt +536 -0
  188. package/lib/mdbx_evn_async.js +211 -0
  189. package/lib/mdbx_worker.js +195 -0
  190. package/lib/nativemou.js +6 -0
  191. package/package.json +38 -0
  192. package/src/async/envmou_close.cpp +34 -0
  193. package/src/async/envmou_close.hpp +32 -0
  194. package/src/async/envmou_copy_to.cpp +29 -0
  195. package/src/async/envmou_copy_to.hpp +38 -0
  196. package/src/async/envmou_keys.cpp +201 -0
  197. package/src/async/envmou_keys.hpp +50 -0
  198. package/src/async/envmou_open.cpp +38 -0
  199. package/src/async/envmou_open.hpp +33 -0
  200. package/src/async/envmou_query.cpp +167 -0
  201. package/src/async/envmou_query.hpp +53 -0
  202. package/src/dbimou.cpp +522 -0
  203. package/src/dbimou.hpp +82 -0
  204. package/src/env_arg0.hpp +24 -0
  205. package/src/envmou.cpp +445 -0
  206. package/src/envmou.hpp +116 -0
  207. package/src/modulemou.cpp +113 -0
  208. package/src/querymou.cpp +177 -0
  209. package/src/querymou.hpp +93 -0
  210. package/src/txnmou.cpp +254 -0
  211. package/src/txnmou.hpp +122 -0
  212. package/src/typemou.hpp +239 -0
  213. package/src/valuemou.hpp +194 -0
  214. package/test/async.js +67 -0
  215. package/test/e3.js +38 -0
  216. package/test/e4.js +89 -0
  217. package/test/e5.js +162 -0
  218. package/test/test-batch-ops.js +243 -0
  219. package/test/test-cursor-mode.js +84 -0
  220. package/test/test-multi-mode.js +87 -0
@@ -0,0 +1,3485 @@
1
+ /// \copyright SPDX-License-Identifier: Apache-2.0
2
+ /// \author Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru> \date 2015-2025
3
+ ///
4
+ /// https://en.wikipedia.org/wiki/Operating_system_abstraction_layer
5
+
6
+ #include "internals.h"
7
+
8
+ #if defined(_WIN32) || defined(_WIN64)
9
+
10
+ #include <psapi.h>
11
+ #include <winioctl.h>
12
+
13
+ #if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG)
14
+ #include <crtdbg.h>
15
+ #endif
16
+
17
+ static int waitstatus2errcode(DWORD result) {
18
+ switch (result) {
19
+ case WAIT_OBJECT_0:
20
+ return MDBX_SUCCESS;
21
+ case WAIT_FAILED:
22
+ return (int)GetLastError();
23
+ case WAIT_ABANDONED:
24
+ return ERROR_ABANDONED_WAIT_0;
25
+ case WAIT_IO_COMPLETION:
26
+ return ERROR_USER_APC;
27
+ case WAIT_TIMEOUT:
28
+ return ERROR_TIMEOUT;
29
+ default:
30
+ return ERROR_UNHANDLED_ERROR;
31
+ }
32
+ }
33
+
34
+ /* Map a result from an NTAPI call to WIN32 error code. */
35
+ static int ntstatus2errcode(NTSTATUS status) {
36
+ DWORD dummy;
37
+ OVERLAPPED ov;
38
+ memset(&ov, 0, sizeof(ov));
39
+ ov.Internal = status;
40
+ /* Zap: '_Param_(1)' could be '0' */
41
+ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6387);
42
+ return GetOverlappedResult(nullptr, &ov, &dummy, FALSE) ? MDBX_SUCCESS : (int)GetLastError();
43
+ }
44
+
45
+ /* We use native NT APIs to setup the memory map, so that we can
46
+ * let the DB file grow incrementally instead of always preallocating
47
+ * the full size. These APIs are defined in <wdm.h> and <ntifs.h>
48
+ * but those headers are meant for driver-level development and
49
+ * conflict with the regular user-level headers, so we explicitly
50
+ * declare them here. Using these APIs also means we must link to
51
+ * ntdll.dll, which is not linked by default in user code. */
52
+
53
+ extern NTSTATUS NTAPI NtCreateSection(OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess,
54
+ IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes,
55
+ IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection,
56
+ IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle);
57
+
58
+ typedef struct _SECTION_BASIC_INFORMATION {
59
+ ULONG Unknown;
60
+ ULONG SectionAttributes;
61
+ LARGE_INTEGER SectionSize;
62
+ } SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION;
63
+
64
+ extern NTSTATUS NTAPI NtMapViewOfSection(IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress,
65
+ IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize,
66
+ IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize,
67
+ IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType,
68
+ IN ULONG Win32Protect);
69
+
70
+ extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, IN OPTIONAL PVOID BaseAddress);
71
+
72
+ /* Zap: Inconsistent annotation for 'NtClose'... */
73
+ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(28251)
74
+ extern NTSTATUS NTAPI NtClose(HANDLE Handle);
75
+
76
+ extern NTSTATUS NTAPI NtAllocateVirtualMemory(IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits,
77
+ IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect);
78
+
79
+ extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, IN PVOID *BaseAddress, IN OUT PSIZE_T RegionSize,
80
+ IN ULONG FreeType);
81
+
82
+ #ifndef WOF_CURRENT_VERSION
83
+ typedef struct _WOF_EXTERNAL_INFO {
84
+ DWORD Version;
85
+ DWORD Provider;
86
+ } WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO;
87
+ #endif /* WOF_CURRENT_VERSION */
88
+
89
+ #ifndef WIM_PROVIDER_CURRENT_VERSION
90
+ #define WIM_PROVIDER_HASH_SIZE 20
91
+
92
+ typedef struct _WIM_PROVIDER_EXTERNAL_INFO {
93
+ DWORD Version;
94
+ DWORD Flags;
95
+ LARGE_INTEGER DataSourceId;
96
+ BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE];
97
+ } WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO;
98
+ #endif /* WIM_PROVIDER_CURRENT_VERSION */
99
+
100
+ #ifndef FILE_PROVIDER_CURRENT_VERSION
101
+ typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 {
102
+ ULONG Version;
103
+ ULONG Algorithm;
104
+ ULONG Flags;
105
+ } FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1;
106
+ #endif /* FILE_PROVIDER_CURRENT_VERSION */
107
+
108
+ #ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED
109
+ #define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL)
110
+ #endif
111
+ #ifndef STATUS_INVALID_DEVICE_REQUEST
112
+ #define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L)
113
+ #endif
114
+ #ifndef STATUS_NOT_SUPPORTED
115
+ #define STATUS_NOT_SUPPORTED ((NTSTATUS)0xC00000BBL)
116
+ #endif
117
+
118
+ #ifndef FILE_DEVICE_FILE_SYSTEM
119
+ #define FILE_DEVICE_FILE_SYSTEM 0x00000009
120
+ #endif
121
+
122
+ #ifndef FSCTL_GET_EXTERNAL_BACKING
123
+ #define FSCTL_GET_EXTERNAL_BACKING CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS)
124
+ #endif
125
+
126
+ #ifndef ERROR_NOT_CAPABLE
127
+ #define ERROR_NOT_CAPABLE 775L
128
+ #endif
129
+
130
+ #endif /* _WIN32 || _WIN64 */
131
+
132
+ /*----------------------------------------------------------------------------*/
133
+
134
+ #if defined(__ANDROID_API__)
135
+ __extern_C void __assert2(const char *file, int line, const char *function, const char *msg) __noreturn;
136
+ #define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion)
137
+
138
+ #elif defined(__UCLIBC__)
139
+ MDBX_NORETURN __extern_C void __assert(const char *, const char *, unsigned, const char *)
140
+ #ifdef __THROW
141
+ __THROW
142
+ #else
143
+ __nothrow
144
+ #endif /* __THROW */
145
+ ;
146
+ #define __assert_fail(assertion, file, line, function) __assert(assertion, file, line, function)
147
+
148
+ #elif _POSIX_C_SOURCE > 200212 && \
149
+ /* workaround for avoid musl libc wrong prototype */ (defined(__GLIBC__) || defined(__GNU_LIBRARY__))
150
+ /* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */
151
+ MDBX_NORETURN __extern_C void __assert_fail(const char *assertion, const char *file, unsigned line,
152
+ const char *function)
153
+ #ifdef __THROW
154
+ __THROW
155
+ #else
156
+ __nothrow
157
+ #endif /* __THROW */
158
+ ;
159
+
160
+ #elif defined(__APPLE__) || defined(__MACH__)
161
+ __extern_C void __assert_rtn(const char *function, const char *file, int line, const char *assertion) /* __nothrow */
162
+ #ifdef __dead2
163
+ __dead2
164
+ #else
165
+ MDBX_NORETURN
166
+ #endif /* __dead2 */
167
+ #ifdef __disable_tail_calls
168
+ __disable_tail_calls
169
+ #endif /* __disable_tail_calls */
170
+ ;
171
+
172
+ #define __assert_fail(assertion, file, line, function) __assert_rtn(function, file, line, assertion)
173
+ #elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
174
+ MDBX_NORETURN __extern_C void __assert_c99(const char *assection, const char *file, int line, const char *function);
175
+ #define __assert_fail(assertion, file, line, function) __assert_c99(assertion, file, line, function)
176
+ #elif defined(__OpenBSD__)
177
+ __extern_C __dead void __assert2(const char *file, int line, const char *function,
178
+ const char *assertion) /* __nothrow */;
179
+ #define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion)
180
+ #elif defined(__NetBSD__)
181
+ __extern_C __dead void __assert13(const char *file, int line, const char *function,
182
+ const char *assertion) /* __nothrow */;
183
+ #define __assert_fail(assertion, file, line, function) __assert13(file, line, function, assertion)
184
+ #elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__)
185
+ __extern_C void __assert(const char *function, const char *file, int line, const char *assertion) /* __nothrow */
186
+ #ifdef __dead2
187
+ __dead2
188
+ #else
189
+ MDBX_NORETURN
190
+ #endif /* __dead2 */
191
+ #ifdef __disable_tail_calls
192
+ __disable_tail_calls
193
+ #endif /* __disable_tail_calls */
194
+ ;
195
+ #define __assert_fail(assertion, file, line, function) __assert(function, file, line, assertion)
196
+
197
+ #endif /* __assert_fail */
198
+
199
+ __cold void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, unsigned line) {
200
+ #if MDBX_DEBUG
201
+ if (env && env->assert_func)
202
+ env->assert_func(env, msg, func, line);
203
+ #else
204
+ (void)env;
205
+ assert_fail(msg, func, line);
206
+ }
207
+
208
+ MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line) {
209
+ #endif /* MDBX_DEBUG */
210
+
211
+ if (globals.logger.ptr)
212
+ debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg);
213
+ else {
214
+ #if defined(_WIN32) || defined(_WIN64)
215
+ char *message = nullptr;
216
+ const int num = osal_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", msg, func ? func : "unknown", line);
217
+ if (num < 1 || !message)
218
+ message = "<troubles with assertion-message preparation>";
219
+ OutputDebugStringA(message);
220
+ #else
221
+ __assert_fail(msg, "mdbx", line, func);
222
+ #endif
223
+ }
224
+
225
+ while (1) {
226
+ #if defined(_WIN32) || defined(_WIN64)
227
+ #if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG)
228
+ _CrtDbgReport(_CRT_ASSERT, func ? func : "unknown", line, "libmdbx", "assertion failed: %s", msg);
229
+ #else
230
+ if (IsDebuggerPresent())
231
+ DebugBreak();
232
+ #endif
233
+ FatalExit(STATUS_ASSERTION_FAILURE);
234
+ #else
235
+ abort();
236
+ #endif
237
+ }
238
+ }
239
+
240
+ __cold void mdbx_panic(const char *fmt, ...) {
241
+ va_list ap;
242
+ va_start(ap, fmt);
243
+
244
+ char *message = nullptr;
245
+ const int num = osal_vasprintf(&message, fmt, ap);
246
+ va_end(ap);
247
+ const char *const const_message =
248
+ unlikely(num < 1 || !message) ? "<troubles with panic-message preparation>" : message;
249
+
250
+ if (globals.logger.ptr)
251
+ debug_log(MDBX_LOG_FATAL, "panic", 0, "%s", const_message);
252
+
253
+ while (1) {
254
+ #if defined(_WIN32) || defined(_WIN64)
255
+ #if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG)
256
+ _CrtDbgReport(_CRT_ASSERT, "mdbx.c", 0, "libmdbx", "panic: %s", const_message);
257
+ #else
258
+ OutputDebugStringA("\r\nMDBX-PANIC: ");
259
+ OutputDebugStringA(const_message);
260
+ if (IsDebuggerPresent())
261
+ DebugBreak();
262
+ #endif
263
+ FatalExit(ERROR_UNHANDLED_ERROR);
264
+ #else
265
+ __assert_fail(const_message, "mdbx", 0, "panic");
266
+ abort();
267
+ #endif
268
+ }
269
+ }
270
+
271
+ /*----------------------------------------------------------------------------*/
272
+
273
+ #ifndef osal_vasprintf
274
+ MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap) {
275
+ va_list ones;
276
+ va_copy(ones, ap);
277
+ const int needed = vsnprintf(nullptr, 0, fmt, ones);
278
+ va_end(ones);
279
+
280
+ if (unlikely(needed < 0 || needed >= INT_MAX)) {
281
+ *strp = nullptr;
282
+ return needed;
283
+ }
284
+
285
+ *strp = osal_malloc(needed + (size_t)1);
286
+ if (unlikely(*strp == nullptr)) {
287
+ #if defined(_WIN32) || defined(_WIN64)
288
+ SetLastError(MDBX_ENOMEM);
289
+ #else
290
+ errno = MDBX_ENOMEM;
291
+ #endif
292
+ return -1;
293
+ }
294
+
295
+ const int actual = vsnprintf(*strp, needed + (size_t)1, fmt, ap);
296
+ assert(actual == needed);
297
+ if (unlikely(actual < 0)) {
298
+ osal_free(*strp);
299
+ *strp = nullptr;
300
+ }
301
+ return actual;
302
+ }
303
+ #endif /* osal_vasprintf */
304
+
305
+ #ifndef osal_asprintf
306
+ MDBX_INTERNAL int osal_asprintf(char **strp, const char *fmt, ...) {
307
+ va_list ap;
308
+ va_start(ap, fmt);
309
+ const int rc = osal_vasprintf(strp, fmt, ap);
310
+ va_end(ap);
311
+ return rc;
312
+ }
313
+ #endif /* osal_asprintf */
314
+
315
+ #ifndef osal_memalign_alloc
316
+ MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result) {
317
+ assert(is_powerof2(alignment) && alignment >= sizeof(void *));
318
+ #if defined(_WIN32) || defined(_WIN64)
319
+ (void)alignment;
320
+ *result = VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
321
+ return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */;
322
+ #elif defined(_ISOC11_SOURCE)
323
+ *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment));
324
+ return *result ? MDBX_SUCCESS : errno;
325
+ #elif _POSIX_VERSION >= 200112L && (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17)
326
+ *result = nullptr;
327
+ return posix_memalign(result, alignment, bytes);
328
+ #elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L
329
+ *result = memalign(alignment, bytes);
330
+ return *result ? MDBX_SUCCESS : errno;
331
+ #else
332
+ #error FIXME
333
+ #endif
334
+ }
335
+ #endif /* osal_memalign_alloc */
336
+
337
+ #ifndef osal_memalign_free
338
+ MDBX_INTERNAL void osal_memalign_free(void *ptr) {
339
+ #if defined(_WIN32) || defined(_WIN64)
340
+ VirtualFree(ptr, 0, MEM_RELEASE);
341
+ #else
342
+ osal_free(ptr);
343
+ #endif
344
+ }
345
+ #endif /* osal_memalign_free */
346
+
347
+ #ifndef osal_strdup
348
+ char *osal_strdup(const char *str) {
349
+ if (!str)
350
+ return nullptr;
351
+ size_t bytes = strlen(str) + 1;
352
+ char *dup = osal_malloc(bytes);
353
+ if (dup)
354
+ memcpy(dup, str, bytes);
355
+ return dup;
356
+ }
357
+ #endif /* osal_strdup */
358
+
359
+ /*----------------------------------------------------------------------------*/
360
+
361
+ MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair) {
362
+ int rc;
363
+ memset(condpair, 0, sizeof(osal_condpair_t));
364
+ #if defined(_WIN32) || defined(_WIN64)
365
+ if (!(condpair->mutex = CreateMutexW(nullptr, FALSE, nullptr))) {
366
+ rc = (int)GetLastError();
367
+ goto bailout_mutex;
368
+ }
369
+ if (!(condpair->event[0] = CreateEventW(nullptr, FALSE, FALSE, nullptr))) {
370
+ rc = (int)GetLastError();
371
+ goto bailout_event;
372
+ }
373
+ if ((condpair->event[1] = CreateEventW(nullptr, FALSE, FALSE, nullptr)))
374
+ return MDBX_SUCCESS;
375
+
376
+ rc = (int)GetLastError();
377
+ (void)CloseHandle(condpair->event[0]);
378
+ bailout_event:
379
+ (void)CloseHandle(condpair->mutex);
380
+ #else
381
+ rc = pthread_mutex_init(&condpair->mutex, nullptr);
382
+ if (unlikely(rc != 0))
383
+ goto bailout_mutex;
384
+ rc = pthread_cond_init(&condpair->cond[0], nullptr);
385
+ if (unlikely(rc != 0))
386
+ goto bailout_cond;
387
+ rc = pthread_cond_init(&condpair->cond[1], nullptr);
388
+ if (likely(rc == 0))
389
+ return MDBX_SUCCESS;
390
+
391
+ (void)pthread_cond_destroy(&condpair->cond[0]);
392
+ bailout_cond:
393
+ (void)pthread_mutex_destroy(&condpair->mutex);
394
+ #endif
395
+ bailout_mutex:
396
+ memset(condpair, 0, sizeof(osal_condpair_t));
397
+ return rc;
398
+ }
399
+
400
+ MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair) {
401
+ #if defined(_WIN32) || defined(_WIN64)
402
+ int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
403
+ rc = CloseHandle(condpair->event[0]) ? rc : (int)GetLastError();
404
+ rc = CloseHandle(condpair->event[1]) ? rc : (int)GetLastError();
405
+ #else
406
+ int err, rc = pthread_mutex_destroy(&condpair->mutex);
407
+ rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc;
408
+ rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc;
409
+ #endif
410
+ memset(condpair, 0, sizeof(osal_condpair_t));
411
+ return rc;
412
+ }
413
+
414
+ MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair) {
415
+ #if defined(_WIN32) || defined(_WIN64)
416
+ DWORD code = WaitForSingleObject(condpair->mutex, INFINITE);
417
+ return waitstatus2errcode(code);
418
+ #else
419
+ return osal_pthread_mutex_lock(&condpair->mutex);
420
+ #endif
421
+ }
422
+
423
+ MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair) {
424
+ #if defined(_WIN32) || defined(_WIN64)
425
+ return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
426
+ #else
427
+ return pthread_mutex_unlock(&condpair->mutex);
428
+ #endif
429
+ }
430
+
431
+ MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part) {
432
+ #if defined(_WIN32) || defined(_WIN64)
433
+ return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : (int)GetLastError();
434
+ #else
435
+ return pthread_cond_signal(&condpair->cond[part]);
436
+ #endif
437
+ }
438
+
439
+ MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part) {
440
+ #if defined(_WIN32) || defined(_WIN64)
441
+ DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], INFINITE, FALSE);
442
+ if (code == WAIT_OBJECT_0) {
443
+ code = WaitForSingleObject(condpair->mutex, INFINITE);
444
+ if (code == WAIT_OBJECT_0)
445
+ return MDBX_SUCCESS;
446
+ }
447
+ return waitstatus2errcode(code);
448
+ #else
449
+ return pthread_cond_wait(&condpair->cond[part], &condpair->mutex);
450
+ #endif
451
+ }
452
+
453
+ /*----------------------------------------------------------------------------*/
454
+
455
+ MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex) {
456
+ #if defined(_WIN32) || defined(_WIN64)
457
+ InitializeCriticalSection(fastmutex);
458
+ return MDBX_SUCCESS;
459
+ #elif MDBX_DEBUG
460
+ pthread_mutexattr_t ma;
461
+ int rc = pthread_mutexattr_init(&ma);
462
+ if (likely(!rc)) {
463
+ rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
464
+ if (likely(!rc) || rc == ENOTSUP)
465
+ rc = pthread_mutex_init(fastmutex, &ma);
466
+ pthread_mutexattr_destroy(&ma);
467
+ }
468
+ return rc;
469
+ #else
470
+ return pthread_mutex_init(fastmutex, nullptr);
471
+ #endif
472
+ }
473
+
474
+ MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex) {
475
+ #if defined(_WIN32) || defined(_WIN64)
476
+ DeleteCriticalSection(fastmutex);
477
+ return MDBX_SUCCESS;
478
+ #else
479
+ return pthread_mutex_destroy(fastmutex);
480
+ #endif
481
+ }
482
+
483
+ MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex) {
484
+ #if defined(_WIN32) || defined(_WIN64)
485
+ __try {
486
+ EnterCriticalSection(fastmutex);
487
+ } __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
488
+ ? EXCEPTION_EXECUTE_HANDLER
489
+ : EXCEPTION_CONTINUE_SEARCH) {
490
+ return MDBX_EDEADLK;
491
+ }
492
+ return MDBX_SUCCESS;
493
+ #else
494
+ return osal_pthread_mutex_lock(fastmutex);
495
+ #endif
496
+ }
497
+
498
+ MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex) {
499
+ #if defined(_WIN32) || defined(_WIN64)
500
+ LeaveCriticalSection(fastmutex);
501
+ return MDBX_SUCCESS;
502
+ #else
503
+ return pthread_mutex_unlock(fastmutex);
504
+ #endif
505
+ }
506
+
507
+ /*----------------------------------------------------------------------------*/
508
+
509
+ #if defined(_WIN32) || defined(_WIN64)
510
+
511
+ MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst) {
512
+ const size_t dst_wlen = MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, nullptr, 0);
513
+ wchar_t *dst = *pdst;
514
+ int rc = ERROR_INVALID_NAME;
515
+ if (unlikely(dst_wlen < 2 || dst_wlen > /* MAX_PATH */ INT16_MAX))
516
+ goto bailout;
517
+
518
+ dst = osal_realloc(dst, dst_wlen * sizeof(wchar_t));
519
+ rc = MDBX_ENOMEM;
520
+ if (unlikely(!dst))
521
+ goto bailout;
522
+
523
+ *pdst = dst;
524
+ if (likely(dst_wlen == (size_t)MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, dst, (int)dst_wlen)))
525
+ return MDBX_SUCCESS;
526
+
527
+ rc = ERROR_INVALID_NAME;
528
+ bailout:
529
+ if (*pdst) {
530
+ osal_free(*pdst);
531
+ *pdst = nullptr;
532
+ }
533
+ return rc;
534
+ }
535
+
536
+ #endif /* Windows */
537
+
538
+ /*----------------------------------------------------------------------------*/
539
+
540
+ #if defined(_WIN32) || defined(_WIN64)
541
+ #define ior_alignment_mask (ior->pagesize - 1)
542
+ #define ior_WriteFile_flag 1
543
+ #define OSAL_IOV_MAX (4096 / sizeof(ior_sgv_element))
544
+
545
+ static void ior_put_event(osal_ioring_t *ior, HANDLE event) {
546
+ assert(event && event != INVALID_HANDLE_VALUE && event != ior);
547
+ assert(ior->event_stack < ior->allocated);
548
+ ior->event_pool[ior->event_stack] = event;
549
+ ior->event_stack += 1;
550
+ }
551
+
552
+ static HANDLE ior_get_event(osal_ioring_t *ior) {
553
+ assert(ior->event_stack <= ior->allocated);
554
+ if (ior->event_stack > 0) {
555
+ ior->event_stack -= 1;
556
+ assert(ior->event_pool[ior->event_stack] != 0);
557
+ return ior->event_pool[ior->event_stack];
558
+ }
559
+ return CreateEventW(nullptr, true, false, nullptr);
560
+ }
561
+
562
+ static void WINAPI ior_wocr(DWORD err, DWORD bytes, OVERLAPPED *ov) {
563
+ osal_ioring_t *ior = ov->hEvent;
564
+ ov->Internal = err;
565
+ ov->InternalHigh = bytes;
566
+ if (++ior->async_completed >= ior->async_waiting)
567
+ SetEvent(ior->async_done);
568
+ }
569
+
570
+ #elif MDBX_HAVE_PWRITEV
571
+ #if defined(_SC_IOV_MAX)
572
+ static size_t osal_iov_max;
573
+ #define OSAL_IOV_MAX osal_iov_max
574
+ #else
575
+ #define OSAL_IOV_MAX IOV_MAX
576
+ #endif
577
+ #else
578
+ #undef OSAL_IOV_MAX
579
+ #endif /* OSAL_IOV_MAX */
580
+
581
+ MDBX_INTERNAL int osal_ioring_create(osal_ioring_t *ior
582
+ #if defined(_WIN32) || defined(_WIN64)
583
+ ,
584
+ bool enable_direct, mdbx_filehandle_t overlapped_fd
585
+ #endif /* Windows */
586
+ ) {
587
+ memset(ior, 0, sizeof(osal_ioring_t));
588
+
589
+ #if defined(_WIN32) || defined(_WIN64)
590
+ ior->overlapped_fd = overlapped_fd;
591
+ ior->direct = enable_direct && overlapped_fd;
592
+ ior->pagesize = globals.sys_pagesize;
593
+ ior->pagesize_ln2 = globals.sys_pagesize_ln2;
594
+ ior->async_done = ior_get_event(ior);
595
+ if (!ior->async_done)
596
+ return GetLastError();
597
+ #endif /* !Windows */
598
+
599
+ #if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX)
600
+ assert(osal_iov_max > 0);
601
+ #endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */
602
+
603
+ ior->boundary = ptr_disp(ior->pool, ior->allocated);
604
+ return MDBX_SUCCESS;
605
+ }
606
+
607
+ static inline size_t ior_offset(const ior_item_t *item) {
608
+ #if defined(_WIN32) || defined(_WIN64)
609
+ return item->ov.Offset |
610
+ (size_t)((sizeof(size_t) > sizeof(item->ov.Offset)) ? (uint64_t)item->ov.OffsetHigh << 32 : 0);
611
+ #else
612
+ return item->offset;
613
+ #endif /* !Windows */
614
+ }
615
+
616
+ static inline ior_item_t *ior_next(ior_item_t *item, size_t sgvcnt) {
617
+ #if defined(ior_sgv_element)
618
+ assert(sgvcnt > 0);
619
+ return (ior_item_t *)ptr_disp(item, sizeof(ior_item_t) - sizeof(ior_sgv_element) + sizeof(ior_sgv_element) * sgvcnt);
620
+ #else
621
+ assert(sgvcnt == 1);
622
+ (void)sgvcnt;
623
+ return item + 1;
624
+ #endif
625
+ }
626
+
627
+ MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, void *data, const size_t bytes) {
628
+ assert(bytes && data);
629
+ assert(bytes % MDBX_MIN_PAGESIZE == 0 && bytes <= MAX_WRITE);
630
+ assert(offset % MDBX_MIN_PAGESIZE == 0 && offset + (uint64_t)bytes <= MAX_MAPSIZE);
631
+
632
+ #if defined(_WIN32) || defined(_WIN64)
633
+ const unsigned segments = (unsigned)(bytes >> ior->pagesize_ln2);
634
+ const bool use_gather = ior->direct && ior->overlapped_fd && ior->slots_left >= segments;
635
+ #endif /* Windows */
636
+
637
+ ior_item_t *item = ior->pool;
638
+ if (likely(ior->last)) {
639
+ item = ior->last;
640
+ if (unlikely(ior_offset(item) + ior_last_bytes(ior, item) == offset) &&
641
+ likely(ior_last_bytes(ior, item) + bytes <= MAX_WRITE)) {
642
+ #if defined(_WIN32) || defined(_WIN64)
643
+ if (use_gather &&
644
+ ((bytes | (uintptr_t)data | ior->last_bytes | (uintptr_t)(uint64_t)item->sgv[0].Buffer) &
645
+ ior_alignment_mask) == 0 &&
646
+ ior->last_sgvcnt + (size_t)segments < OSAL_IOV_MAX) {
647
+ assert(ior->overlapped_fd);
648
+ assert((item->single.iov_len & ior_WriteFile_flag) == 0);
649
+ assert(item->sgv[ior->last_sgvcnt].Buffer == 0);
650
+ ior->last_bytes += bytes;
651
+ size_t i = 0;
652
+ do {
653
+ item->sgv[ior->last_sgvcnt + i].Buffer = PtrToPtr64(data);
654
+ data = ptr_disp(data, ior->pagesize);
655
+ } while (++i < segments);
656
+ ior->slots_left -= segments;
657
+ item->sgv[ior->last_sgvcnt += segments].Buffer = 0;
658
+ assert((item->single.iov_len & ior_WriteFile_flag) == 0);
659
+ return MDBX_SUCCESS;
660
+ }
661
+ const void *end = ptr_disp(item->single.iov_base, item->single.iov_len - ior_WriteFile_flag);
662
+ if (unlikely(end == data)) {
663
+ assert((item->single.iov_len & ior_WriteFile_flag) != 0);
664
+ item->single.iov_len += bytes;
665
+ return MDBX_SUCCESS;
666
+ }
667
+ #elif MDBX_HAVE_PWRITEV
668
+ assert((int)item->sgvcnt > 0);
669
+ const void *end = ptr_disp(item->sgv[item->sgvcnt - 1].iov_base, item->sgv[item->sgvcnt - 1].iov_len);
670
+ if (unlikely(end == data)) {
671
+ item->sgv[item->sgvcnt - 1].iov_len += bytes;
672
+ ior->last_bytes += bytes;
673
+ return MDBX_SUCCESS;
674
+ }
675
+ if (likely(item->sgvcnt < OSAL_IOV_MAX)) {
676
+ if (unlikely(ior->slots_left < 1))
677
+ return MDBX_RESULT_TRUE;
678
+ item->sgv[item->sgvcnt].iov_base = data;
679
+ item->sgv[item->sgvcnt].iov_len = bytes;
680
+ ior->last_bytes += bytes;
681
+ item->sgvcnt += 1;
682
+ ior->slots_left -= 1;
683
+ return MDBX_SUCCESS;
684
+ }
685
+ #else
686
+ const void *end = ptr_disp(item->single.iov_base, item->single.iov_len);
687
+ if (unlikely(end == data)) {
688
+ item->single.iov_len += bytes;
689
+ return MDBX_SUCCESS;
690
+ }
691
+ #endif
692
+ }
693
+ item = ior_next(item, ior_last_sgvcnt(ior, item));
694
+ }
695
+
696
+ if (unlikely(ior->slots_left < 1))
697
+ return MDBX_RESULT_TRUE;
698
+
699
+ unsigned slots_used = 1;
700
+ #if defined(_WIN32) || defined(_WIN64)
701
+ item->ov.Internal = item->ov.InternalHigh = 0;
702
+ item->ov.Offset = (DWORD)offset;
703
+ item->ov.OffsetHigh = HIGH_DWORD(offset);
704
+ item->ov.hEvent = 0;
705
+ if (!use_gather || ((bytes | (uintptr_t)(data)) & ior_alignment_mask) != 0 || segments > OSAL_IOV_MAX) {
706
+ /* WriteFile() */
707
+ item->single.iov_base = data;
708
+ item->single.iov_len = bytes + ior_WriteFile_flag;
709
+ assert((item->single.iov_len & ior_WriteFile_flag) != 0);
710
+ } else {
711
+ /* WriteFileGather() */
712
+ assert(ior->overlapped_fd);
713
+ item->sgv[0].Buffer = PtrToPtr64(data);
714
+ for (size_t i = 1; i < segments; ++i) {
715
+ data = ptr_disp(data, ior->pagesize);
716
+ item->sgv[i].Buffer = PtrToPtr64(data);
717
+ }
718
+ item->sgv[slots_used = segments].Buffer = 0;
719
+ assert((item->single.iov_len & ior_WriteFile_flag) == 0);
720
+ }
721
+ ior->last_bytes = bytes;
722
+ ior_last_sgvcnt(ior, item) = slots_used;
723
+ #elif MDBX_HAVE_PWRITEV
724
+ item->offset = offset;
725
+ item->sgv[0].iov_base = data;
726
+ item->sgv[0].iov_len = bytes;
727
+ ior->last_bytes = bytes;
728
+ ior_last_sgvcnt(ior, item) = slots_used;
729
+ #else
730
+ item->offset = offset;
731
+ item->single.iov_base = data;
732
+ item->single.iov_len = bytes;
733
+ #endif /* !Windows */
734
+ ior->slots_left -= slots_used;
735
+ ior->last = item;
736
+ return MDBX_SUCCESS;
737
+ }
738
+
739
+ MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx,
740
+ void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)) {
741
+ for (ior_item_t *item = ior->pool; item <= ior->last;) {
742
+ #if defined(_WIN32) || defined(_WIN64)
743
+ size_t offset = ior_offset(item);
744
+ char *data = item->single.iov_base;
745
+ size_t bytes = item->single.iov_len - ior_WriteFile_flag;
746
+ size_t i = 1;
747
+ if (bytes & ior_WriteFile_flag) {
748
+ data = Ptr64ToPtr(item->sgv[0].Buffer);
749
+ bytes = ior->pagesize;
750
+ /* Zap: Reading invalid data from 'item->sgv' */
751
+ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
752
+ while (item->sgv[i].Buffer) {
753
+ if (data + ior->pagesize != item->sgv[i].Buffer) {
754
+ callback(ctx, offset, data, bytes);
755
+ offset += bytes;
756
+ data = Ptr64ToPtr(item->sgv[i].Buffer);
757
+ bytes = 0;
758
+ }
759
+ bytes += ior->pagesize;
760
+ ++i;
761
+ }
762
+ }
763
+ assert(bytes < MAX_WRITE);
764
+ callback(ctx, offset, data, bytes);
765
+ #elif MDBX_HAVE_PWRITEV
766
+ assert(item->sgvcnt > 0);
767
+ size_t offset = item->offset;
768
+ size_t i = 0;
769
+ do {
770
+ callback(ctx, offset, item->sgv[i].iov_base, item->sgv[i].iov_len);
771
+ offset += item->sgv[i].iov_len;
772
+ } while (++i != item->sgvcnt);
773
+ #else
774
+ const size_t i = 1;
775
+ callback(ctx, item->offset, item->single.iov_base, item->single.iov_len);
776
+ #endif
777
+ item = ior_next(item, i);
778
+ }
779
+ }
780
+
781
+ MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) {
782
+ osal_ioring_write_result_t r = {MDBX_SUCCESS, 0};
783
+
784
+ #if defined(_WIN32) || defined(_WIN64)
785
+ HANDLE *const end_wait_for = ior->event_pool + ior->allocated +
786
+ /* был выделен один дополнительный элемент для async_done */ 1;
787
+ HANDLE *wait_for = end_wait_for;
788
+ LONG async_started = 0;
789
+ for (ior_item_t *item = ior->pool; item <= ior->last;) {
790
+ item->ov.Internal = STATUS_PENDING;
791
+ size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag;
792
+ r.wops += 1;
793
+ if (bytes & ior_WriteFile_flag) {
794
+ assert(ior->overlapped_fd && fd == ior->overlapped_fd);
795
+ bytes = ior->pagesize;
796
+ /* Zap: Reading invalid data from 'item->sgv' */
797
+ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
798
+ while (item->sgv[i].Buffer) {
799
+ bytes += ior->pagesize;
800
+ ++i;
801
+ }
802
+ assert(bytes < MAX_WRITE);
803
+ item->ov.hEvent = ior_get_event(ior);
804
+ if (unlikely(!item->ov.hEvent)) {
805
+ bailout_geterr:
806
+ r.err = GetLastError();
807
+ bailout_rc:
808
+ assert(r.err != MDBX_SUCCESS);
809
+ CancelIo(fd);
810
+ return r;
811
+ }
812
+ if (WriteFileGather(fd, item->sgv, (DWORD)bytes, nullptr, &item->ov)) {
813
+ assert(item->ov.Internal == 0 && WaitForSingleObject(item->ov.hEvent, 0) == WAIT_OBJECT_0);
814
+ ior_put_event(ior, item->ov.hEvent);
815
+ item->ov.hEvent = 0;
816
+ } else {
817
+ r.err = (int)GetLastError();
818
+ if (unlikely(r.err != ERROR_IO_PENDING)) {
819
+ void *data = Ptr64ToPtr(item->sgv[0].Buffer);
820
+ ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
821
+ " offset %" PRId64 ", err %d",
822
+ "WriteFileGather", fd, __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno,
823
+ bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err);
824
+ goto bailout_rc;
825
+ }
826
+ assert(wait_for > ior->event_pool + ior->event_stack);
827
+ *--wait_for = item->ov.hEvent;
828
+ }
829
+ } else if (fd == ior->overlapped_fd) {
830
+ assert(bytes < MAX_WRITE);
831
+ retry:
832
+ item->ov.hEvent = ior;
833
+ if (WriteFileEx(fd, item->single.iov_base, (DWORD)bytes, &item->ov, ior_wocr)) {
834
+ async_started += 1;
835
+ } else {
836
+ r.err = (int)GetLastError();
837
+ switch (r.err) {
838
+ default:
839
+ ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
840
+ " offset %" PRId64 ", err %d",
841
+ "WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
842
+ ((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
843
+ r.err);
844
+ goto bailout_rc;
845
+ case ERROR_NOT_FOUND:
846
+ case ERROR_USER_MAPPED_FILE:
847
+ case ERROR_LOCK_VIOLATION:
848
+ WARNING("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
849
+ " offset %" PRId64 ", err %d",
850
+ "WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
851
+ ((page_t *)item->single.iov_base)->pgno, bytes,
852
+ item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err);
853
+ SleepEx(0, true);
854
+ goto retry;
855
+ case ERROR_INVALID_USER_BUFFER:
856
+ case ERROR_NOT_ENOUGH_MEMORY:
857
+ if (SleepEx(0, true) == WAIT_IO_COMPLETION)
858
+ goto retry;
859
+ goto bailout_rc;
860
+ case ERROR_IO_PENDING:
861
+ async_started += 1;
862
+ }
863
+ }
864
+ } else {
865
+ assert(bytes < MAX_WRITE);
866
+ DWORD written = 0;
867
+ if (!WriteFile(fd, item->single.iov_base, (DWORD)bytes, &written, &item->ov)) {
868
+ r.err = (int)GetLastError();
869
+ ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
870
+ " offset %" PRId64 ", err %d",
871
+ "WriteFile", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
872
+ ((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
873
+ r.err);
874
+ goto bailout_rc;
875
+ } else if (unlikely(written != bytes)) {
876
+ r.err = ERROR_WRITE_FAULT;
877
+ goto bailout_rc;
878
+ }
879
+ }
880
+ item = ior_next(item, i);
881
+ }
882
+
883
+ assert(ior->async_waiting > ior->async_completed && ior->async_waiting == INT_MAX);
884
+ ior->async_waiting = async_started;
885
+ if (async_started > ior->async_completed && end_wait_for == wait_for) {
886
+ assert(wait_for > ior->event_pool + ior->event_stack);
887
+ *--wait_for = ior->async_done;
888
+ }
889
+
890
+ const size_t pending_count = end_wait_for - wait_for;
891
+ if (pending_count) {
892
+ /* Ждем до MAXIMUM_WAIT_OBJECTS (64) последних хендлов, а после избирательно
893
+ * ждем посредством GetOverlappedResult(), если какие-то более ранние
894
+ * элементы еще не завершены. В целом, так получается меньше системных
895
+ * вызовов, т.е. меньше накладных расходов. Однако, не факт что эта экономия
896
+ * не будет перекрыта неэффективностью реализации
897
+ * WaitForMultipleObjectsEx(), но тогда это проблемы на стороне M$. */
898
+ DWORD madness;
899
+ do
900
+ madness = WaitForMultipleObjectsEx(
901
+ (pending_count < MAXIMUM_WAIT_OBJECTS) ? (DWORD)pending_count : MAXIMUM_WAIT_OBJECTS, wait_for, true,
902
+ /* сутки */ 86400000ul, true);
903
+ while (madness == WAIT_IO_COMPLETION);
904
+ STATIC_ASSERT(WAIT_OBJECT_0 == 0);
905
+ if (/* madness >= WAIT_OBJECT_0 && */
906
+ madness < WAIT_OBJECT_0 + MAXIMUM_WAIT_OBJECTS)
907
+ r.err = MDBX_SUCCESS;
908
+ else if (madness >= WAIT_ABANDONED_0 && madness < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) {
909
+ r.err = ERROR_ABANDONED_WAIT_0;
910
+ goto bailout_rc;
911
+ } else if (madness == WAIT_TIMEOUT) {
912
+ r.err = WAIT_TIMEOUT;
913
+ goto bailout_rc;
914
+ } else {
915
+ r.err = /* madness == WAIT_FAILED */ MDBX_PROBLEM;
916
+ goto bailout_rc;
917
+ }
918
+
919
+ assert(ior->async_waiting == ior->async_completed);
920
+ for (ior_item_t *item = ior->pool; item <= ior->last;) {
921
+ size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag;
922
+ void *data = item->single.iov_base;
923
+ if (bytes & ior_WriteFile_flag) {
924
+ data = Ptr64ToPtr(item->sgv[0].Buffer);
925
+ bytes = ior->pagesize;
926
+ /* Zap: Reading invalid data from 'item->sgv' */
927
+ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
928
+ while (item->sgv[i].Buffer) {
929
+ bytes += ior->pagesize;
930
+ ++i;
931
+ }
932
+ if (!HasOverlappedIoCompleted(&item->ov)) {
933
+ DWORD written = 0;
934
+ if (unlikely(!GetOverlappedResult(fd, &item->ov, &written, true))) {
935
+ ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu,"
936
+ " offset %" PRId64 ", err %d",
937
+ "GetOverlappedResult", __Wpedantic_format_voidptr(item), item - ior->pool, data,
938
+ ((page_t *)data)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
939
+ (int)GetLastError());
940
+ goto bailout_geterr;
941
+ }
942
+ assert(MDBX_SUCCESS == item->ov.Internal);
943
+ assert(written == item->ov.InternalHigh);
944
+ }
945
+ } else {
946
+ assert(HasOverlappedIoCompleted(&item->ov));
947
+ }
948
+ assert(item->ov.Internal != ERROR_IO_PENDING);
949
+ if (unlikely(item->ov.Internal != MDBX_SUCCESS)) {
950
+ DWORD written = 0;
951
+ r.err = (int)item->ov.Internal;
952
+ if ((r.err & 0x80000000) && GetOverlappedResult(nullptr, &item->ov, &written, true))
953
+ r.err = (int)GetLastError();
954
+ ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu,"
955
+ " offset %" PRId64 ", err %d",
956
+ "Result", __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno, bytes,
957
+ item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), (int)GetLastError());
958
+ goto bailout_rc;
959
+ }
960
+ if (unlikely(item->ov.InternalHigh != bytes)) {
961
+ r.err = ERROR_WRITE_FAULT;
962
+ goto bailout_rc;
963
+ }
964
+ item = ior_next(item, i);
965
+ }
966
+ assert(ior->async_waiting == ior->async_completed);
967
+ } else {
968
+ assert(r.err == MDBX_SUCCESS);
969
+ }
970
+ assert(ior->async_waiting == ior->async_completed);
971
+
972
+ #else
973
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
974
+ for (ior_item_t *item = ior->pool; item <= ior->last;) {
975
+ #if MDBX_HAVE_PWRITEV
976
+ assert(item->sgvcnt > 0);
977
+ if (item->sgvcnt == 1)
978
+ r.err = osal_pwrite(fd, item->sgv[0].iov_base, item->sgv[0].iov_len, item->offset);
979
+ else
980
+ r.err = osal_pwritev(fd, item->sgv, item->sgvcnt, item->offset);
981
+
982
+ // TODO: io_uring_prep_write(sqe, fd, ...);
983
+
984
+ item = ior_next(item, item->sgvcnt);
985
+ #else
986
+ r.err = osal_pwrite(fd, item->single.iov_base, item->single.iov_len, item->offset);
987
+ item = ior_next(item, 1);
988
+ #endif
989
+ r.wops += 1;
990
+ if (unlikely(r.err != MDBX_SUCCESS))
991
+ break;
992
+ }
993
+
994
+ // TODO: io_uring_submit(&ring)
995
+ // TODO: err = io_uring_wait_cqe(&ring, &cqe);
996
+ // TODO: io_uring_cqe_seen(&ring, cqe);
997
+
998
+ #endif /* !Windows */
999
+ return r;
1000
+ }
1001
+
1002
+ MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *ior) {
1003
+ #if defined(_WIN32) || defined(_WIN64)
1004
+ if (ior->last) {
1005
+ for (ior_item_t *item = ior->pool; item <= ior->last;) {
1006
+ if (!HasOverlappedIoCompleted(&item->ov)) {
1007
+ assert(ior->overlapped_fd);
1008
+ CancelIoEx(ior->overlapped_fd, &item->ov);
1009
+ }
1010
+ if (item->ov.hEvent && item->ov.hEvent != ior)
1011
+ ior_put_event(ior, item->ov.hEvent);
1012
+ size_t i = 1;
1013
+ if ((item->single.iov_len & ior_WriteFile_flag) == 0) {
1014
+ /* Zap: Reading invalid data from 'item->sgv' */
1015
+ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
1016
+ while (item->sgv[i].Buffer)
1017
+ ++i;
1018
+ }
1019
+ item = ior_next(item, i);
1020
+ }
1021
+ }
1022
+ ior->async_waiting = INT_MAX;
1023
+ ior->async_completed = 0;
1024
+ ResetEvent(ior->async_done);
1025
+ #endif /* !Windows */
1026
+ ior->slots_left = ior->allocated;
1027
+ ior->last = nullptr;
1028
+ }
1029
+
1030
+ static void ior_cleanup(osal_ioring_t *ior, const size_t since) {
1031
+ osal_ioring_reset(ior);
1032
+ #if defined(_WIN32) || defined(_WIN64)
1033
+ for (size_t i = since; i < ior->event_stack; ++i) {
1034
+ /* Zap: Using uninitialized memory '**ior.event_pool' */
1035
+ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6001);
1036
+ CloseHandle(ior->event_pool[i]);
1037
+ }
1038
+ ior->event_stack = 0;
1039
+ #else
1040
+ (void)since;
1041
+ #endif /* Windows */
1042
+ }
1043
+
1044
+ MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *ior, size_t items) {
1045
+ assert(items > 0 && items < INT_MAX / sizeof(ior_item_t));
1046
+ #if defined(_WIN32) || defined(_WIN64)
1047
+ if (ior->state & IOR_STATE_LOCKED)
1048
+ return MDBX_SUCCESS;
1049
+ const bool useSetFileIoOverlappedRange = ior->overlapped_fd && imports.SetFileIoOverlappedRange && items > 42;
1050
+ const size_t ceiling =
1051
+ useSetFileIoOverlappedRange ? ((items < 65536 / 2 / sizeof(ior_item_t)) ? 65536 : 65536 * 4) : 1024;
1052
+ const size_t bytes = ceil_powerof2(sizeof(ior_item_t) * items, ceiling);
1053
+ items = bytes / sizeof(ior_item_t);
1054
+ #endif /* Windows */
1055
+
1056
+ if (items != ior->allocated) {
1057
+ assert(items >= osal_ioring_used(ior));
1058
+ if (items < ior->allocated)
1059
+ ior_cleanup(ior, items);
1060
+ #if defined(_WIN32) || defined(_WIN64)
1061
+ void *ptr = osal_realloc(ior->event_pool, (items + /* extra for waiting the async_done */ 1) * sizeof(HANDLE));
1062
+ if (unlikely(!ptr))
1063
+ return MDBX_ENOMEM;
1064
+ ior->event_pool = ptr;
1065
+
1066
+ int err = osal_memalign_alloc(ceiling, bytes, &ptr);
1067
+ if (unlikely(err != MDBX_SUCCESS))
1068
+ return err;
1069
+ if (ior->pool) {
1070
+ memcpy(ptr, ior->pool, ior->allocated * sizeof(ior_item_t));
1071
+ osal_memalign_free(ior->pool);
1072
+ }
1073
+ #else
1074
+ void *ptr = osal_realloc(ior->pool, sizeof(ior_item_t) * items);
1075
+ if (unlikely(!ptr))
1076
+ return MDBX_ENOMEM;
1077
+ #endif
1078
+ ior->pool = ptr;
1079
+
1080
+ if (items > ior->allocated)
1081
+ memset(ior->pool + ior->allocated, 0, sizeof(ior_item_t) * (items - ior->allocated));
1082
+ ior->allocated = (unsigned)items;
1083
+ ior->boundary = ptr_disp(ior->pool, ior->allocated);
1084
+ #if defined(_WIN32) || defined(_WIN64)
1085
+ if (useSetFileIoOverlappedRange) {
1086
+ if (imports.SetFileIoOverlappedRange(ior->overlapped_fd, ptr, (ULONG)bytes))
1087
+ ior->state += IOR_STATE_LOCKED;
1088
+ else
1089
+ return GetLastError();
1090
+ }
1091
+ #endif /* Windows */
1092
+ }
1093
+ return MDBX_SUCCESS;
1094
+ }
1095
+
1096
+ MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *ior) {
1097
+ if (ior->allocated)
1098
+ ior_cleanup(ior, 0);
1099
+ #if defined(_WIN32) || defined(_WIN64)
1100
+ osal_memalign_free(ior->pool);
1101
+ osal_free(ior->event_pool);
1102
+ CloseHandle(ior->async_done);
1103
+ if (ior->overlapped_fd)
1104
+ CloseHandle(ior->overlapped_fd);
1105
+ #else
1106
+ osal_free(ior->pool);
1107
+ #endif
1108
+ memset(ior, 0, sizeof(osal_ioring_t));
1109
+ }
1110
+
1111
+ /*----------------------------------------------------------------------------*/
1112
+
1113
+ MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname) {
1114
+ #if defined(_WIN32) || defined(_WIN64)
1115
+ return DeleteFileW(pathname) ? MDBX_SUCCESS : (int)GetLastError();
1116
+ #else
1117
+ return unlink(pathname) ? errno : MDBX_SUCCESS;
1118
+ #endif
1119
+ }
1120
+
1121
+ #if !(defined(_WIN32) || defined(_WIN64))
1122
+ static bool is_valid_fd(int fd) { return !(isatty(fd) < 0 && errno == EBADF); }
1123
+ #endif /*! Windows */
1124
+
1125
+ MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname) {
1126
+ #if defined(_WIN32) || defined(_WIN64)
1127
+ return RemoveDirectoryW(pathname) ? MDBX_SUCCESS : (int)GetLastError();
1128
+ #else
1129
+ return rmdir(pathname) ? errno : MDBX_SUCCESS;
1130
+ #endif
1131
+ }
1132
+
1133
+ MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname) {
1134
+ #if defined(_WIN32) || defined(_WIN64)
1135
+ if (GetFileAttributesW(pathname) != INVALID_FILE_ATTRIBUTES)
1136
+ return MDBX_RESULT_TRUE;
1137
+ int err = GetLastError();
1138
+ return (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) ? MDBX_RESULT_FALSE : err;
1139
+ #else
1140
+ if (access(pathname, F_OK) == 0)
1141
+ return MDBX_RESULT_TRUE;
1142
+ int err = errno;
1143
+ return (err == ENOENT || err == ENOTDIR) ? MDBX_RESULT_FALSE : err;
1144
+ #endif
1145
+ }
1146
+
1147
+ MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len) {
1148
+ const pathchar_t *ext = nullptr;
1149
+ for (size_t i = 0; i < len && pathname[i]; i++)
1150
+ if (pathname[i] == '.')
1151
+ ext = pathname + i;
1152
+ else if (osal_isdirsep(pathname[i]))
1153
+ ext = nullptr;
1154
+ return (pathchar_t *)ext;
1155
+ }
1156
+
1157
+ MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len) {
1158
+ #if defined(_WIN32) || defined(_WIN64)
1159
+ for (size_t i = 0; i < len; ++i) {
1160
+ pathchar_t a = l[i];
1161
+ pathchar_t b = r[i];
1162
+ a = (a == '\\') ? '/' : a;
1163
+ b = (b == '\\') ? '/' : b;
1164
+ if (a != b)
1165
+ return false;
1166
+ }
1167
+ return true;
1168
+ #else
1169
+ return memcmp(l, r, len * sizeof(pathchar_t)) == 0;
1170
+ #endif
1171
+ }
1172
+
1173
+ MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env,
1174
+ const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits) {
1175
+ *fd = INVALID_HANDLE_VALUE;
1176
+
1177
+ #if defined(_WIN32) || defined(_WIN64)
1178
+ DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING;
1179
+ DWORD FlagsAndAttributes = FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED;
1180
+ DWORD DesiredAccess = FILE_READ_ATTRIBUTES;
1181
+ DWORD ShareMode = (env->flags & MDBX_EXCLUSIVE) ? 0 : (FILE_SHARE_READ | FILE_SHARE_WRITE);
1182
+
1183
+ switch (purpose) {
1184
+ default:
1185
+ return ERROR_INVALID_PARAMETER;
1186
+ case MDBX_OPEN_LCK:
1187
+ CreationDisposition = OPEN_ALWAYS;
1188
+ DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
1189
+ FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY;
1190
+ break;
1191
+ case MDBX_OPEN_DXB_READ:
1192
+ CreationDisposition = OPEN_EXISTING;
1193
+ DesiredAccess |= GENERIC_READ;
1194
+ ShareMode |= FILE_SHARE_READ;
1195
+ break;
1196
+ case MDBX_OPEN_DXB_LAZY:
1197
+ DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
1198
+ break;
1199
+ case MDBX_OPEN_DXB_OVERLAPPED_DIRECT:
1200
+ FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING;
1201
+ /* fall through */
1202
+ __fallthrough;
1203
+ case MDBX_OPEN_DXB_OVERLAPPED:
1204
+ FlagsAndAttributes |= FILE_FLAG_OVERLAPPED;
1205
+ /* fall through */
1206
+ __fallthrough;
1207
+ case MDBX_OPEN_DXB_DSYNC:
1208
+ CreationDisposition = OPEN_EXISTING;
1209
+ DesiredAccess |= GENERIC_WRITE | GENERIC_READ;
1210
+ FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH;
1211
+ break;
1212
+ case MDBX_OPEN_COPY:
1213
+ CreationDisposition = CREATE_NEW;
1214
+ ShareMode = 0;
1215
+ DesiredAccess |= GENERIC_WRITE;
1216
+ if (env->ps >= globals.sys_pagesize)
1217
+ FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING;
1218
+ break;
1219
+ case MDBX_OPEN_DELETE:
1220
+ CreationDisposition = OPEN_EXISTING;
1221
+ ShareMode |= FILE_SHARE_DELETE;
1222
+ DesiredAccess = FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE;
1223
+ break;
1224
+ }
1225
+
1226
+ *fd = CreateFileW(pathname, DesiredAccess, ShareMode, nullptr, CreationDisposition, FlagsAndAttributes, nullptr);
1227
+ if (*fd == INVALID_HANDLE_VALUE) {
1228
+ int err = (int)GetLastError();
1229
+ if (err == ERROR_ACCESS_DENIED && purpose == MDBX_OPEN_LCK) {
1230
+ if (GetFileAttributesW(pathname) == INVALID_FILE_ATTRIBUTES && GetLastError() == ERROR_FILE_NOT_FOUND)
1231
+ err = ERROR_FILE_NOT_FOUND;
1232
+ }
1233
+ return err;
1234
+ }
1235
+
1236
+ BY_HANDLE_FILE_INFORMATION info;
1237
+ if (!GetFileInformationByHandle(*fd, &info)) {
1238
+ int err = (int)GetLastError();
1239
+ CloseHandle(*fd);
1240
+ *fd = INVALID_HANDLE_VALUE;
1241
+ return err;
1242
+ }
1243
+ const DWORD AttributesDiff =
1244
+ (info.dwFileAttributes ^ FlagsAndAttributes) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED |
1245
+ FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED);
1246
+ if (AttributesDiff)
1247
+ (void)SetFileAttributesW(pathname, info.dwFileAttributes ^ AttributesDiff);
1248
+
1249
+ #else
1250
+ int flags = unix_mode_bits ? O_CREAT : 0;
1251
+ switch (purpose) {
1252
+ default:
1253
+ return EINVAL;
1254
+ case MDBX_OPEN_LCK:
1255
+ flags |= O_RDWR;
1256
+ break;
1257
+ case MDBX_OPEN_DXB_READ:
1258
+ flags = O_RDONLY;
1259
+ break;
1260
+ case MDBX_OPEN_DXB_LAZY:
1261
+ flags |= O_RDWR;
1262
+ break;
1263
+ case MDBX_OPEN_COPY:
1264
+ flags = O_CREAT | O_WRONLY | O_EXCL;
1265
+ break;
1266
+ case MDBX_OPEN_DXB_DSYNC:
1267
+ flags |= O_WRONLY;
1268
+ #if defined(O_DSYNC)
1269
+ flags |= O_DSYNC;
1270
+ #elif defined(O_SYNC)
1271
+ flags |= O_SYNC;
1272
+ #elif defined(O_FSYNC)
1273
+ flags |= O_FSYNC;
1274
+ #endif
1275
+ break;
1276
+ case MDBX_OPEN_DELETE:
1277
+ flags = O_RDWR;
1278
+ break;
1279
+ }
1280
+
1281
+ const bool direct_nocache_for_copy = env->ps >= globals.sys_pagesize && purpose == MDBX_OPEN_COPY;
1282
+ if (direct_nocache_for_copy) {
1283
+ #if defined(O_DIRECT)
1284
+ flags |= O_DIRECT;
1285
+ #endif /* O_DIRECT */
1286
+ #if defined(O_NOCACHE)
1287
+ flags |= O_NOCACHE;
1288
+ #endif /* O_NOCACHE */
1289
+ }
1290
+
1291
+ #ifdef O_CLOEXEC
1292
+ flags |= O_CLOEXEC;
1293
+ #endif /* O_CLOEXEC */
1294
+
1295
+ /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */
1296
+ #if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
1297
+ int stub_fd0 = -1, stub_fd1 = -1, stub_fd2 = -1;
1298
+ static const char dev_null[] = "/dev/null";
1299
+ if (!is_valid_fd(STDIN_FILENO)) {
1300
+ WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN", STDIN_FILENO, dev_null);
1301
+ stub_fd0 = open(dev_null, O_RDONLY | O_NOCTTY);
1302
+ }
1303
+ if (!is_valid_fd(STDOUT_FILENO)) {
1304
+ WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "OUT", STDOUT_FILENO, dev_null);
1305
+ stub_fd1 = open(dev_null, O_WRONLY | O_NOCTTY);
1306
+ }
1307
+ if (!is_valid_fd(STDERR_FILENO)) {
1308
+ WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "ERR", STDERR_FILENO, dev_null);
1309
+ stub_fd2 = open(dev_null, O_WRONLY | O_NOCTTY);
1310
+ }
1311
+ #else
1312
+ #error "Unexpected or unsupported UNIX or POSIX system"
1313
+ #endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */
1314
+
1315
+ *fd = open(pathname, flags, unix_mode_bits);
1316
+ #if defined(O_DIRECT)
1317
+ if (*fd < 0 && (flags & O_DIRECT) && (errno == EINVAL || errno == EAFNOSUPPORT)) {
1318
+ flags &= ~(O_DIRECT | O_EXCL);
1319
+ *fd = open(pathname, flags, unix_mode_bits);
1320
+ }
1321
+ #endif /* O_DIRECT */
1322
+
1323
+ if (*fd < 0 && errno == EACCES && purpose == MDBX_OPEN_LCK) {
1324
+ struct stat unused;
1325
+ if (stat(pathname, &unused) == 0 || errno != ENOENT)
1326
+ errno = EACCES /* restore errno if file exists */;
1327
+ }
1328
+
1329
+ /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */
1330
+ #if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
1331
+ if (*fd == STDIN_FILENO) {
1332
+ WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN", STDIN_FILENO);
1333
+ assert(stub_fd0 == -1);
1334
+ *fd = dup(stub_fd0 = *fd);
1335
+ }
1336
+ if (*fd == STDOUT_FILENO) {
1337
+ WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT", STDOUT_FILENO);
1338
+ assert(stub_fd1 == -1);
1339
+ *fd = dup(stub_fd1 = *fd);
1340
+ }
1341
+ if (*fd == STDERR_FILENO) {
1342
+ WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR", STDERR_FILENO);
1343
+ assert(stub_fd2 == -1);
1344
+ *fd = dup(stub_fd2 = *fd);
1345
+ }
1346
+ const int err = errno;
1347
+ if (stub_fd0 != -1)
1348
+ close(stub_fd0);
1349
+ if (stub_fd1 != -1)
1350
+ close(stub_fd1);
1351
+ if (stub_fd2 != -1)
1352
+ close(stub_fd2);
1353
+ if (*fd >= STDIN_FILENO && *fd <= STDERR_FILENO) {
1354
+ ERROR("Rejecting the use of a FD in the range "
1355
+ "STDIN_FILENO/%d..STDERR_FILENO/%d to prevent database corruption",
1356
+ STDIN_FILENO, STDERR_FILENO);
1357
+ close(*fd);
1358
+ return EBADF;
1359
+ }
1360
+ #else
1361
+ #error "Unexpected or unsupported UNIX or POSIX system"
1362
+ #endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */
1363
+
1364
+ if (*fd < 0)
1365
+ return err;
1366
+
1367
+ #if defined(FD_CLOEXEC) && !defined(O_CLOEXEC)
1368
+ const int fd_flags = fcntl(*fd, F_GETFD);
1369
+ if (fd_flags != -1)
1370
+ (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC);
1371
+ #endif /* FD_CLOEXEC && !O_CLOEXEC */
1372
+
1373
+ if (direct_nocache_for_copy) {
1374
+ #if defined(F_NOCACHE) && !defined(O_NOCACHE)
1375
+ (void)fcntl(*fd, F_NOCACHE, 1);
1376
+ #endif /* F_NOCACHE */
1377
+ }
1378
+
1379
+ #endif
1380
+ return MDBX_SUCCESS;
1381
+ }
1382
+
1383
+ MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd) {
1384
+ #if defined(_WIN32) || defined(_WIN64)
1385
+ return CloseHandle(fd) ? MDBX_SUCCESS : (int)GetLastError();
1386
+ #else
1387
+ assert(fd > STDERR_FILENO);
1388
+ return (close(fd) == 0) ? MDBX_SUCCESS : errno;
1389
+ #endif
1390
+ }
1391
+
1392
+ MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, uint64_t offset) {
1393
+ if (bytes > MAX_WRITE)
1394
+ return MDBX_EINVAL;
1395
+ #if defined(_WIN32) || defined(_WIN64)
1396
+ OVERLAPPED ov;
1397
+ ov.hEvent = 0;
1398
+ ov.Offset = (DWORD)offset;
1399
+ ov.OffsetHigh = HIGH_DWORD(offset);
1400
+
1401
+ DWORD read = 0;
1402
+ if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) {
1403
+ int rc = (int)GetLastError();
1404
+ return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc;
1405
+ }
1406
+ #else
1407
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
1408
+ intptr_t read = pread(fd, buf, bytes, offset);
1409
+ if (read < 0) {
1410
+ int rc = errno;
1411
+ return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc;
1412
+ }
1413
+ #endif
1414
+ return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA;
1415
+ }
1416
+
1417
+ MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t bytes, uint64_t offset) {
1418
+ while (true) {
1419
+ #if defined(_WIN32) || defined(_WIN64)
1420
+ OVERLAPPED ov;
1421
+ ov.hEvent = 0;
1422
+ ov.Offset = (DWORD)offset;
1423
+ ov.OffsetHigh = HIGH_DWORD(offset);
1424
+
1425
+ DWORD written;
1426
+ if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, &ov)))
1427
+ return (int)GetLastError();
1428
+ if (likely(bytes == written))
1429
+ return MDBX_SUCCESS;
1430
+ #else
1431
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
1432
+ const intptr_t written = pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset);
1433
+ if (likely(bytes == (size_t)written))
1434
+ return MDBX_SUCCESS;
1435
+ if (written < 0) {
1436
+ const int rc = errno;
1437
+ if (rc != EINTR)
1438
+ return rc;
1439
+ continue;
1440
+ }
1441
+ #endif
1442
+ bytes -= written;
1443
+ offset += written;
1444
+ buf = ptr_disp(buf, written);
1445
+ }
1446
+ }
1447
+
1448
+ MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t bytes) {
1449
+ while (true) {
1450
+ #if defined(_WIN32) || defined(_WIN64)
1451
+ DWORD written;
1452
+ if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, nullptr)))
1453
+ return (int)GetLastError();
1454
+ if (likely(bytes == written))
1455
+ return MDBX_SUCCESS;
1456
+ #else
1457
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
1458
+ const intptr_t written = write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE);
1459
+ if (likely(bytes == (size_t)written))
1460
+ return MDBX_SUCCESS;
1461
+ if (written < 0) {
1462
+ const int rc = errno;
1463
+ if (rc != EINTR)
1464
+ return rc;
1465
+ continue;
1466
+ }
1467
+ #endif
1468
+ bytes -= written;
1469
+ buf = ptr_disp(buf, written);
1470
+ }
1471
+ }
1472
+
1473
+ int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset) {
1474
+ size_t expected = 0;
1475
+ for (size_t i = 0; i < sgvcnt; ++i)
1476
+ expected += iov[i].iov_len;
1477
+ #if !MDBX_HAVE_PWRITEV
1478
+ size_t written = 0;
1479
+ for (size_t i = 0; i < sgvcnt; ++i) {
1480
+ int rc = osal_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset);
1481
+ if (unlikely(rc != MDBX_SUCCESS))
1482
+ return rc;
1483
+ written += iov[i].iov_len;
1484
+ offset += iov[i].iov_len;
1485
+ }
1486
+ return (expected == written) ? MDBX_SUCCESS : MDBX_EIO /* ERROR_WRITE_FAULT */;
1487
+ #else
1488
+ int rc;
1489
+ intptr_t written;
1490
+ do {
1491
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
1492
+ written = pwritev(fd, iov, sgvcnt, offset);
1493
+ if (likely(expected == (size_t)written))
1494
+ return MDBX_SUCCESS;
1495
+ rc = errno;
1496
+ } while (rc == EINTR);
1497
+ return (written < 0) ? rc : MDBX_EIO /* Use which error code? */;
1498
+ #endif
1499
+ }
1500
+
1501
+ MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, enum osal_syncmode_bits mode_bits) {
1502
+ #if defined(_WIN32) || defined(_WIN64)
1503
+ if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd))
1504
+ return (int)GetLastError();
1505
+ return MDBX_SUCCESS;
1506
+ #else
1507
+
1508
+ #if defined(__APPLE__) && MDBX_APPLE_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY
1509
+ if (mode_bits & MDBX_SYNC_IODQ)
1510
+ return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno;
1511
+ #endif /* MacOS */
1512
+
1513
+ /* LY: This approach is always safe and without appreciable performance
1514
+ * degradation, even on a kernel with fdatasync's bug.
1515
+ *
1516
+ * For more info about of a corresponding fdatasync() bug
1517
+ * see http://www.spinics.net/lists/linux-ext4/msg33714.html */
1518
+ while (1) {
1519
+ switch (mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_SIZE)) {
1520
+ case MDBX_SYNC_NONE:
1521
+ case MDBX_SYNC_KICK:
1522
+ return MDBX_SUCCESS /* nothing to do */;
1523
+ #if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
1524
+ case MDBX_SYNC_DATA:
1525
+ if (likely(fdatasync(fd) == 0))
1526
+ return MDBX_SUCCESS;
1527
+ break /* error */;
1528
+ #if defined(__linux__) || defined(__gnu_linux__)
1529
+ case MDBX_SYNC_SIZE:
1530
+ assert(globals.linux_kernel_version >= 0x03060000);
1531
+ return MDBX_SUCCESS;
1532
+ #endif /* Linux */
1533
+ #endif /* _POSIX_SYNCHRONIZED_IO > 0 */
1534
+ default:
1535
+ if (likely(fsync(fd) == 0))
1536
+ return MDBX_SUCCESS;
1537
+ }
1538
+
1539
+ int rc = errno;
1540
+ if (rc != EINTR)
1541
+ return rc;
1542
+ }
1543
+ #endif
1544
+ }
1545
+
1546
+ int osal_filesize(mdbx_filehandle_t fd, uint64_t *length) {
1547
+ #if defined(_WIN32) || defined(_WIN64)
1548
+ BY_HANDLE_FILE_INFORMATION info;
1549
+ if (!GetFileInformationByHandle(fd, &info))
1550
+ return (int)GetLastError();
1551
+ *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32;
1552
+ #else
1553
+ struct stat st;
1554
+
1555
+ STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
1556
+ if (fstat(fd, &st))
1557
+ return errno;
1558
+
1559
+ *length = st.st_size;
1560
+ #endif
1561
+ return MDBX_SUCCESS;
1562
+ }
1563
+
1564
+ MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd) {
1565
+ #if defined(_WIN32) || defined(_WIN64)
1566
+ switch (GetFileType(fd)) {
1567
+ case FILE_TYPE_DISK:
1568
+ return MDBX_RESULT_FALSE;
1569
+ case FILE_TYPE_CHAR:
1570
+ case FILE_TYPE_PIPE:
1571
+ return MDBX_RESULT_TRUE;
1572
+ default:
1573
+ return (int)GetLastError();
1574
+ }
1575
+ #else
1576
+ struct stat info;
1577
+ if (fstat(fd, &info))
1578
+ return errno;
1579
+ switch (info.st_mode & S_IFMT) {
1580
+ case S_IFBLK:
1581
+ case S_IFREG:
1582
+ return MDBX_RESULT_FALSE;
1583
+ case S_IFCHR:
1584
+ case S_IFIFO:
1585
+ case S_IFSOCK:
1586
+ return MDBX_RESULT_TRUE;
1587
+ case S_IFDIR:
1588
+ case S_IFLNK:
1589
+ default:
1590
+ return MDBX_INCOMPATIBLE;
1591
+ }
1592
+ #endif
1593
+ }
1594
+
1595
+ MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length) {
1596
+ #if defined(_WIN32) || defined(_WIN64)
1597
+ if (imports.SetFileInformationByHandle) {
1598
+ FILE_END_OF_FILE_INFO EndOfFileInfo;
1599
+ EndOfFileInfo.EndOfFile.QuadPart = length;
1600
+ return imports.SetFileInformationByHandle(fd, FileEndOfFileInfo, &EndOfFileInfo, sizeof(FILE_END_OF_FILE_INFO))
1601
+ ? MDBX_SUCCESS
1602
+ : (int)GetLastError();
1603
+ } else {
1604
+ LARGE_INTEGER li;
1605
+ li.QuadPart = length;
1606
+ return (SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) && SetEndOfFile(fd)) ? MDBX_SUCCESS : (int)GetLastError();
1607
+ }
1608
+ #else
1609
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
1610
+ return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno;
1611
+ #endif
1612
+ }
1613
+
1614
+ MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos) {
1615
+ #if defined(_WIN32) || defined(_WIN64)
1616
+ LARGE_INTEGER li;
1617
+ li.QuadPart = pos;
1618
+ return SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) ? MDBX_SUCCESS : (int)GetLastError();
1619
+ #else
1620
+ STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
1621
+ return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS;
1622
+ #endif
1623
+ }
1624
+
1625
+ /*----------------------------------------------------------------------------*/
1626
+
1627
+ MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *),
1628
+ void *arg) {
1629
+ #if defined(_WIN32) || defined(_WIN64)
1630
+ *thread = CreateThread(nullptr, 0, start_routine, arg, 0, nullptr);
1631
+ return *thread ? MDBX_SUCCESS : (int)GetLastError();
1632
+ #else
1633
+ return pthread_create(thread, nullptr, start_routine, arg);
1634
+ #endif
1635
+ }
1636
+
1637
+ MDBX_INTERNAL int osal_thread_join(osal_thread_t thread) {
1638
+ #if defined(_WIN32) || defined(_WIN64)
1639
+ DWORD code = WaitForSingleObject(thread, INFINITE);
1640
+ return waitstatus2errcode(code);
1641
+ #else
1642
+ void *unused_retval = &unused_retval;
1643
+ return pthread_join(thread, &unused_retval);
1644
+ #endif
1645
+ }
1646
+
1647
+ /*----------------------------------------------------------------------------*/
1648
+
1649
+ MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits) {
1650
+ if (!MDBX_MMAP_NEEDS_JOLT && mode_bits == MDBX_SYNC_NONE)
1651
+ return MDBX_SUCCESS;
1652
+
1653
+ void *ptr = ptr_disp(map->base, offset);
1654
+ #if defined(_WIN32) || defined(_WIN64)
1655
+ if (!FlushViewOfFile(ptr, length))
1656
+ return (int)GetLastError();
1657
+ if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(map->fd))
1658
+ return (int)GetLastError();
1659
+ #else
1660
+ #if defined(__linux__) || defined(__gnu_linux__)
1661
+ /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op. The kernel properly
1662
+ * tracks dirty pages and flushes ones as necessary. */
1663
+ //
1664
+ // However, this behavior may be changed in custom kernels,
1665
+ // so just leave such optimization to the libc discretion.
1666
+ // NOTE: The MDBX_MMAP_NEEDS_JOLT must be defined to 1 for such cases.
1667
+ //
1668
+ // assert(mdbx.linux_kernel_version > 0x02061300);
1669
+ // if (mode_bits <= MDBX_SYNC_KICK)
1670
+ // return MDBX_SUCCESS;
1671
+ #endif /* Linux */
1672
+ if (msync(ptr, length, (mode_bits & MDBX_SYNC_DATA) ? MS_SYNC : MS_ASYNC))
1673
+ return errno;
1674
+ if ((mode_bits & MDBX_SYNC_SIZE) && fsync(map->fd))
1675
+ return errno;
1676
+ #endif
1677
+ return MDBX_SUCCESS;
1678
+ }
1679
+
1680
+ MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err) {
1681
+ #if defined(_WIN32) || defined(_WIN64)
1682
+ (void)pathname;
1683
+ (void)err;
1684
+ if (!imports.GetVolumeInformationByHandleW)
1685
+ return MDBX_ENOSYS;
1686
+ DWORD unused, flags;
1687
+ if (!imports.GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused, &flags, nullptr, 0))
1688
+ return (int)GetLastError();
1689
+ if ((flags & FILE_READ_ONLY_VOLUME) == 0)
1690
+ return MDBX_EACCESS;
1691
+ #else
1692
+ struct statvfs info;
1693
+ if (err != MDBX_ENOFILE) {
1694
+ if (statvfs(pathname, &info) == 0)
1695
+ return (info.f_flag & ST_RDONLY) ? MDBX_SUCCESS : err;
1696
+ if (errno != MDBX_ENOFILE)
1697
+ return errno;
1698
+ }
1699
+ if (fstatvfs(handle, &info))
1700
+ return errno;
1701
+ if ((info.f_flag & ST_RDONLY) == 0)
1702
+ return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err;
1703
+ #endif /* !Windows */
1704
+ return MDBX_SUCCESS;
1705
+ }
1706
+
1707
+ MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle) {
1708
+ #if defined(_WIN32) || defined(_WIN64)
1709
+ (void)handle;
1710
+ #else
1711
+ struct statfs statfs_info;
1712
+ if (fstatfs(handle, &statfs_info))
1713
+ return errno;
1714
+
1715
+ #if defined(__OpenBSD__)
1716
+ const unsigned type = 0;
1717
+ #else
1718
+ const unsigned type = statfs_info.f_type;
1719
+ #endif
1720
+ switch (type) {
1721
+ case 0x28cd3d45 /* CRAMFS_MAGIC */:
1722
+ case 0x858458f6 /* RAMFS_MAGIC */:
1723
+ case 0x01021994 /* TMPFS_MAGIC */:
1724
+ case 0x73717368 /* SQUASHFS_MAGIC */:
1725
+ case 0x7275 /* ROMFS_MAGIC */:
1726
+ return MDBX_RESULT_TRUE;
1727
+ }
1728
+
1729
+ #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \
1730
+ defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \
1731
+ defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN)
1732
+ const char *const name = statfs_info.f_fstypename;
1733
+ const size_t name_len = sizeof(statfs_info.f_fstypename);
1734
+ #else
1735
+ const char *const name = "";
1736
+ const size_t name_len = 0;
1737
+ #endif
1738
+ if (name_len) {
1739
+ if (strncasecmp("tmpfs", name, 6) == 0 || strncasecmp("mfs", name, 4) == 0 || strncasecmp("ramfs", name, 6) == 0 ||
1740
+ strncasecmp("romfs", name, 6) == 0)
1741
+ return MDBX_RESULT_TRUE;
1742
+ }
1743
+ #endif /* !Windows */
1744
+
1745
+ return MDBX_RESULT_FALSE;
1746
+ }
1747
+
1748
+ MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags) {
1749
+ #if defined(_WIN32) || defined(_WIN64)
1750
+ if (globals.running_under_Wine && !(flags & MDBX_EXCLUSIVE))
1751
+ return ERROR_NOT_CAPABLE /* workaround for Wine */;
1752
+
1753
+ if (GetFileType(handle) != FILE_TYPE_DISK)
1754
+ return ERROR_FILE_OFFLINE;
1755
+
1756
+ if (imports.GetFileInformationByHandleEx) {
1757
+ FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo;
1758
+ if (imports.GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, &RemoteProtocolInfo,
1759
+ sizeof(RemoteProtocolInfo))) {
1760
+ if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && !(flags & MDBX_RDONLY))
1761
+ return ERROR_FILE_OFFLINE;
1762
+ if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && !(flags & MDBX_EXCLUSIVE))
1763
+ return MDBX_EREMOTE;
1764
+ }
1765
+ }
1766
+
1767
+ if (imports.NtFsControlFile) {
1768
+ NTSTATUS rc;
1769
+ struct {
1770
+ WOF_EXTERNAL_INFO wof_info;
1771
+ union {
1772
+ WIM_PROVIDER_EXTERNAL_INFO wim_info;
1773
+ FILE_PROVIDER_EXTERNAL_INFO_V1 file_info;
1774
+ };
1775
+ size_t reserved_for_microsoft_madness[42];
1776
+ } GetExternalBacking_OutputBuffer;
1777
+ IO_STATUS_BLOCK StatusBlock;
1778
+ rc = imports.NtFsControlFile(handle, nullptr, nullptr, nullptr, &StatusBlock, FSCTL_GET_EXTERNAL_BACKING, nullptr,
1779
+ 0, &GetExternalBacking_OutputBuffer, sizeof(GetExternalBacking_OutputBuffer));
1780
+ if (NT_SUCCESS(rc)) {
1781
+ if (!(flags & MDBX_EXCLUSIVE))
1782
+ return MDBX_EREMOTE;
1783
+ } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && rc != STATUS_INVALID_DEVICE_REQUEST &&
1784
+ rc != STATUS_NOT_SUPPORTED)
1785
+ return ntstatus2errcode(rc);
1786
+ }
1787
+
1788
+ if (imports.GetVolumeInformationByHandleW && imports.GetFinalPathNameByHandleW) {
1789
+ WCHAR *PathBuffer = osal_malloc(sizeof(WCHAR) * INT16_MAX);
1790
+ if (!PathBuffer)
1791
+ return MDBX_ENOMEM;
1792
+
1793
+ int rc = MDBX_SUCCESS;
1794
+ DWORD VolumeSerialNumber, FileSystemFlags;
1795
+ if (!imports.GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, &VolumeSerialNumber, nullptr,
1796
+ &FileSystemFlags, nullptr, 0)) {
1797
+ rc = (int)GetLastError();
1798
+ goto bailout;
1799
+ }
1800
+
1801
+ if ((flags & MDBX_RDONLY) == 0) {
1802
+ if (FileSystemFlags & (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | FILE_VOLUME_IS_COMPRESSED)) {
1803
+ rc = MDBX_EREMOTE;
1804
+ goto bailout;
1805
+ }
1806
+ }
1807
+
1808
+ if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_NT)) {
1809
+ if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) {
1810
+ if (!(flags & MDBX_EXCLUSIVE)) {
1811
+ rc = MDBX_EREMOTE;
1812
+ goto bailout;
1813
+ }
1814
+ }
1815
+ }
1816
+
1817
+ if (F_ISSET(flags, MDBX_RDONLY | MDBX_EXCLUSIVE) && (FileSystemFlags & FILE_READ_ONLY_VOLUME)) {
1818
+ /* without-LCK (exclusive readonly) mode for DB on a read-only volume */
1819
+ goto bailout;
1820
+ }
1821
+
1822
+ if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS)) {
1823
+ UINT DriveType = GetDriveTypeW(PathBuffer);
1824
+ if (DriveType == DRIVE_NO_ROOT_DIR && _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 &&
1825
+ _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) {
1826
+ PathBuffer[7] = 0;
1827
+ DriveType = GetDriveTypeW(PathBuffer + 4);
1828
+ }
1829
+ switch (DriveType) {
1830
+ case DRIVE_CDROM:
1831
+ if (flags & MDBX_RDONLY)
1832
+ break;
1833
+ // fall through
1834
+ case DRIVE_UNKNOWN:
1835
+ case DRIVE_NO_ROOT_DIR:
1836
+ case DRIVE_REMOTE:
1837
+ default:
1838
+ if (!(flags & MDBX_EXCLUSIVE))
1839
+ rc = MDBX_EREMOTE;
1840
+ // fall through
1841
+ case DRIVE_REMOVABLE:
1842
+ case DRIVE_FIXED:
1843
+ case DRIVE_RAMDISK:
1844
+ break;
1845
+ }
1846
+ }
1847
+
1848
+ bailout:
1849
+ osal_free(PathBuffer);
1850
+ return rc;
1851
+ }
1852
+
1853
+ #else
1854
+
1855
+ struct statvfs statvfs_info;
1856
+ if (fstatvfs(handle, &statvfs_info))
1857
+ return errno;
1858
+ #if defined(ST_LOCAL) || defined(ST_EXPORTED)
1859
+ const unsigned long st_flags = statvfs_info.f_flag;
1860
+ #endif /* ST_LOCAL || ST_EXPORTED */
1861
+
1862
+ #if defined(__NetBSD__)
1863
+ const unsigned type = 0;
1864
+ const char *const name = statvfs_info.f_fstypename;
1865
+ const size_t name_len = VFS_NAMELEN;
1866
+ #elif defined(_AIX) || defined(__OS400__)
1867
+ const char *const name = statvfs_info.f_basetype;
1868
+ const size_t name_len = sizeof(statvfs_info.f_basetype);
1869
+ struct stat st;
1870
+ if (fstat(handle, &st))
1871
+ return errno;
1872
+ const unsigned type = st.st_vfstype;
1873
+ if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE))
1874
+ return MDBX_EREMOTE;
1875
+ #elif defined(FSTYPSZ) || defined(_FSTYPSZ)
1876
+ const unsigned type = 0;
1877
+ const char *const name = statvfs_info.f_basetype;
1878
+ const size_t name_len = sizeof(statvfs_info.f_basetype);
1879
+ #elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ)
1880
+ const unsigned type = 0;
1881
+ struct stat st;
1882
+ if (fstat(handle, &st))
1883
+ return errno;
1884
+ const char *const name = st.st_fstype;
1885
+ const size_t name_len = strlen(name);
1886
+ #else
1887
+ struct statfs statfs_info;
1888
+ if (fstatfs(handle, &statfs_info))
1889
+ return errno;
1890
+ #if defined(__OpenBSD__)
1891
+ const unsigned type = 0;
1892
+ #else
1893
+ const unsigned type = statfs_info.f_type;
1894
+ #endif
1895
+ #if defined(MNT_LOCAL) || defined(MNT_EXPORTED)
1896
+ const unsigned long mnt_flags = statfs_info.f_flags;
1897
+ #endif /* MNT_LOCAL || MNT_EXPORTED */
1898
+ #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \
1899
+ defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \
1900
+ defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN)
1901
+ const char *const name = statfs_info.f_fstypename;
1902
+ const size_t name_len = sizeof(statfs_info.f_fstypename);
1903
+ #elif defined(__ANDROID_API__) && __ANDROID_API__ < 21
1904
+ const char *const name = "";
1905
+ const unsigned name_len = 0;
1906
+ #else
1907
+
1908
+ const char *name = "";
1909
+ unsigned name_len = 0;
1910
+
1911
+ struct stat st;
1912
+ if (fstat(handle, &st))
1913
+ return errno;
1914
+
1915
+ char pathbuf[PATH_MAX];
1916
+ FILE *mounted = nullptr;
1917
+ #if defined(__linux__) || defined(__gnu_linux__)
1918
+ mounted = setmntent("/proc/mounts", "r");
1919
+ #endif /* Linux */
1920
+ if (!mounted)
1921
+ mounted = setmntent("/etc/mtab", "r");
1922
+ if (mounted) {
1923
+ const struct mntent *ent;
1924
+ #if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \
1925
+ (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19))
1926
+ struct mntent entbuf;
1927
+ const bool should_copy = false;
1928
+ while (nullptr != (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf))))
1929
+ #else
1930
+ const bool should_copy = true;
1931
+ while (nullptr != (ent = getmntent(mounted)))
1932
+ #endif
1933
+ {
1934
+ struct stat mnt;
1935
+ if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) {
1936
+ if (should_copy) {
1937
+ name = strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1);
1938
+ pathbuf[name_len] = 0;
1939
+ } else {
1940
+ name = ent->mnt_fsname;
1941
+ name_len = strlen(name);
1942
+ }
1943
+ break;
1944
+ }
1945
+ }
1946
+ endmntent(mounted);
1947
+ }
1948
+ #endif /* !xBSD && !Android/Bionic */
1949
+ #endif
1950
+
1951
+ if (name_len) {
1952
+ if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || strncasecmp("cifs", name, name_len) == 0 ||
1953
+ strncasecmp("ncpfs", name, name_len) == 0 || strncasecmp("smbfs", name, name_len) == 0 ||
1954
+ strcasecmp("9P" /* WSL2 */, name) == 0 ||
1955
+ ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && strncasecmp("fuseblk", name, name_len) != 0)) &&
1956
+ !(flags & MDBX_EXCLUSIVE))
1957
+ return MDBX_EREMOTE;
1958
+ if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || strcasecmp("sshfs", name) == 0)
1959
+ return MDBX_EREMOTE;
1960
+ }
1961
+
1962
+ #ifdef ST_LOCAL
1963
+ if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
1964
+ return MDBX_EREMOTE;
1965
+ #elif defined(MNT_LOCAL)
1966
+ if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
1967
+ return MDBX_EREMOTE;
1968
+ #endif /* ST/MNT_LOCAL */
1969
+
1970
+ #ifdef ST_EXPORTED
1971
+ if ((st_flags & ST_EXPORTED) != 0 && !(flags & (MDBX_RDONLY | MDBX_EXCLUSIVE))))
1972
+ return MDBX_RESULT_TRUE;
1973
+ #elif defined(MNT_EXPORTED)
1974
+ if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)))
1975
+ return MDBX_RESULT_TRUE;
1976
+ #endif /* ST/MNT_EXPORTED */
1977
+
1978
+ switch (type) {
1979
+ case 0xFF534D42 /* CIFS_MAGIC_NUMBER */:
1980
+ case 0x6969 /* NFS_SUPER_MAGIC */:
1981
+ case 0x564c /* NCP_SUPER_MAGIC */:
1982
+ case 0x517B /* SMB_SUPER_MAGIC */:
1983
+ #if defined(__digital__) || defined(__osf__) || defined(__osf)
1984
+ case 0x0E /* Tru64 NFS */:
1985
+ #endif
1986
+ #ifdef ST_FST_NFS
1987
+ case ST_FST_NFS:
1988
+ #endif
1989
+ if ((flags & MDBX_EXCLUSIVE) == 0)
1990
+ return MDBX_EREMOTE;
1991
+ case 0:
1992
+ default:
1993
+ break;
1994
+ }
1995
+ #endif /* Unix */
1996
+
1997
+ return MDBX_SUCCESS;
1998
+ }
1999
+
2000
+ static int check_mmap_limit(const size_t limit) {
2001
+ const bool should_check =
2002
+ #if defined(__SANITIZE_ADDRESS__)
2003
+ true;
2004
+ #else
2005
+ RUNNING_ON_VALGRIND;
2006
+ #endif /* __SANITIZE_ADDRESS__ */
2007
+
2008
+ if (should_check) {
2009
+ intptr_t pagesize, total_ram_pages, avail_ram_pages;
2010
+ int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages);
2011
+ if (unlikely(err != MDBX_SUCCESS))
2012
+ return err;
2013
+
2014
+ const int log2page = log2n_powerof2(pagesize);
2015
+ if ((limit >> (log2page + 7)) > (size_t)total_ram_pages || (limit >> (log2page + 6)) > (size_t)avail_ram_pages) {
2016
+ ERROR("%s (%zu pages) is too large for available (%zu pages) or total "
2017
+ "(%zu pages) system RAM",
2018
+ "database upper size limit", limit >> log2page, avail_ram_pages, total_ram_pages);
2019
+ return MDBX_TOO_LARGE;
2020
+ }
2021
+ }
2022
+
2023
+ return MDBX_SUCCESS;
2024
+ }
2025
+
2026
+ MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options,
2027
+ const pathchar_t *pathname4logging) {
2028
+ assert(size <= limit);
2029
+ map->limit = 0;
2030
+ map->current = 0;
2031
+ map->base = nullptr;
2032
+ map->filesize = 0;
2033
+ #if defined(_WIN32) || defined(_WIN64)
2034
+ map->section = nullptr;
2035
+ #endif /* Windows */
2036
+
2037
+ int err = osal_check_fs_local(map->fd, flags);
2038
+ if (unlikely(err != MDBX_SUCCESS)) {
2039
+ #if defined(_WIN32) || defined(_WIN64)
2040
+ if (globals.running_under_Wine)
2041
+ NOTICE("%s", "Please use native Linux application or WSL at least, instead of trouble-full Wine!");
2042
+ #endif /* Windows */
2043
+ switch (err) {
2044
+ case MDBX_RESULT_TRUE:
2045
+ #if MDBX_ENABLE_NON_READONLY_EXPORT
2046
+ WARNING("%" MDBX_PRIsPATH " is exported via NFS, avoid using the file on a remote side!", pathname4logging);
2047
+ break;
2048
+ #else
2049
+ ERROR("%" MDBX_PRIsPATH " is exported via NFS", pathname4logging);
2050
+ return MDBX_EREMOTE;
2051
+ #endif /* MDBX_PROHIBIT_NON_READONLY_EXPORT */
2052
+ case MDBX_EREMOTE:
2053
+ ERROR("%" MDBX_PRIsPATH " is on a remote file system, the %s is required", pathname4logging, "MDBX_EXCLUSIVE");
2054
+ __fallthrough /* fall through */;
2055
+ default:
2056
+ return err;
2057
+ }
2058
+ }
2059
+
2060
+ err = check_mmap_limit(limit);
2061
+ if (unlikely(err != MDBX_SUCCESS))
2062
+ return err;
2063
+
2064
+ if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_TRUNCATE) != 0) {
2065
+ err = osal_ftruncate(map->fd, size);
2066
+ VERBOSE("ftruncate %zu, err %d", size, err);
2067
+ if (err != MDBX_SUCCESS)
2068
+ return err;
2069
+ map->filesize = size;
2070
+ #if !(defined(_WIN32) || defined(_WIN64))
2071
+ map->current = size;
2072
+ #endif /* !Windows */
2073
+ } else {
2074
+ err = osal_filesize(map->fd, &map->filesize);
2075
+ VERBOSE("filesize %" PRIu64 ", err %d", map->filesize, err);
2076
+ if (err != MDBX_SUCCESS)
2077
+ return err;
2078
+ #if defined(_WIN32) || defined(_WIN64)
2079
+ if (map->filesize < size) {
2080
+ WARNING("file size (%zu) less than requested for mapping (%zu)", (size_t)map->filesize, size);
2081
+ size = (size_t)map->filesize;
2082
+ }
2083
+ #else
2084
+ map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
2085
+ #endif /* !Windows */
2086
+ }
2087
+
2088
+ #if defined(_WIN32) || defined(_WIN64)
2089
+ LARGE_INTEGER SectionSize;
2090
+ SectionSize.QuadPart = size;
2091
+ err = NtCreateSection(&map->section,
2092
+ /* DesiredAccess */
2093
+ (flags & MDBX_WRITEMAP)
2094
+ ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE
2095
+ : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
2096
+ /* ObjectAttributes */ nullptr,
2097
+ /* MaximumSize (InitialSize) */ &SectionSize,
2098
+ /* SectionPageProtection */
2099
+ (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
2100
+ /* AllocationAttributes */ SEC_RESERVE, map->fd);
2101
+ if (!NT_SUCCESS(err))
2102
+ return ntstatus2errcode(err);
2103
+
2104
+ SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 : globals.running_under_Wine ? size : limit;
2105
+ err = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base,
2106
+ /* ZeroBits */ 0,
2107
+ /* CommitSize */ 0,
2108
+ /* SectionOffset */ nullptr, &ViewSize,
2109
+ /* InheritDisposition */ ViewUnmap,
2110
+ /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
2111
+ /* Win32Protect */
2112
+ (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
2113
+ if (!NT_SUCCESS(err)) {
2114
+ NtClose(map->section);
2115
+ map->section = 0;
2116
+ map->base = nullptr;
2117
+ return ntstatus2errcode(err);
2118
+ }
2119
+ assert(map->base != MAP_FAILED);
2120
+
2121
+ map->current = (size_t)SectionSize.QuadPart;
2122
+ map->limit = ViewSize;
2123
+
2124
+ #else /* Windows */
2125
+
2126
+ #ifndef MAP_TRYFIXED
2127
+ #define MAP_TRYFIXED 0
2128
+ #endif
2129
+
2130
+ #ifndef MAP_HASSEMAPHORE
2131
+ #define MAP_HASSEMAPHORE 0
2132
+ #endif
2133
+
2134
+ #ifndef MAP_CONCEAL
2135
+ #define MAP_CONCEAL 0
2136
+ #endif
2137
+
2138
+ #ifndef MAP_NOSYNC
2139
+ #define MAP_NOSYNC 0
2140
+ #endif
2141
+
2142
+ #ifndef MAP_FIXED_NOREPLACE
2143
+ #define MAP_FIXED_NOREPLACE 0
2144
+ #endif
2145
+
2146
+ #ifndef MAP_NORESERVE
2147
+ #define MAP_NORESERVE 0
2148
+ #endif
2149
+
2150
+ map->base = mmap(nullptr, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ,
2151
+ MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) |
2152
+ ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC : MAP_CONCEAL),
2153
+ map->fd, 0);
2154
+
2155
+ if (unlikely(map->base == MAP_FAILED)) {
2156
+ map->limit = 0;
2157
+ map->current = 0;
2158
+ map->base = nullptr;
2159
+ assert(errno != 0);
2160
+ return errno;
2161
+ }
2162
+ map->limit = limit;
2163
+
2164
+ #ifdef MADV_DONTFORK
2165
+ if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0))
2166
+ return errno;
2167
+ #endif /* MADV_DONTFORK */
2168
+ #ifdef MADV_NOHUGEPAGE
2169
+ (void)madvise(map->base, map->limit, MADV_NOHUGEPAGE);
2170
+ #endif /* MADV_NOHUGEPAGE */
2171
+
2172
+ #endif /* ! Windows */
2173
+
2174
+ VALGRIND_MAKE_MEM_DEFINED(map->base, map->current);
2175
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->current);
2176
+ return MDBX_SUCCESS;
2177
+ }
2178
+
2179
+ MDBX_INTERNAL int osal_munmap(osal_mmap_t *map) {
2180
+ VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
2181
+ /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
2182
+ * when this memory will re-used by malloc or another mmapping.
2183
+ * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */
2184
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base,
2185
+ (map->filesize && map->filesize < map->limit) ? map->filesize : map->limit);
2186
+ #if defined(_WIN32) || defined(_WIN64)
2187
+ if (map->section)
2188
+ NtClose(map->section);
2189
+ NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->base);
2190
+ if (!NT_SUCCESS(rc))
2191
+ ntstatus2errcode(rc);
2192
+ #else
2193
+ if (unlikely(munmap(map->base, map->limit))) {
2194
+ assert(errno != 0);
2195
+ return errno;
2196
+ }
2197
+ #endif /* ! Windows */
2198
+
2199
+ map->limit = 0;
2200
+ map->current = 0;
2201
+ map->base = nullptr;
2202
+ return MDBX_SUCCESS;
2203
+ }
2204
+
2205
+ MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit) {
2206
+ int rc = osal_filesize(map->fd, &map->filesize);
2207
+ VERBOSE("flags 0x%x, size %zu, limit %zu, filesize %" PRIu64, flags, size, limit, map->filesize);
2208
+ assert(size <= limit);
2209
+ if (rc != MDBX_SUCCESS) {
2210
+ map->filesize = 0;
2211
+ return rc;
2212
+ }
2213
+
2214
+ #if defined(_WIN32) || defined(_WIN64)
2215
+ assert(size != map->current || limit != map->limit || size < map->filesize);
2216
+
2217
+ NTSTATUS status;
2218
+ LARGE_INTEGER SectionSize;
2219
+ int err;
2220
+
2221
+ if (limit == map->limit && size > map->current) {
2222
+ if ((flags & MDBX_RDONLY) && map->filesize >= size) {
2223
+ map->current = size;
2224
+ return MDBX_SUCCESS;
2225
+ } else if (!(flags & MDBX_RDONLY) &&
2226
+ /* workaround for Wine */ imports.NtExtendSection) {
2227
+ /* growth rw-section */
2228
+ SectionSize.QuadPart = size;
2229
+ status = imports.NtExtendSection(map->section, &SectionSize);
2230
+ if (!NT_SUCCESS(status))
2231
+ return ntstatus2errcode(status);
2232
+ map->current = size;
2233
+ if (map->filesize < size)
2234
+ map->filesize = size;
2235
+ return MDBX_SUCCESS;
2236
+ }
2237
+ }
2238
+
2239
+ if (limit > map->limit) {
2240
+ err = check_mmap_limit(limit);
2241
+ if (unlikely(err != MDBX_SUCCESS))
2242
+ return err;
2243
+
2244
+ /* check ability of address space for growth before unmap */
2245
+ PVOID BaseAddress = (PBYTE)map->base + map->limit;
2246
+ SIZE_T RegionSize = limit - map->limit;
2247
+ status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, &RegionSize, MEM_RESERVE, PAGE_NOACCESS);
2248
+ if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
2249
+ return MDBX_UNABLE_EXTEND_MAPSIZE;
2250
+ if (!NT_SUCCESS(status))
2251
+ return ntstatus2errcode(status);
2252
+
2253
+ status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, MEM_RELEASE);
2254
+ if (!NT_SUCCESS(status))
2255
+ return ntstatus2errcode(status);
2256
+ }
2257
+
2258
+ /* Windows unable:
2259
+ * - shrink a mapped file;
2260
+ * - change size of mapped view;
2261
+ * - extend read-only mapping;
2262
+ * Therefore we should unmap/map entire section. */
2263
+ if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) {
2264
+ if (size <= map->current && limit == map->limit)
2265
+ return MDBX_SUCCESS;
2266
+ return MDBX_EPERM;
2267
+ }
2268
+
2269
+ /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
2270
+ * when this memory will re-used by malloc or another mmapping.
2271
+ * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */
2272
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->limit);
2273
+ status = NtUnmapViewOfSection(GetCurrentProcess(), map->base);
2274
+ if (!NT_SUCCESS(status))
2275
+ return ntstatus2errcode(status);
2276
+ status = NtClose(map->section);
2277
+ map->section = nullptr;
2278
+ PVOID ReservedAddress = nullptr;
2279
+ SIZE_T ReservedSize = limit;
2280
+
2281
+ if (!NT_SUCCESS(status)) {
2282
+ bailout_ntstatus:
2283
+ err = ntstatus2errcode(status);
2284
+ map->base = nullptr;
2285
+ map->current = map->limit = 0;
2286
+ if (ReservedAddress) {
2287
+ ReservedSize = 0;
2288
+ status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE);
2289
+ assert(NT_SUCCESS(status));
2290
+ (void)status;
2291
+ }
2292
+ return err;
2293
+ }
2294
+
2295
+ retry_file_and_section:
2296
+ /* resizing of the file may take a while,
2297
+ * therefore we reserve address space to avoid occupy it by other threads */
2298
+ ReservedAddress = map->base;
2299
+ status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, &ReservedSize, MEM_RESERVE, PAGE_NOACCESS);
2300
+ if (!NT_SUCCESS(status)) {
2301
+ ReservedAddress = nullptr;
2302
+ if (status != (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
2303
+ goto bailout_ntstatus /* no way to recovery */;
2304
+
2305
+ if (flags & MDBX_MRESIZE_MAY_MOVE)
2306
+ /* the base address could be changed */
2307
+ map->base = nullptr;
2308
+ }
2309
+
2310
+ if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) {
2311
+ err = osal_ftruncate(map->fd, size);
2312
+ if (err == MDBX_SUCCESS)
2313
+ map->filesize = size;
2314
+ /* ignore error, because Windows unable shrink file
2315
+ * that already mapped (by another process) */
2316
+ }
2317
+
2318
+ SectionSize.QuadPart = size;
2319
+ status = NtCreateSection(&map->section,
2320
+ /* DesiredAccess */
2321
+ (flags & MDBX_WRITEMAP)
2322
+ ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE
2323
+ : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
2324
+ /* ObjectAttributes */ nullptr,
2325
+ /* MaximumSize (InitialSize) */ &SectionSize,
2326
+ /* SectionPageProtection */
2327
+ (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
2328
+ /* AllocationAttributes */ SEC_RESERVE, map->fd);
2329
+
2330
+ if (!NT_SUCCESS(status))
2331
+ goto bailout_ntstatus;
2332
+
2333
+ if (ReservedAddress) {
2334
+ /* release reserved address space */
2335
+ ReservedSize = 0;
2336
+ status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE);
2337
+ ReservedAddress = nullptr;
2338
+ if (!NT_SUCCESS(status))
2339
+ goto bailout_ntstatus;
2340
+ }
2341
+
2342
+ retry_mapview:;
2343
+ SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit;
2344
+ status = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base,
2345
+ /* ZeroBits */ 0,
2346
+ /* CommitSize */ 0,
2347
+ /* SectionOffset */ nullptr, &ViewSize,
2348
+ /* InheritDisposition */ ViewUnmap,
2349
+ /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
2350
+ /* Win32Protect */
2351
+ (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
2352
+
2353
+ if (!NT_SUCCESS(status)) {
2354
+ if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && map->base &&
2355
+ (flags & MDBX_MRESIZE_MAY_MOVE) != 0) {
2356
+ /* try remap at another base address */
2357
+ map->base = nullptr;
2358
+ goto retry_mapview;
2359
+ }
2360
+ NtClose(map->section);
2361
+ map->section = nullptr;
2362
+
2363
+ if (map->base && (size != map->current || limit != map->limit)) {
2364
+ /* try remap with previously size and limit,
2365
+ * but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */
2366
+ rc = (limit > map->limit) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM;
2367
+ size = map->current;
2368
+ ReservedSize = limit = map->limit;
2369
+ goto retry_file_and_section;
2370
+ }
2371
+
2372
+ /* no way to recovery */
2373
+ goto bailout_ntstatus;
2374
+ }
2375
+ assert(map->base != MAP_FAILED);
2376
+
2377
+ map->current = (size_t)SectionSize.QuadPart;
2378
+ map->limit = ViewSize;
2379
+
2380
+ #else /* Windows */
2381
+
2382
+ if (flags & MDBX_RDONLY) {
2383
+ if (size > map->filesize)
2384
+ rc = MDBX_UNABLE_EXTEND_MAPSIZE;
2385
+ else if (size < map->filesize && map->filesize > limit)
2386
+ rc = MDBX_EPERM;
2387
+ map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
2388
+ } else {
2389
+ if (size > map->filesize || (size < map->filesize && (flags & txn_shrink_allowed))) {
2390
+ rc = osal_ftruncate(map->fd, size);
2391
+ VERBOSE("ftruncate %zu, err %d", size, rc);
2392
+ if (rc != MDBX_SUCCESS)
2393
+ return rc;
2394
+ map->filesize = size;
2395
+ }
2396
+
2397
+ if (map->current > size) {
2398
+ /* Clearing asan's bitmask for the region which released in shrinking,
2399
+ * since:
2400
+ * - after the shrinking we will get an exception when accessing
2401
+ * this region and (therefore) do not need the help of ASAN.
2402
+ * - this allows us to clear the mask only within the file size
2403
+ * when closing the mapping. */
2404
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr_disp(map->base, size),
2405
+ ((map->current < map->limit) ? map->current : map->limit) - size);
2406
+ }
2407
+ map->current = (size < map->limit) ? size : map->limit;
2408
+ }
2409
+
2410
+ if (limit == map->limit)
2411
+ return rc;
2412
+
2413
+ if (limit < map->limit) {
2414
+ /* unmap an excess at end of mapping. */
2415
+ // coverity[offset_free : FALSE]
2416
+ if (unlikely(munmap(ptr_disp(map->base, limit), map->limit - limit))) {
2417
+ assert(errno != 0);
2418
+ return errno;
2419
+ }
2420
+ map->limit = limit;
2421
+ return rc;
2422
+ }
2423
+
2424
+ int err = check_mmap_limit(limit);
2425
+ if (unlikely(err != MDBX_SUCCESS))
2426
+ return err;
2427
+
2428
+ assert(limit > map->limit);
2429
+ void *ptr = MAP_FAILED;
2430
+
2431
+ #if (defined(__linux__) || defined(__gnu_linux__)) && defined(_GNU_SOURCE)
2432
+ ptr = mremap(map->base, map->limit, limit,
2433
+ #if defined(MREMAP_MAYMOVE)
2434
+ (flags & MDBX_MRESIZE_MAY_MOVE) ? MREMAP_MAYMOVE :
2435
+ #endif /* MREMAP_MAYMOVE */
2436
+ 0);
2437
+ if (ptr == MAP_FAILED) {
2438
+ err = errno;
2439
+ assert(err != 0);
2440
+ switch (err) {
2441
+ default:
2442
+ return err;
2443
+ case 0 /* paranoia */:
2444
+ case EAGAIN:
2445
+ case ENOMEM:
2446
+ return MDBX_UNABLE_EXTEND_MAPSIZE;
2447
+ case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */:
2448
+ break;
2449
+ }
2450
+ }
2451
+ #endif /* Linux & _GNU_SOURCE */
2452
+
2453
+ const unsigned mmap_flags =
2454
+ MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0);
2455
+ const unsigned mmap_prot = (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ;
2456
+
2457
+ if (ptr == MAP_FAILED) {
2458
+ /* Try to mmap additional space beyond the end of mapping. */
2459
+ ptr = mmap(ptr_disp(map->base, map->limit), limit - map->limit, mmap_prot, mmap_flags | MAP_FIXED_NOREPLACE,
2460
+ map->fd, map->limit);
2461
+ if (ptr == ptr_disp(map->base, map->limit))
2462
+ /* успешно прилепили отображение в конец */
2463
+ ptr = map->base;
2464
+ else if (ptr != MAP_FAILED) {
2465
+ /* the desired address is busy, unmap unsuitable one */
2466
+ if (unlikely(munmap(ptr, limit - map->limit))) {
2467
+ assert(errno != 0);
2468
+ return errno;
2469
+ }
2470
+ ptr = MAP_FAILED;
2471
+ } else {
2472
+ err = errno;
2473
+ assert(err != 0);
2474
+ switch (err) {
2475
+ default:
2476
+ return err;
2477
+ case 0 /* paranoia */:
2478
+ case EAGAIN:
2479
+ case ENOMEM:
2480
+ return MDBX_UNABLE_EXTEND_MAPSIZE;
2481
+ case EEXIST: /* address busy */
2482
+ case EINVAL: /* kernel don't support MAP_FIXED_NOREPLACE */
2483
+ break;
2484
+ }
2485
+ }
2486
+ }
2487
+
2488
+ if (ptr == MAP_FAILED) {
2489
+ /* unmap and map again whole region */
2490
+ if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) {
2491
+ /* TODO: Perhaps here it is worth to implement suspend/resume threads
2492
+ * and perform unmap/map as like for Windows. */
2493
+ return MDBX_UNABLE_EXTEND_MAPSIZE;
2494
+ }
2495
+
2496
+ if (unlikely(munmap(map->base, map->limit))) {
2497
+ assert(errno != 0);
2498
+ return errno;
2499
+ }
2500
+
2501
+ // coverity[pass_freed_arg : FALSE]
2502
+ ptr = mmap(map->base, limit, mmap_prot,
2503
+ (flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags
2504
+ : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED),
2505
+ map->fd, 0);
2506
+ if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) &&
2507
+ !(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
2508
+ // coverity[pass_freed_arg : FALSE]
2509
+ ptr = mmap(map->base, limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0);
2510
+
2511
+ if (unlikely(ptr == MAP_FAILED)) {
2512
+ /* try to restore prev mapping */
2513
+ // coverity[pass_freed_arg : FALSE]
2514
+ ptr = mmap(map->base, map->limit, mmap_prot,
2515
+ (flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags
2516
+ : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED),
2517
+ map->fd, 0);
2518
+ if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) &&
2519
+ !(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
2520
+ // coverity[pass_freed_arg : FALSE]
2521
+ ptr = mmap(map->base, map->limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0);
2522
+ if (unlikely(ptr == MAP_FAILED)) {
2523
+ VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
2524
+ /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
2525
+ * when this memory will re-used by malloc or another mmapping.
2526
+ * See
2527
+ * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203
2528
+ */
2529
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit);
2530
+ map->limit = 0;
2531
+ map->current = 0;
2532
+ map->base = nullptr;
2533
+ assert(errno != 0);
2534
+ return errno;
2535
+ }
2536
+ rc = MDBX_UNABLE_EXTEND_MAPSIZE;
2537
+ limit = map->limit;
2538
+ }
2539
+ }
2540
+
2541
+ assert(ptr && ptr != MAP_FAILED);
2542
+ if (map->base != ptr) {
2543
+ VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
2544
+ /* Unpoisoning is required for ASAN to avoid false-positive diagnostic
2545
+ * when this memory will re-used by malloc or another mmapping.
2546
+ * See
2547
+ * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203
2548
+ */
2549
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit);
2550
+
2551
+ VALGRIND_MAKE_MEM_DEFINED(ptr, map->current);
2552
+ MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr, map->current);
2553
+ map->base = ptr;
2554
+ }
2555
+ map->limit = limit;
2556
+ map->current = size;
2557
+
2558
+ #ifdef MADV_DONTFORK
2559
+ if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0)) {
2560
+ assert(errno != 0);
2561
+ return errno;
2562
+ }
2563
+ #endif /* MADV_DONTFORK */
2564
+ #ifdef MADV_NOHUGEPAGE
2565
+ (void)madvise(map->base, map->limit, MADV_NOHUGEPAGE);
2566
+ #endif /* MADV_NOHUGEPAGE */
2567
+
2568
+ #endif /* POSIX / Windows */
2569
+
2570
+ /* Zap: Redundant code */
2571
+ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6287);
2572
+ assert(rc != MDBX_SUCCESS || (map->base != nullptr && map->base != MAP_FAILED && map->current == size &&
2573
+ map->limit == limit && map->filesize >= size));
2574
+ return rc;
2575
+ }
2576
+
2577
+ /*----------------------------------------------------------------------------*/
2578
+
2579
+ __cold MDBX_INTERNAL void osal_jitter(bool tiny) {
2580
+ for (;;) {
2581
+ #if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__)
2582
+ unsigned salt = 5296013u * (unsigned)__rdtsc();
2583
+ salt ^= salt >> 11;
2584
+ salt *= 25810541u;
2585
+ #elif (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT
2586
+ static ULONG state;
2587
+ const unsigned salt = (unsigned)RtlRandomEx(&state);
2588
+ #else
2589
+ const unsigned salt = rand();
2590
+ #endif
2591
+
2592
+ const int coin = salt % (tiny ? 29u : 43u);
2593
+ if (coin < 43 / 3)
2594
+ break;
2595
+ #if defined(_WIN32) || defined(_WIN64)
2596
+ if (coin < 43 * 2 / 3)
2597
+ SwitchToThread();
2598
+ else {
2599
+ static HANDLE timer;
2600
+ if (!timer)
2601
+ timer = CreateWaitableTimer(NULL, TRUE, NULL);
2602
+
2603
+ LARGE_INTEGER ft;
2604
+ ft.QuadPart = coin * (int64_t)-10; // Convert to 100 nanosecond interval,
2605
+ // negative value indicates relative time.
2606
+ SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0);
2607
+ WaitForSingleObject(timer, INFINITE);
2608
+ // CloseHandle(timer);
2609
+ break;
2610
+ }
2611
+ #else
2612
+ sched_yield();
2613
+ if (coin > 43 * 2 / 3)
2614
+ usleep(coin);
2615
+ #endif
2616
+ }
2617
+ }
2618
+
2619
+ /*----------------------------------------------------------------------------*/
2620
+
2621
+ #if defined(_WIN32) || defined(_WIN64)
2622
+ static LARGE_INTEGER performance_frequency;
2623
+ #elif defined(__APPLE__) || defined(__MACH__)
2624
+ #include <mach/mach_time.h>
2625
+ static uint64_t ratio_16dot16_to_monotine;
2626
+ #elif defined(__linux__) || defined(__gnu_linux__)
2627
+ static clockid_t posix_clockid;
2628
+ __cold static clockid_t choice_monoclock(void) {
2629
+ struct timespec probe;
2630
+ #if defined(CLOCK_BOOTTIME)
2631
+ if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0)
2632
+ return CLOCK_BOOTTIME;
2633
+ #elif defined(CLOCK_MONOTONIC_RAW)
2634
+ if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0)
2635
+ return CLOCK_MONOTONIC_RAW;
2636
+ #elif defined(CLOCK_MONOTONIC_COARSE)
2637
+ if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0)
2638
+ return CLOCK_MONOTONIC_COARSE;
2639
+ #endif
2640
+ return CLOCK_MONOTONIC;
2641
+ }
2642
+ #elif defined(CLOCK_MONOTONIC)
2643
+ #define posix_clockid CLOCK_MONOTONIC
2644
+ #else
2645
+ #define posix_clockid CLOCK_REALTIME
2646
+ #endif
2647
+
2648
+ MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16) {
2649
+ #if defined(_WIN32) || defined(_WIN64)
2650
+ const uint64_t ratio = performance_frequency.QuadPart;
2651
+ #elif defined(__APPLE__) || defined(__MACH__)
2652
+ const uint64_t ratio = ratio_16dot16_to_monotine;
2653
+ #else
2654
+ const uint64_t ratio = UINT64_C(1000000000);
2655
+ #endif
2656
+ const uint64_t ret = (ratio * seconds_16dot16 + 32768) >> 16;
2657
+ return likely(ret || seconds_16dot16 == 0) ? ret : /* fix underflow */ 1;
2658
+ }
2659
+
2660
+ static uint64_t monotime_limit;
2661
+ MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime) {
2662
+ if (unlikely(monotime > monotime_limit))
2663
+ return UINT32_MAX;
2664
+
2665
+ const uint32_t ret =
2666
+ #if defined(_WIN32) || defined(_WIN64)
2667
+ (uint32_t)((monotime << 16) / performance_frequency.QuadPart);
2668
+ #elif defined(__APPLE__) || defined(__MACH__)
2669
+ (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine);
2670
+ #else
2671
+ (uint32_t)((monotime << 7) / 1953125);
2672
+ #endif
2673
+ return ret;
2674
+ }
2675
+
2676
+ MDBX_INTERNAL uint64_t osal_monotime(void) {
2677
+ #if defined(_WIN32) || defined(_WIN64)
2678
+ LARGE_INTEGER counter;
2679
+ if (QueryPerformanceCounter(&counter))
2680
+ return counter.QuadPart;
2681
+ #elif defined(__APPLE__) || defined(__MACH__)
2682
+ return mach_absolute_time();
2683
+ #else
2684
+ struct timespec ts;
2685
+ if (likely(clock_gettime(posix_clockid, &ts) == 0))
2686
+ return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
2687
+ #endif
2688
+ return 0;
2689
+ }
2690
+
2691
+ MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults) {
2692
+ #if defined(_WIN32) || defined(_WIN64)
2693
+ if (optional_page_faults) {
2694
+ PROCESS_MEMORY_COUNTERS pmc;
2695
+ *optional_page_faults = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) ? pmc.PageFaultCount : 0;
2696
+ }
2697
+ FILETIME unused, usermode;
2698
+ if (GetThreadTimes(GetCurrentThread(),
2699
+ /* CreationTime */ &unused,
2700
+ /* ExitTime */ &unused,
2701
+ /* KernelTime */ &unused,
2702
+ /* UserTime */ &usermode)) {
2703
+ /* one second = 10_000_000 * 100ns = 78125 * (1 << 7) * 100ns;
2704
+ * result = (h * f / 10_000_000) << 32) + l * f / 10_000_000 =
2705
+ * = ((h * f) >> 7) / 78125) << 32) + ((l * f) >> 7) / 78125;
2706
+ * 1) {h, l} *= f;
2707
+ * 2) {h, l} >>= 7;
2708
+ * 3) result = ((h / 78125) << 32) + l / 78125; */
2709
+ uint64_t l = usermode.dwLowDateTime * performance_frequency.QuadPart;
2710
+ uint64_t h = usermode.dwHighDateTime * performance_frequency.QuadPart;
2711
+ l = h << (64 - 7) | l >> 7;
2712
+ h = h >> 7;
2713
+ return ((h / 78125) << 32) + l / 78125;
2714
+ }
2715
+ #elif defined(RUSAGE_THREAD) || defined(RUSAGE_LWP)
2716
+ #ifndef RUSAGE_THREAD
2717
+ #define RUSAGE_THREAD RUSAGE_LWP /* Solaris */
2718
+ #endif
2719
+ struct rusage usage;
2720
+ if (getrusage(RUSAGE_THREAD, &usage) == 0) {
2721
+ if (optional_page_faults)
2722
+ *optional_page_faults = usage.ru_majflt;
2723
+ return usage.ru_utime.tv_sec * UINT64_C(1000000000) + usage.ru_utime.tv_usec * 1000u;
2724
+ }
2725
+ if (optional_page_faults)
2726
+ *optional_page_faults = 0;
2727
+ #elif defined(CLOCK_THREAD_CPUTIME_ID)
2728
+ if (optional_page_faults)
2729
+ *optional_page_faults = 0;
2730
+ struct timespec ts;
2731
+ if (likely(clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0))
2732
+ return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
2733
+ #else
2734
+ /* FIXME */
2735
+ if (optional_page_faults)
2736
+ *optional_page_faults = 0;
2737
+ #endif
2738
+ return 0;
2739
+ }
2740
+
2741
+ /*----------------------------------------------------------------------------*/
2742
+
2743
+ static void bootid_shake(bin128_t *p) {
2744
+ /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */
2745
+ const uint32_t e = p->a - (p->b << 23 | p->b >> 9);
2746
+ p->a = p->b ^ (p->c << 16 | p->c >> 16);
2747
+ p->b = p->c + (p->d << 11 | p->d >> 21);
2748
+ p->c = p->d + e;
2749
+ p->d = e + p->a;
2750
+ }
2751
+
2752
+ __cold static void bootid_collect(bin128_t *p, const void *s, size_t n) {
2753
+ p->y += UINT64_C(64526882297375213);
2754
+ bootid_shake(p);
2755
+ for (size_t i = 0; i < n; ++i) {
2756
+ bootid_shake(p);
2757
+ p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i];
2758
+ bootid_shake(p);
2759
+ p->y += 14621231;
2760
+ }
2761
+ bootid_shake(p);
2762
+
2763
+ /* minor non-linear tomfoolery */
2764
+ const unsigned z = p->x % 61 + 1;
2765
+ p->y = p->y << z | p->y >> (64 - z);
2766
+ bootid_shake(p);
2767
+ bootid_shake(p);
2768
+ const unsigned q = p->x % 59 + 1;
2769
+ p->y = p->y << q | p->y >> (64 - q);
2770
+ bootid_shake(p);
2771
+ bootid_shake(p);
2772
+ bootid_shake(p);
2773
+ }
2774
+
2775
+ static size_t hamming_weight(size_t v) {
2776
+ const size_t m1 = (size_t)UINT64_C(0x5555555555555555);
2777
+ const size_t m2 = (size_t)UINT64_C(0x3333333333333333);
2778
+ const size_t m4 = (size_t)UINT64_C(0x0f0f0f0f0f0f0f0f);
2779
+ const size_t h01 = (size_t)UINT64_C(0x0101010101010101);
2780
+ v -= (v >> 1) & m1;
2781
+ v = (v & m2) + ((v >> 2) & m2);
2782
+ v = (v + (v >> 4)) & m4;
2783
+ return (v * h01) >> (sizeof(v) * 8 - 8);
2784
+ }
2785
+
2786
+ static inline size_t hw64(uint64_t v) {
2787
+ size_t r = hamming_weight((size_t)v);
2788
+ if (sizeof(v) > sizeof(r))
2789
+ r += hamming_weight((size_t)(v >> sizeof(r) * 4 >> sizeof(r) * 4));
2790
+ return r;
2791
+ }
2792
+
2793
+ static bool check_uuid(bin128_t uuid) {
2794
+ size_t hw = hw64(uuid.x) + hw64(uuid.y) + hw64(uuid.x ^ uuid.y);
2795
+ return (hw >> 6) == 1;
2796
+ }
2797
+
2798
+ #if defined(_WIN32) || defined(_WIN64)
2799
+
2800
+ __cold static uint64_t windows_systemtime_ms() {
2801
+ FILETIME ft;
2802
+ GetSystemTimeAsFileTime(&ft);
2803
+ return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul;
2804
+ }
2805
+
2806
+ __cold static uint64_t windows_bootime(void) {
2807
+ unsigned confirmed = 0;
2808
+ uint64_t boottime = 0;
2809
+ uint64_t up0 = imports.GetTickCount64();
2810
+ uint64_t st0 = windows_systemtime_ms();
2811
+ for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) {
2812
+ YieldProcessor();
2813
+ const uint64_t up1 = imports.GetTickCount64();
2814
+ const uint64_t st1 = windows_systemtime_ms();
2815
+ if (st1 > fuse && st1 == st0 && up1 == up0) {
2816
+ uint64_t diff = st1 - up1;
2817
+ if (boottime == diff) {
2818
+ if (++confirmed > 4)
2819
+ return boottime;
2820
+ } else {
2821
+ confirmed = 0;
2822
+ boottime = diff;
2823
+ }
2824
+ fuse = st1;
2825
+ Sleep(1);
2826
+ }
2827
+ st0 = st1;
2828
+ up0 = up1;
2829
+ }
2830
+ return 0;
2831
+ }
2832
+
2833
+ __cold static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue, PVOID pvData, LPDWORD pcbData) {
2834
+ LSTATUS rc;
2835
+ if (!imports.RegGetValueA) {
2836
+ /* an old Windows 2000/XP */
2837
+ HKEY hSubKey;
2838
+ rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey);
2839
+ if (rc == ERROR_SUCCESS) {
2840
+ rc = RegQueryValueExA(hSubKey, lpValue, nullptr, nullptr, pvData, pcbData);
2841
+ RegCloseKey(hSubKey);
2842
+ }
2843
+ return rc;
2844
+ }
2845
+
2846
+ rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, nullptr, pvData, pcbData);
2847
+ if (rc != ERROR_FILE_NOT_FOUND)
2848
+ return rc;
2849
+
2850
+ rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, nullptr,
2851
+ pvData, pcbData);
2852
+ if (rc != ERROR_FILE_NOT_FOUND)
2853
+ return rc;
2854
+ return imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, nullptr,
2855
+ pvData, pcbData);
2856
+ }
2857
+ #endif
2858
+
2859
+ MDBX_MAYBE_UNUSED __cold static bool bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) {
2860
+ if (n > 31) {
2861
+ unsigned bits = 0;
2862
+ for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ {
2863
+ uint8_t c = ((const uint8_t *)p)[i];
2864
+ if (c >= '0' && c <= '9')
2865
+ c -= '0';
2866
+ else if (c >= 'a' && c <= 'f')
2867
+ c -= 'a' - 10;
2868
+ else if (c >= 'A' && c <= 'F')
2869
+ c -= 'A' - 10;
2870
+ else
2871
+ continue;
2872
+ assert(c <= 15);
2873
+ c ^= s->y >> 60;
2874
+ s->y = s->y << 4 | s->x >> 60;
2875
+ s->x = s->x << 4 | c;
2876
+ bits += 4;
2877
+ }
2878
+ if (bits > 42 * 3)
2879
+ /* UUID parsed successfully */
2880
+ return true;
2881
+ }
2882
+
2883
+ if (n > 15) /* is enough handle it as a binary? */ {
2884
+ if (n == sizeof(bin128_t)) {
2885
+ bin128_t aligned;
2886
+ memcpy(&aligned, p, sizeof(bin128_t));
2887
+ s->x += aligned.x;
2888
+ s->y += aligned.y;
2889
+ } else
2890
+ bootid_collect(s, p, n);
2891
+ return check_uuid(*s);
2892
+ }
2893
+
2894
+ if (n)
2895
+ bootid_collect(s, p, n);
2896
+ return false;
2897
+ }
2898
+
2899
+ #if defined(__linux__) || defined(__gnu_linux__)
2900
+
2901
+ __cold static bool is_inside_lxc(void) {
2902
+ bool inside_lxc = false;
2903
+ FILE *mounted = setmntent("/proc/mounts", "r");
2904
+ if (mounted) {
2905
+ const struct mntent *ent;
2906
+ while (nullptr != (ent = getmntent(mounted))) {
2907
+ if (strcmp(ent->mnt_fsname, "lxcfs") == 0 && strncmp(ent->mnt_dir, "/proc/", 6) == 0) {
2908
+ inside_lxc = true;
2909
+ break;
2910
+ }
2911
+ }
2912
+ endmntent(mounted);
2913
+ }
2914
+ return inside_lxc;
2915
+ }
2916
+
2917
+ __cold static bool proc_read_uuid(const char *path, bin128_t *target) {
2918
+ const int fd = open(path, O_RDONLY | O_NOFOLLOW);
2919
+ if (fd != -1) {
2920
+ struct statfs fs;
2921
+ char buf[42];
2922
+ const ssize_t len = (fstatfs(fd, &fs) == 0 &&
2923
+ (fs.f_type == /* procfs */ 0x9FA0 || (fs.f_type == /* tmpfs */ 0x1021994 && is_inside_lxc())))
2924
+ ? read(fd, buf, sizeof(buf))
2925
+ : -1;
2926
+ const int err = close(fd);
2927
+ assert(err == 0);
2928
+ (void)err;
2929
+ if (len > 0)
2930
+ return bootid_parse_uuid(target, buf, len);
2931
+ }
2932
+ return false;
2933
+ }
2934
+ #endif /* Linux */
2935
+
2936
+ __cold static bin128_t osal_bootid(void) {
2937
+ bin128_t uuid = {{0, 0}};
2938
+ bool got_machineid = false, got_boottime = false, got_bootseq = false;
2939
+
2940
+ #if defined(__linux__) || defined(__gnu_linux__)
2941
+ if (proc_read_uuid("/proc/sys/kernel/random/boot_id", &uuid))
2942
+ return uuid;
2943
+ #endif /* Linux */
2944
+
2945
+ #if defined(__APPLE__) || defined(__MACH__)
2946
+ {
2947
+ char buf[42];
2948
+ size_t len = sizeof(buf);
2949
+ if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && bootid_parse_uuid(&uuid, buf, len))
2950
+ return uuid;
2951
+
2952
+ #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED > 1050
2953
+ uuid_t hostuuid;
2954
+ struct timespec wait = {0, 1000000000u / 42};
2955
+ if (!gethostuuid(hostuuid, &wait))
2956
+ got_machineid = bootid_parse_uuid(&uuid, hostuuid, sizeof(hostuuid));
2957
+ #endif /* > 10.5 */
2958
+
2959
+ struct timeval boottime;
2960
+ len = sizeof(boottime);
2961
+ if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && len == sizeof(boottime) && boottime.tv_sec)
2962
+ got_boottime = true;
2963
+ }
2964
+ #endif /* Apple/Darwin */
2965
+
2966
+ #if defined(_WIN32) || defined(_WIN64)
2967
+ {
2968
+ union buf {
2969
+ DWORD BootId;
2970
+ DWORD BaseTime;
2971
+ SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo;
2972
+ struct {
2973
+ LARGE_INTEGER BootTime;
2974
+ LARGE_INTEGER CurrentTime;
2975
+ LARGE_INTEGER TimeZoneBias;
2976
+ ULONG TimeZoneId;
2977
+ ULONG Reserved;
2978
+ ULONGLONG BootTimeBias;
2979
+ ULONGLONG SleepTimeBias;
2980
+ } SysTimeOfDayInfoHacked;
2981
+ wchar_t MachineGuid[42];
2982
+ char DigitalProductId[248];
2983
+ } buf;
2984
+
2985
+ static const char HKLM_MicrosoftCryptography[] = "SOFTWARE\\Microsoft\\Cryptography";
2986
+ DWORD len = sizeof(buf);
2987
+ /* Windows is madness and must die */
2988
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, "MachineGuid", &buf.MachineGuid, &len) ==
2989
+ ERROR_SUCCESS &&
2990
+ len < sizeof(buf))
2991
+ got_machineid = bootid_parse_uuid(&uuid, &buf.MachineGuid, len);
2992
+
2993
+ if (!got_machineid) {
2994
+ /* again, Windows is madness */
2995
+ static const char HKLM_WindowsNT[] = "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion";
2996
+ static const char HKLM_WindowsNT_DPK[] = "SOFTWARE\\Microsoft\\Windows "
2997
+ "NT\\CurrentVersion\\DefaultProductKey";
2998
+ static const char HKLM_WindowsNT_DPK2[] = "SOFTWARE\\Microsoft\\Windows "
2999
+ "NT\\CurrentVersion\\DefaultProductKey2";
3000
+
3001
+ len = sizeof(buf);
3002
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, "DigitalProductId", &buf.DigitalProductId, &len) ==
3003
+ ERROR_SUCCESS &&
3004
+ len > 42 && len < sizeof(buf)) {
3005
+ bootid_collect(&uuid, &buf.DigitalProductId, len);
3006
+ got_machineid = true;
3007
+ }
3008
+ len = sizeof(buf);
3009
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, "DigitalProductId", &buf.DigitalProductId, &len) ==
3010
+ ERROR_SUCCESS &&
3011
+ len > 42 && len < sizeof(buf)) {
3012
+ bootid_collect(&uuid, &buf.DigitalProductId, len);
3013
+ got_machineid = true;
3014
+ }
3015
+ len = sizeof(buf);
3016
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, "DigitalProductId", &buf.DigitalProductId, &len) ==
3017
+ ERROR_SUCCESS &&
3018
+ len > 42 && len < sizeof(buf)) {
3019
+ bootid_collect(&uuid, &buf.DigitalProductId, len);
3020
+ got_machineid = true;
3021
+ }
3022
+ }
3023
+
3024
+ static const char HKLM_PrefetcherParams[] = "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory "
3025
+ "Management\\PrefetchParameters";
3026
+ len = sizeof(buf);
3027
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", &buf.BootId, &len) == ERROR_SUCCESS &&
3028
+ len > 1 && len < sizeof(buf)) {
3029
+ bootid_collect(&uuid, &buf.BootId, len);
3030
+ got_bootseq = true;
3031
+ }
3032
+
3033
+ len = sizeof(buf);
3034
+ if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", &buf.BaseTime, &len) == ERROR_SUCCESS &&
3035
+ len >= sizeof(buf.BaseTime) && buf.BaseTime) {
3036
+ bootid_collect(&uuid, &buf.BaseTime, len);
3037
+ got_boottime = true;
3038
+ }
3039
+
3040
+ /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */
3041
+ NTSTATUS status = NtQuerySystemInformation(0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo,
3042
+ sizeof(buf.SysTimeOfDayInfo), &len);
3043
+ if (NT_SUCCESS(status) &&
3044
+ len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTimeBias) +
3045
+ sizeof(buf.SysTimeOfDayInfoHacked.BootTimeBias) &&
3046
+ buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) {
3047
+ const uint64_t UnbiasedBootTime =
3048
+ buf.SysTimeOfDayInfoHacked.BootTime.QuadPart - buf.SysTimeOfDayInfoHacked.BootTimeBias;
3049
+ if (UnbiasedBootTime) {
3050
+ bootid_collect(&uuid, &UnbiasedBootTime, sizeof(UnbiasedBootTime));
3051
+ got_boottime = true;
3052
+ }
3053
+ }
3054
+
3055
+ if (!got_boottime) {
3056
+ uint64_t boottime = windows_bootime();
3057
+ if (boottime) {
3058
+ bootid_collect(&uuid, &boottime, sizeof(boottime));
3059
+ got_boottime = true;
3060
+ }
3061
+ }
3062
+ }
3063
+ #endif /* Windows */
3064
+
3065
+ #if defined(CTL_HW) && defined(HW_UUID)
3066
+ if (!got_machineid) {
3067
+ static const int mib[] = {CTL_HW, HW_UUID};
3068
+ char buf[42];
3069
+ size_t len = sizeof(buf);
3070
+ if (sysctl(
3071
+ #ifdef SYSCTL_LEGACY_NONCONST_MIB
3072
+ (int *)
3073
+ #endif
3074
+ mib,
3075
+ ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0)
3076
+ got_machineid = bootid_parse_uuid(&uuid, buf, len);
3077
+ }
3078
+ #endif /* CTL_HW && HW_UUID */
3079
+
3080
+ #if defined(CTL_KERN) && defined(KERN_HOSTUUID)
3081
+ if (!got_machineid) {
3082
+ static const int mib[] = {CTL_KERN, KERN_HOSTUUID};
3083
+ char buf[42];
3084
+ size_t len = sizeof(buf);
3085
+ if (sysctl(
3086
+ #ifdef SYSCTL_LEGACY_NONCONST_MIB
3087
+ (int *)
3088
+ #endif
3089
+ mib,
3090
+ ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0)
3091
+ got_machineid = bootid_parse_uuid(&uuid, buf, len);
3092
+ }
3093
+ #endif /* CTL_KERN && KERN_HOSTUUID */
3094
+
3095
+ #if defined(__NetBSD__)
3096
+ if (!got_machineid) {
3097
+ char buf[42];
3098
+ size_t len = sizeof(buf);
3099
+ if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, nullptr, 0) == 0)
3100
+ got_machineid = bootid_parse_uuid(&uuid, buf, len);
3101
+ }
3102
+ #endif /* __NetBSD__ */
3103
+
3104
+ #if !(defined(_WIN32) || defined(_WIN64))
3105
+ if (!got_machineid) {
3106
+ int fd = open("/etc/machine-id", O_RDONLY);
3107
+ if (fd == -1)
3108
+ fd = open("/var/lib/dbus/machine-id", O_RDONLY);
3109
+ if (fd != -1) {
3110
+ char buf[42];
3111
+ const ssize_t len = read(fd, buf, sizeof(buf));
3112
+ const int err = close(fd);
3113
+ assert(err == 0);
3114
+ (void)err;
3115
+ if (len > 0)
3116
+ got_machineid = bootid_parse_uuid(&uuid, buf, len);
3117
+ }
3118
+ }
3119
+ #endif /* !Windows */
3120
+
3121
+ #if _XOPEN_SOURCE_EXTENDED
3122
+ if (!got_machineid) {
3123
+ const long hostid = gethostid();
3124
+ if (hostid != 0 && hostid != -1) {
3125
+ bootid_collect(&uuid, &hostid, sizeof(hostid));
3126
+ got_machineid = true;
3127
+ }
3128
+ }
3129
+ #endif /* _XOPEN_SOURCE_EXTENDED */
3130
+
3131
+ if (!got_machineid) {
3132
+ lack:
3133
+ uuid.x = uuid.y = 0;
3134
+ return uuid;
3135
+ }
3136
+
3137
+ /*--------------------------------------------------------------------------*/
3138
+
3139
+ #if defined(CTL_KERN) && defined(KERN_BOOTTIME)
3140
+ if (!got_boottime) {
3141
+ static const int mib[] = {CTL_KERN, KERN_BOOTTIME};
3142
+ struct timeval boottime;
3143
+ size_t len = sizeof(boottime);
3144
+ if (sysctl(
3145
+ #ifdef SYSCTL_LEGACY_NONCONST_MIB
3146
+ (int *)
3147
+ #endif
3148
+ mib,
3149
+ ARRAY_LENGTH(mib), &boottime, &len, nullptr, 0) == 0 &&
3150
+ len == sizeof(boottime) && boottime.tv_sec) {
3151
+ bootid_collect(&uuid, &boottime, len);
3152
+ got_boottime = true;
3153
+ }
3154
+ }
3155
+ #endif /* CTL_KERN && KERN_BOOTTIME */
3156
+
3157
+ #if defined(__sun) || defined(__SVR4) || defined(__svr4__)
3158
+ if (!got_boottime) {
3159
+ kstat_ctl_t *kc = kstat_open();
3160
+ if (kc) {
3161
+ kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc");
3162
+ if (kp && kstat_read(kc, kp, 0) != -1) {
3163
+ kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time");
3164
+ if (kn) {
3165
+ switch (kn->data_type) {
3166
+ case KSTAT_DATA_INT32:
3167
+ case KSTAT_DATA_UINT32:
3168
+ bootid_collect(&uuid, &kn->value, sizeof(int32_t));
3169
+ got_boottime = true;
3170
+ case KSTAT_DATA_INT64:
3171
+ case KSTAT_DATA_UINT64:
3172
+ bootid_collect(&uuid, &kn->value, sizeof(int64_t));
3173
+ got_boottime = true;
3174
+ }
3175
+ }
3176
+ }
3177
+ kstat_close(kc);
3178
+ }
3179
+ }
3180
+ #endif /* SunOS / Solaris */
3181
+
3182
+ #if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME)
3183
+ if (!got_boottime) {
3184
+ setutxent();
3185
+ const struct utmpx id = {.ut_type = BOOT_TIME};
3186
+ const struct utmpx *entry = getutxid(&id);
3187
+ if (entry) {
3188
+ bootid_collect(&uuid, entry, sizeof(*entry));
3189
+ got_boottime = true;
3190
+ while (unlikely((entry = getutxid(&id)) != nullptr)) {
3191
+ /* have multiple reboot records, assuming we can distinguish next
3192
+ * bootsession even if RTC is wrong or absent */
3193
+ bootid_collect(&uuid, entry, sizeof(*entry));
3194
+ got_bootseq = true;
3195
+ }
3196
+ }
3197
+ endutxent();
3198
+ }
3199
+ #endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */
3200
+
3201
+ if (!got_bootseq) {
3202
+ if (!got_boottime || !MDBX_TRUST_RTC)
3203
+ goto lack;
3204
+
3205
+ #if defined(_WIN32) || defined(_WIN64)
3206
+ FILETIME now;
3207
+ GetSystemTimeAsFileTime(&now);
3208
+ if (0x1CCCCCC > now.dwHighDateTime)
3209
+ #else
3210
+ struct timespec mono, real;
3211
+ if (clock_gettime(CLOCK_MONOTONIC, &mono) || clock_gettime(CLOCK_REALTIME, &real) ||
3212
+ /* wrong time, RTC is mad or absent */
3213
+ 1555555555l > real.tv_sec ||
3214
+ /* seems no adjustment by RTC/NTP, i.e. a fake time */
3215
+ real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || (real.tv_sec - mono.tv_sec) % 900u == 0)
3216
+ #endif
3217
+ goto lack;
3218
+ }
3219
+
3220
+ return uuid;
3221
+ }
3222
+
3223
+ __cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, intptr_t *avail_pages) {
3224
+ if (!page_size && !total_pages && !avail_pages)
3225
+ return LOG_IFERR(MDBX_EINVAL);
3226
+ if (total_pages)
3227
+ *total_pages = -1;
3228
+ if (avail_pages)
3229
+ *avail_pages = -1;
3230
+
3231
+ const intptr_t pagesize = globals.sys_pagesize;
3232
+ if (page_size)
3233
+ *page_size = pagesize;
3234
+ if (unlikely(pagesize < MDBX_MIN_PAGESIZE || !is_powerof2(pagesize)))
3235
+ return LOG_IFERR(MDBX_INCOMPATIBLE);
3236
+
3237
+ MDBX_MAYBE_UNUSED const int log2page = log2n_powerof2(pagesize);
3238
+ assert(pagesize == (INT64_C(1) << log2page));
3239
+ (void)log2page;
3240
+
3241
+ #if defined(_WIN32) || defined(_WIN64)
3242
+ MEMORYSTATUSEX info;
3243
+ memset(&info, 0, sizeof(info));
3244
+ info.dwLength = sizeof(info);
3245
+ if (!GlobalMemoryStatusEx(&info))
3246
+ return LOG_IFERR((int)GetLastError());
3247
+ #endif
3248
+
3249
+ if (total_pages) {
3250
+ #if defined(_WIN32) || defined(_WIN64)
3251
+ const intptr_t total_ram_pages = (intptr_t)(info.ullTotalPhys >> log2page);
3252
+ #elif defined(_SC_PHYS_PAGES)
3253
+ const intptr_t total_ram_pages = sysconf(_SC_PHYS_PAGES);
3254
+ if (total_ram_pages == -1)
3255
+ return LOG_IFERR(errno);
3256
+ #elif defined(_SC_AIX_REALMEM)
3257
+ const intptr_t total_ram_Kb = sysconf(_SC_AIX_REALMEM);
3258
+ if (total_ram_Kb == -1)
3259
+ return LOG_IFERR(errno);
3260
+ const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page;
3261
+ #elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || defined(HW_PHYSMEM)
3262
+ size_t ram, len = sizeof(ram);
3263
+ static const int mib[] = {CTL_HW,
3264
+ #if defined(HW_USERMEM)
3265
+ HW_USERMEM
3266
+ #elif defined(HW_PHYSMEM64)
3267
+ HW_PHYSMEM64
3268
+ #elif defined(HW_MEMSIZE)
3269
+ HW_MEMSIZE
3270
+ #else
3271
+ HW_PHYSMEM
3272
+ #endif
3273
+ };
3274
+ if (sysctl(
3275
+ #ifdef SYSCTL_LEGACY_NONCONST_MIB
3276
+ (int *)
3277
+ #endif
3278
+ mib,
3279
+ ARRAY_LENGTH(mib), &ram, &len, nullptr, 0) != 0)
3280
+ return LOG_IFERR(errno);
3281
+ if (len != sizeof(ram))
3282
+ return LOG_IFERR(MDBX_ENOSYS);
3283
+ const intptr_t total_ram_pages = (intptr_t)(ram >> log2page);
3284
+ #else
3285
+ #error "FIXME: Get User-accessible or physical RAM"
3286
+ #endif
3287
+ *total_pages = total_ram_pages;
3288
+ if (total_ram_pages < 1)
3289
+ return LOG_IFERR(MDBX_ENOSYS);
3290
+ }
3291
+
3292
+ if (avail_pages) {
3293
+ #if defined(_WIN32) || defined(_WIN64)
3294
+ const intptr_t avail_ram_pages = (intptr_t)(info.ullAvailPhys >> log2page);
3295
+ #elif defined(_SC_AVPHYS_PAGES)
3296
+ const intptr_t avail_ram_pages = sysconf(_SC_AVPHYS_PAGES);
3297
+ if (avail_ram_pages == -1)
3298
+ return LOG_IFERR(errno);
3299
+ #elif defined(__MACH__)
3300
+ mach_msg_type_number_t count = HOST_VM_INFO_COUNT;
3301
+ vm_statistics_data_t vmstat;
3302
+ mach_port_t mport = mach_host_self();
3303
+ kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t)&vmstat, &count);
3304
+ mach_port_deallocate(mach_task_self(), mport);
3305
+ if (unlikely(kerr != KERN_SUCCESS))
3306
+ return LOG_IFERR(MDBX_ENOSYS);
3307
+ const intptr_t avail_ram_pages = vmstat.free_count;
3308
+ #elif defined(VM_TOTAL) || defined(VM_METER)
3309
+ struct vmtotal info;
3310
+ size_t len = sizeof(info);
3311
+ static const int mib[] = {CTL_VM,
3312
+ #if defined(VM_TOTAL)
3313
+ VM_TOTAL
3314
+ #elif defined(VM_METER)
3315
+ VM_METER
3316
+ #endif
3317
+ };
3318
+ if (sysctl(
3319
+ #ifdef SYSCTL_LEGACY_NONCONST_MIB
3320
+ (int *)
3321
+ #endif
3322
+ mib,
3323
+ ARRAY_LENGTH(mib), &info, &len, nullptr, 0) != 0)
3324
+ return LOG_IFERR(errno);
3325
+ if (len != sizeof(info))
3326
+ return LOG_IFERR(MDBX_ENOSYS);
3327
+ const intptr_t avail_ram_pages = info.t_free;
3328
+ #else
3329
+ #error "FIXME: Get Available RAM"
3330
+ #endif
3331
+ *avail_pages = avail_ram_pages;
3332
+ if (avail_ram_pages < 1)
3333
+ return LOG_IFERR(MDBX_ENOSYS);
3334
+ }
3335
+
3336
+ return MDBX_SUCCESS;
3337
+ }
3338
+
3339
+ /*----------------------------------------------------------------------------*/
3340
+
3341
+ #ifdef __FreeBSD__
3342
+ #include <sys/uuid.h>
3343
+ #endif /* FreeBSD */
3344
+
3345
+ #ifdef __IPHONE_OS_VERSION_MIN_REQUIRED
3346
+ #include <CommonCrypto/CommonRandom.h>
3347
+ #elif __GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \
3348
+ defined(__DragonFly__) || defined(__APPLE__) || __has_include(<sys/random.h>)
3349
+ #include <sys/random.h>
3350
+ #endif /* sys/random.h */
3351
+
3352
+ #if defined(_WIN32) || defined(_WIN64)
3353
+ #include <wincrypt.h>
3354
+ #endif /* Windows */
3355
+
3356
+ MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *env) {
3357
+ struct {
3358
+ uint64_t begin, end, cputime;
3359
+ uintptr_t thread, pid;
3360
+ const void *x, *y;
3361
+ bin128_t (*z)(const MDBX_env *env);
3362
+ } salt;
3363
+
3364
+ salt.begin = osal_monotime();
3365
+ bin128_t uuid = {{0, 0}};
3366
+
3367
+ #if defined(__linux__) || defined(__gnu_linux__)
3368
+ if (proc_read_uuid("/proc/sys/kernel/random/uuid", &uuid) && check_uuid(uuid))
3369
+ return uuid;
3370
+ #endif /* Linux */
3371
+
3372
+ #ifdef __FreeBSD__
3373
+ STATIC_ASSERT(sizeof(uuid) == sizeof(struct uuid));
3374
+ if (uuidgen((struct uuid *)&uuid, 1) == 0 && check_uuid(uuid))
3375
+ return uuid;
3376
+ #endif /* FreeBSD */
3377
+
3378
+ #if defined(_WIN32) || defined(_WIN64)
3379
+ if (imports.CoCreateGuid && imports.CoCreateGuid(&uuid) == 0 && check_uuid(uuid))
3380
+ return uuid;
3381
+
3382
+ HCRYPTPROV hCryptProv = 0;
3383
+ if (CryptAcquireContextW(&hCryptProv, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) {
3384
+ const BOOL ok = CryptGenRandom(hCryptProv, sizeof(uuid), (unsigned char *)&uuid);
3385
+ CryptReleaseContext(hCryptProv, 0);
3386
+ if (ok && check_uuid(uuid))
3387
+ return uuid;
3388
+ }
3389
+ #elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && defined(__IPHONE_8_0)
3390
+ #if __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_8_0
3391
+ if (CCRandomGenerateBytes(&uuid, sizeof(uuid)) == kCCSuccess && check_uuid(uuid))
3392
+ return uuid;
3393
+ #endif /* iOS >= 8.x */
3394
+ #else
3395
+ const int fd = open("/dev/urandom", O_RDONLY);
3396
+ if (fd != -1) {
3397
+ const ssize_t len = read(fd, &uuid, sizeof(uuid));
3398
+ const int err = close(fd);
3399
+ assert(err == 0);
3400
+ (void)err;
3401
+ if (len == sizeof(uuid) && check_uuid(uuid))
3402
+ return uuid;
3403
+ }
3404
+ #if (__GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \
3405
+ defined(__DragonFly__)) && \
3406
+ !defined(__APPLE__) && !defined(__ANDROID_API__)
3407
+ if (getrandom(&uuid, sizeof(uuid), 0) == sizeof(uuid) && check_uuid(uuid))
3408
+ return uuid;
3409
+ #elif defined(__OpenBSD__) || (defined(__sun) && defined(__SVR4)) || \
3410
+ (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
3411
+ if (getentropy(&uuid, sizeof(uuid)) == 0 && check_uuid(uuid))
3412
+ return uuid;
3413
+ #endif /* getrandom() / getentropy() */
3414
+ #endif /* !Windows */
3415
+
3416
+ uuid = globals.bootid;
3417
+ bootid_collect(&uuid, env, sizeof(*env));
3418
+ salt.thread = osal_thread_self();
3419
+ salt.pid = osal_getpid();
3420
+ salt.x = &salt;
3421
+ salt.y = env;
3422
+ salt.z = &osal_guid;
3423
+ do {
3424
+ salt.cputime = osal_cputime(nullptr);
3425
+ salt.end = osal_monotime();
3426
+ bootid_collect(&uuid, &salt, sizeof(salt));
3427
+ } while (!check_uuid(uuid));
3428
+ return uuid;
3429
+ }
3430
+
3431
+ /*--------------------------------------------------------------------------*/
3432
+
3433
+ void osal_ctor(void) {
3434
+ #if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX)
3435
+ osal_iov_max = sysconf(_SC_IOV_MAX);
3436
+ if (RUNNING_ON_VALGRIND && osal_iov_max > 64)
3437
+ /* чтобы не описывать все 1024 исключения в valgrind_suppress.txt */
3438
+ osal_iov_max = 64;
3439
+ #endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */
3440
+
3441
+ #if defined(_WIN32) || defined(_WIN64)
3442
+ SYSTEM_INFO si;
3443
+ GetSystemInfo(&si);
3444
+ globals.sys_pagesize = si.dwPageSize;
3445
+ globals.sys_allocation_granularity = si.dwAllocationGranularity;
3446
+ #else
3447
+ globals.sys_pagesize = sysconf(_SC_PAGE_SIZE);
3448
+ globals.sys_allocation_granularity = (MDBX_WORDBITS > 32) ? 65536 : 4096;
3449
+ globals.sys_allocation_granularity = (globals.sys_allocation_granularity > globals.sys_pagesize)
3450
+ ? globals.sys_allocation_granularity
3451
+ : globals.sys_pagesize;
3452
+ #endif
3453
+ assert(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0);
3454
+ assert(globals.sys_allocation_granularity >= globals.sys_pagesize &&
3455
+ globals.sys_allocation_granularity % globals.sys_pagesize == 0);
3456
+ globals.sys_pagesize_ln2 = log2n_powerof2(globals.sys_pagesize);
3457
+
3458
+ #if defined(__linux__) || defined(__gnu_linux__)
3459
+ posix_clockid = choice_monoclock();
3460
+ #endif
3461
+
3462
+ #if defined(_WIN32) || defined(_WIN64)
3463
+ QueryPerformanceFrequency(&performance_frequency);
3464
+ #elif defined(__APPLE__) || defined(__MACH__)
3465
+ mach_timebase_info_data_t ti;
3466
+ mach_timebase_info(&ti);
3467
+ ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer;
3468
+ #endif
3469
+ monotime_limit = osal_16dot16_to_monotime(UINT32_MAX - 1);
3470
+
3471
+ uint32_t proba = UINT32_MAX;
3472
+ while (true) {
3473
+ unsigned time_conversion_checkup = osal_monotime_to_16dot16(osal_16dot16_to_monotime(proba));
3474
+ unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba;
3475
+ unsigned one_less = (proba > 0) ? proba - 1 : proba;
3476
+ ENSURE(nullptr, time_conversion_checkup >= one_less && time_conversion_checkup <= one_more);
3477
+ if (proba == 0)
3478
+ break;
3479
+ proba >>= 1;
3480
+ }
3481
+
3482
+ globals.bootid = osal_bootid();
3483
+ }
3484
+
3485
+ void osal_dtor(void) {}