mdbxmou 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +32 -0
- package/.github/workflows/publish.yml +27 -0
- package/.gitmodules +3 -0
- package/CMakeLists.txt +53 -0
- package/LICENSE +201 -0
- package/README.md +639 -0
- package/build.js +11 -0
- package/deps/libmdbx/.clang-format +3 -0
- package/deps/libmdbx/.cmake-format.yaml +3 -0
- package/deps/libmdbx/.le.ini +40 -0
- package/deps/libmdbx/CMakeLists.txt +1269 -0
- package/deps/libmdbx/COPYRIGHT +159 -0
- package/deps/libmdbx/ChangeLog.md +2786 -0
- package/deps/libmdbx/GNUmakefile +950 -0
- package/deps/libmdbx/LICENSE +177 -0
- package/deps/libmdbx/Makefile +16 -0
- package/deps/libmdbx/NOTICE +39 -0
- package/deps/libmdbx/README.md +863 -0
- package/deps/libmdbx/TODO.md +43 -0
- package/deps/libmdbx/cmake/compiler.cmake +1221 -0
- package/deps/libmdbx/cmake/profile.cmake +58 -0
- package/deps/libmdbx/cmake/utils.cmake +524 -0
- package/deps/libmdbx/conanfile.py +323 -0
- package/deps/libmdbx/docs/Doxyfile.in +2734 -0
- package/deps/libmdbx/docs/_preface.md +47 -0
- package/deps/libmdbx/docs/_restrictions.md +248 -0
- package/deps/libmdbx/docs/_starting.md +245 -0
- package/deps/libmdbx/docs/_toc.md +34 -0
- package/deps/libmdbx/docs/header.html +96 -0
- package/deps/libmdbx/example/CMakeLists.txt +6 -0
- package/deps/libmdbx/example/README.md +1 -0
- package/deps/libmdbx/example/example-mdbx.c +154 -0
- package/deps/libmdbx/example/sample-bdb.txt +77 -0
- package/deps/libmdbx/mdbx.h +6655 -0
- package/deps/libmdbx/mdbx.h++ +6428 -0
- package/deps/libmdbx/packages/buildroot/0001-package-libmdbx-new-package-library-database.patch +173 -0
- package/deps/libmdbx/src/alloy.c +54 -0
- package/deps/libmdbx/src/api-cold.c +543 -0
- package/deps/libmdbx/src/api-copy.c +912 -0
- package/deps/libmdbx/src/api-cursor.c +754 -0
- package/deps/libmdbx/src/api-dbi.c +315 -0
- package/deps/libmdbx/src/api-env.c +1434 -0
- package/deps/libmdbx/src/api-extra.c +165 -0
- package/deps/libmdbx/src/api-key-transform.c +197 -0
- package/deps/libmdbx/src/api-misc.c +286 -0
- package/deps/libmdbx/src/api-opts.c +575 -0
- package/deps/libmdbx/src/api-range-estimate.c +365 -0
- package/deps/libmdbx/src/api-txn-data.c +454 -0
- package/deps/libmdbx/src/api-txn.c +921 -0
- package/deps/libmdbx/src/atomics-ops.h +364 -0
- package/deps/libmdbx/src/atomics-types.h +97 -0
- package/deps/libmdbx/src/audit.c +109 -0
- package/deps/libmdbx/src/bits.md +34 -0
- package/deps/libmdbx/src/chk.c +1796 -0
- package/deps/libmdbx/src/cogs.c +309 -0
- package/deps/libmdbx/src/cogs.h +506 -0
- package/deps/libmdbx/src/coherency.c +170 -0
- package/deps/libmdbx/src/config.h.in +88 -0
- package/deps/libmdbx/src/cursor.c +2396 -0
- package/deps/libmdbx/src/cursor.h +391 -0
- package/deps/libmdbx/src/dbi.c +717 -0
- package/deps/libmdbx/src/dbi.h +142 -0
- package/deps/libmdbx/src/debug_begin.h +36 -0
- package/deps/libmdbx/src/debug_end.h +15 -0
- package/deps/libmdbx/src/dpl.c +486 -0
- package/deps/libmdbx/src/dpl.h +134 -0
- package/deps/libmdbx/src/dxb.c +1335 -0
- package/deps/libmdbx/src/env.c +607 -0
- package/deps/libmdbx/src/essentials.h +125 -0
- package/deps/libmdbx/src/gc-get.c +1345 -0
- package/deps/libmdbx/src/gc-put.c +970 -0
- package/deps/libmdbx/src/gc.h +40 -0
- package/deps/libmdbx/src/global.c +474 -0
- package/deps/libmdbx/src/internals.h +585 -0
- package/deps/libmdbx/src/layout-dxb.h +288 -0
- package/deps/libmdbx/src/layout-lck.h +289 -0
- package/deps/libmdbx/src/lck-posix.c +859 -0
- package/deps/libmdbx/src/lck-windows.c +607 -0
- package/deps/libmdbx/src/lck.c +174 -0
- package/deps/libmdbx/src/lck.h +110 -0
- package/deps/libmdbx/src/logging_and_debug.c +250 -0
- package/deps/libmdbx/src/logging_and_debug.h +159 -0
- package/deps/libmdbx/src/man1/mdbx_chk.1 +106 -0
- package/deps/libmdbx/src/man1/mdbx_copy.1 +95 -0
- package/deps/libmdbx/src/man1/mdbx_drop.1 +48 -0
- package/deps/libmdbx/src/man1/mdbx_dump.1 +101 -0
- package/deps/libmdbx/src/man1/mdbx_load.1 +105 -0
- package/deps/libmdbx/src/man1/mdbx_stat.1 +86 -0
- package/deps/libmdbx/src/mdbx.c++ +1837 -0
- package/deps/libmdbx/src/meta.c +656 -0
- package/deps/libmdbx/src/meta.h +168 -0
- package/deps/libmdbx/src/mvcc-readers.c +414 -0
- package/deps/libmdbx/src/node.c +365 -0
- package/deps/libmdbx/src/node.h +102 -0
- package/deps/libmdbx/src/ntdll.def +1246 -0
- package/deps/libmdbx/src/options.h +534 -0
- package/deps/libmdbx/src/osal.c +3485 -0
- package/deps/libmdbx/src/osal.h +587 -0
- package/deps/libmdbx/src/page-get.c +483 -0
- package/deps/libmdbx/src/page-iov.c +185 -0
- package/deps/libmdbx/src/page-iov.h +34 -0
- package/deps/libmdbx/src/page-ops.c +744 -0
- package/deps/libmdbx/src/page-ops.h +142 -0
- package/deps/libmdbx/src/pnl.c +236 -0
- package/deps/libmdbx/src/pnl.h +146 -0
- package/deps/libmdbx/src/preface.h +990 -0
- package/deps/libmdbx/src/proto.h +105 -0
- package/deps/libmdbx/src/refund.c +212 -0
- package/deps/libmdbx/src/sort.h +484 -0
- package/deps/libmdbx/src/spill.c +431 -0
- package/deps/libmdbx/src/spill.h +74 -0
- package/deps/libmdbx/src/table.c +107 -0
- package/deps/libmdbx/src/tls.c +551 -0
- package/deps/libmdbx/src/tls.h +43 -0
- package/deps/libmdbx/src/tools/chk.c +673 -0
- package/deps/libmdbx/src/tools/copy.c +166 -0
- package/deps/libmdbx/src/tools/drop.c +199 -0
- package/deps/libmdbx/src/tools/dump.c +515 -0
- package/deps/libmdbx/src/tools/load.c +831 -0
- package/deps/libmdbx/src/tools/stat.c +516 -0
- package/deps/libmdbx/src/tools/wingetopt.c +87 -0
- package/deps/libmdbx/src/tools/wingetopt.h +30 -0
- package/deps/libmdbx/src/tree-ops.c +1554 -0
- package/deps/libmdbx/src/tree-search.c +140 -0
- package/deps/libmdbx/src/txl.c +99 -0
- package/deps/libmdbx/src/txl.h +26 -0
- package/deps/libmdbx/src/txn.c +1083 -0
- package/deps/libmdbx/src/unaligned.h +205 -0
- package/deps/libmdbx/src/utils.c +32 -0
- package/deps/libmdbx/src/utils.h +76 -0
- package/deps/libmdbx/src/version.c.in +44 -0
- package/deps/libmdbx/src/walk.c +290 -0
- package/deps/libmdbx/src/walk.h +20 -0
- package/deps/libmdbx/src/windows-import.c +152 -0
- package/deps/libmdbx/src/windows-import.h +128 -0
- package/deps/libmdbx/test/CMakeLists.txt +317 -0
- package/deps/libmdbx/test/append.c++ +237 -0
- package/deps/libmdbx/test/base.h++ +92 -0
- package/deps/libmdbx/test/battery-tmux.sh +64 -0
- package/deps/libmdbx/test/cases.c++ +118 -0
- package/deps/libmdbx/test/chrono.c++ +134 -0
- package/deps/libmdbx/test/chrono.h++ +85 -0
- package/deps/libmdbx/test/config.c++ +643 -0
- package/deps/libmdbx/test/config.h++ +334 -0
- package/deps/libmdbx/test/copy.c++ +62 -0
- package/deps/libmdbx/test/dead.c++ +39 -0
- package/deps/libmdbx/test/dump-load.sh +40 -0
- package/deps/libmdbx/test/extra/crunched_delete.c++ +409 -0
- package/deps/libmdbx/test/extra/cursor_closing.c++ +410 -0
- package/deps/libmdbx/test/extra/dbi.c++ +229 -0
- package/deps/libmdbx/test/extra/doubtless_positioning.c++ +253 -0
- package/deps/libmdbx/test/extra/dupfix_addodd.c +94 -0
- package/deps/libmdbx/test/extra/dupfix_multiple.c++ +311 -0
- package/deps/libmdbx/test/extra/early_close_dbi.c++ +137 -0
- package/deps/libmdbx/test/extra/hex_base64_base58.c++ +118 -0
- package/deps/libmdbx/test/extra/maindb_ordinal.c++ +61 -0
- package/deps/libmdbx/test/extra/open.c++ +96 -0
- package/deps/libmdbx/test/extra/pcrf/README.md +2 -0
- package/deps/libmdbx/test/extra/pcrf/pcrf_test.c +380 -0
- package/deps/libmdbx/test/extra/probe.c++ +10 -0
- package/deps/libmdbx/test/extra/txn.c++ +407 -0
- package/deps/libmdbx/test/extra/upsert_alldups.c +193 -0
- package/deps/libmdbx/test/fork.c++ +263 -0
- package/deps/libmdbx/test/hill.c++ +447 -0
- package/deps/libmdbx/test/jitter.c++ +197 -0
- package/deps/libmdbx/test/keygen.c++ +393 -0
- package/deps/libmdbx/test/keygen.h++ +130 -0
- package/deps/libmdbx/test/log.c++ +358 -0
- package/deps/libmdbx/test/log.h++ +91 -0
- package/deps/libmdbx/test/main.c++ +706 -0
- package/deps/libmdbx/test/nested.c++ +318 -0
- package/deps/libmdbx/test/osal-unix.c++ +647 -0
- package/deps/libmdbx/test/osal-windows.c++ +440 -0
- package/deps/libmdbx/test/osal.h++ +41 -0
- package/deps/libmdbx/test/stochastic.sh +690 -0
- package/deps/libmdbx/test/stub/LICENSE +24 -0
- package/deps/libmdbx/test/stub/README.md +8 -0
- package/deps/libmdbx/test/stub/pthread_barrier.c +104 -0
- package/deps/libmdbx/test/stub/pthread_barrier.h +77 -0
- package/deps/libmdbx/test/test.c++ +1551 -0
- package/deps/libmdbx/test/test.h++ +298 -0
- package/deps/libmdbx/test/tmux.conf +3 -0
- package/deps/libmdbx/test/try.c++ +30 -0
- package/deps/libmdbx/test/ttl.c++ +240 -0
- package/deps/libmdbx/test/utils.c++ +203 -0
- package/deps/libmdbx/test/utils.h++ +326 -0
- package/deps/libmdbx/test/valgrind_suppress.txt +536 -0
- package/lib/mdbx_evn_async.js +211 -0
- package/lib/mdbx_worker.js +195 -0
- package/lib/nativemou.js +6 -0
- package/package.json +38 -0
- package/src/async/envmou_close.cpp +34 -0
- package/src/async/envmou_close.hpp +32 -0
- package/src/async/envmou_copy_to.cpp +29 -0
- package/src/async/envmou_copy_to.hpp +38 -0
- package/src/async/envmou_keys.cpp +201 -0
- package/src/async/envmou_keys.hpp +50 -0
- package/src/async/envmou_open.cpp +38 -0
- package/src/async/envmou_open.hpp +33 -0
- package/src/async/envmou_query.cpp +167 -0
- package/src/async/envmou_query.hpp +53 -0
- package/src/dbimou.cpp +522 -0
- package/src/dbimou.hpp +82 -0
- package/src/env_arg0.hpp +24 -0
- package/src/envmou.cpp +445 -0
- package/src/envmou.hpp +116 -0
- package/src/modulemou.cpp +113 -0
- package/src/querymou.cpp +177 -0
- package/src/querymou.hpp +93 -0
- package/src/txnmou.cpp +254 -0
- package/src/txnmou.hpp +122 -0
- package/src/typemou.hpp +239 -0
- package/src/valuemou.hpp +194 -0
- package/test/async.js +67 -0
- package/test/e3.js +38 -0
- package/test/e4.js +89 -0
- package/test/e5.js +162 -0
- package/test/test-batch-ops.js +243 -0
- package/test/test-cursor-mode.js +84 -0
- package/test/test-multi-mode.js +87 -0
|
@@ -0,0 +1,3485 @@
|
|
|
1
|
+
/// \copyright SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
/// \author Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru> \date 2015-2025
|
|
3
|
+
///
|
|
4
|
+
/// https://en.wikipedia.org/wiki/Operating_system_abstraction_layer
|
|
5
|
+
|
|
6
|
+
#include "internals.h"
|
|
7
|
+
|
|
8
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
9
|
+
|
|
10
|
+
#include <psapi.h>
|
|
11
|
+
#include <winioctl.h>
|
|
12
|
+
|
|
13
|
+
#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG)
|
|
14
|
+
#include <crtdbg.h>
|
|
15
|
+
#endif
|
|
16
|
+
|
|
17
|
+
static int waitstatus2errcode(DWORD result) {
|
|
18
|
+
switch (result) {
|
|
19
|
+
case WAIT_OBJECT_0:
|
|
20
|
+
return MDBX_SUCCESS;
|
|
21
|
+
case WAIT_FAILED:
|
|
22
|
+
return (int)GetLastError();
|
|
23
|
+
case WAIT_ABANDONED:
|
|
24
|
+
return ERROR_ABANDONED_WAIT_0;
|
|
25
|
+
case WAIT_IO_COMPLETION:
|
|
26
|
+
return ERROR_USER_APC;
|
|
27
|
+
case WAIT_TIMEOUT:
|
|
28
|
+
return ERROR_TIMEOUT;
|
|
29
|
+
default:
|
|
30
|
+
return ERROR_UNHANDLED_ERROR;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/* Map a result from an NTAPI call to WIN32 error code. */
|
|
35
|
+
static int ntstatus2errcode(NTSTATUS status) {
|
|
36
|
+
DWORD dummy;
|
|
37
|
+
OVERLAPPED ov;
|
|
38
|
+
memset(&ov, 0, sizeof(ov));
|
|
39
|
+
ov.Internal = status;
|
|
40
|
+
/* Zap: '_Param_(1)' could be '0' */
|
|
41
|
+
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6387);
|
|
42
|
+
return GetOverlappedResult(nullptr, &ov, &dummy, FALSE) ? MDBX_SUCCESS : (int)GetLastError();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/* We use native NT APIs to setup the memory map, so that we can
|
|
46
|
+
* let the DB file grow incrementally instead of always preallocating
|
|
47
|
+
* the full size. These APIs are defined in <wdm.h> and <ntifs.h>
|
|
48
|
+
* but those headers are meant for driver-level development and
|
|
49
|
+
* conflict with the regular user-level headers, so we explicitly
|
|
50
|
+
* declare them here. Using these APIs also means we must link to
|
|
51
|
+
* ntdll.dll, which is not linked by default in user code. */
|
|
52
|
+
|
|
53
|
+
extern NTSTATUS NTAPI NtCreateSection(OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess,
|
|
54
|
+
IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes,
|
|
55
|
+
IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection,
|
|
56
|
+
IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle);
|
|
57
|
+
|
|
58
|
+
typedef struct _SECTION_BASIC_INFORMATION {
|
|
59
|
+
ULONG Unknown;
|
|
60
|
+
ULONG SectionAttributes;
|
|
61
|
+
LARGE_INTEGER SectionSize;
|
|
62
|
+
} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION;
|
|
63
|
+
|
|
64
|
+
extern NTSTATUS NTAPI NtMapViewOfSection(IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress,
|
|
65
|
+
IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize,
|
|
66
|
+
IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize,
|
|
67
|
+
IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType,
|
|
68
|
+
IN ULONG Win32Protect);
|
|
69
|
+
|
|
70
|
+
extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, IN OPTIONAL PVOID BaseAddress);
|
|
71
|
+
|
|
72
|
+
/* Zap: Inconsistent annotation for 'NtClose'... */
|
|
73
|
+
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(28251)
|
|
74
|
+
extern NTSTATUS NTAPI NtClose(HANDLE Handle);
|
|
75
|
+
|
|
76
|
+
extern NTSTATUS NTAPI NtAllocateVirtualMemory(IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits,
|
|
77
|
+
IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect);
|
|
78
|
+
|
|
79
|
+
extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, IN PVOID *BaseAddress, IN OUT PSIZE_T RegionSize,
|
|
80
|
+
IN ULONG FreeType);
|
|
81
|
+
|
|
82
|
+
#ifndef WOF_CURRENT_VERSION
|
|
83
|
+
typedef struct _WOF_EXTERNAL_INFO {
|
|
84
|
+
DWORD Version;
|
|
85
|
+
DWORD Provider;
|
|
86
|
+
} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO;
|
|
87
|
+
#endif /* WOF_CURRENT_VERSION */
|
|
88
|
+
|
|
89
|
+
#ifndef WIM_PROVIDER_CURRENT_VERSION
|
|
90
|
+
#define WIM_PROVIDER_HASH_SIZE 20
|
|
91
|
+
|
|
92
|
+
typedef struct _WIM_PROVIDER_EXTERNAL_INFO {
|
|
93
|
+
DWORD Version;
|
|
94
|
+
DWORD Flags;
|
|
95
|
+
LARGE_INTEGER DataSourceId;
|
|
96
|
+
BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE];
|
|
97
|
+
} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO;
|
|
98
|
+
#endif /* WIM_PROVIDER_CURRENT_VERSION */
|
|
99
|
+
|
|
100
|
+
#ifndef FILE_PROVIDER_CURRENT_VERSION
|
|
101
|
+
typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 {
|
|
102
|
+
ULONG Version;
|
|
103
|
+
ULONG Algorithm;
|
|
104
|
+
ULONG Flags;
|
|
105
|
+
} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1;
|
|
106
|
+
#endif /* FILE_PROVIDER_CURRENT_VERSION */
|
|
107
|
+
|
|
108
|
+
#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED
|
|
109
|
+
#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL)
|
|
110
|
+
#endif
|
|
111
|
+
#ifndef STATUS_INVALID_DEVICE_REQUEST
|
|
112
|
+
#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L)
|
|
113
|
+
#endif
|
|
114
|
+
#ifndef STATUS_NOT_SUPPORTED
|
|
115
|
+
#define STATUS_NOT_SUPPORTED ((NTSTATUS)0xC00000BBL)
|
|
116
|
+
#endif
|
|
117
|
+
|
|
118
|
+
#ifndef FILE_DEVICE_FILE_SYSTEM
|
|
119
|
+
#define FILE_DEVICE_FILE_SYSTEM 0x00000009
|
|
120
|
+
#endif
|
|
121
|
+
|
|
122
|
+
#ifndef FSCTL_GET_EXTERNAL_BACKING
|
|
123
|
+
#define FSCTL_GET_EXTERNAL_BACKING CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS)
|
|
124
|
+
#endif
|
|
125
|
+
|
|
126
|
+
#ifndef ERROR_NOT_CAPABLE
|
|
127
|
+
#define ERROR_NOT_CAPABLE 775L
|
|
128
|
+
#endif
|
|
129
|
+
|
|
130
|
+
#endif /* _WIN32 || _WIN64 */
|
|
131
|
+
|
|
132
|
+
/*----------------------------------------------------------------------------*/
|
|
133
|
+
|
|
134
|
+
#if defined(__ANDROID_API__)
|
|
135
|
+
__extern_C void __assert2(const char *file, int line, const char *function, const char *msg) __noreturn;
|
|
136
|
+
#define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion)
|
|
137
|
+
|
|
138
|
+
#elif defined(__UCLIBC__)
|
|
139
|
+
MDBX_NORETURN __extern_C void __assert(const char *, const char *, unsigned, const char *)
|
|
140
|
+
#ifdef __THROW
|
|
141
|
+
__THROW
|
|
142
|
+
#else
|
|
143
|
+
__nothrow
|
|
144
|
+
#endif /* __THROW */
|
|
145
|
+
;
|
|
146
|
+
#define __assert_fail(assertion, file, line, function) __assert(assertion, file, line, function)
|
|
147
|
+
|
|
148
|
+
#elif _POSIX_C_SOURCE > 200212 && \
|
|
149
|
+
/* workaround for avoid musl libc wrong prototype */ (defined(__GLIBC__) || defined(__GNU_LIBRARY__))
|
|
150
|
+
/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */
|
|
151
|
+
MDBX_NORETURN __extern_C void __assert_fail(const char *assertion, const char *file, unsigned line,
|
|
152
|
+
const char *function)
|
|
153
|
+
#ifdef __THROW
|
|
154
|
+
__THROW
|
|
155
|
+
#else
|
|
156
|
+
__nothrow
|
|
157
|
+
#endif /* __THROW */
|
|
158
|
+
;
|
|
159
|
+
|
|
160
|
+
#elif defined(__APPLE__) || defined(__MACH__)
|
|
161
|
+
__extern_C void __assert_rtn(const char *function, const char *file, int line, const char *assertion) /* __nothrow */
|
|
162
|
+
#ifdef __dead2
|
|
163
|
+
__dead2
|
|
164
|
+
#else
|
|
165
|
+
MDBX_NORETURN
|
|
166
|
+
#endif /* __dead2 */
|
|
167
|
+
#ifdef __disable_tail_calls
|
|
168
|
+
__disable_tail_calls
|
|
169
|
+
#endif /* __disable_tail_calls */
|
|
170
|
+
;
|
|
171
|
+
|
|
172
|
+
#define __assert_fail(assertion, file, line, function) __assert_rtn(function, file, line, assertion)
|
|
173
|
+
#elif defined(__sun) || defined(__SVR4) || defined(__svr4__)
|
|
174
|
+
MDBX_NORETURN __extern_C void __assert_c99(const char *assection, const char *file, int line, const char *function);
|
|
175
|
+
#define __assert_fail(assertion, file, line, function) __assert_c99(assertion, file, line, function)
|
|
176
|
+
#elif defined(__OpenBSD__)
|
|
177
|
+
__extern_C __dead void __assert2(const char *file, int line, const char *function,
|
|
178
|
+
const char *assertion) /* __nothrow */;
|
|
179
|
+
#define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion)
|
|
180
|
+
#elif defined(__NetBSD__)
|
|
181
|
+
__extern_C __dead void __assert13(const char *file, int line, const char *function,
|
|
182
|
+
const char *assertion) /* __nothrow */;
|
|
183
|
+
#define __assert_fail(assertion, file, line, function) __assert13(file, line, function, assertion)
|
|
184
|
+
#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__)
|
|
185
|
+
__extern_C void __assert(const char *function, const char *file, int line, const char *assertion) /* __nothrow */
|
|
186
|
+
#ifdef __dead2
|
|
187
|
+
__dead2
|
|
188
|
+
#else
|
|
189
|
+
MDBX_NORETURN
|
|
190
|
+
#endif /* __dead2 */
|
|
191
|
+
#ifdef __disable_tail_calls
|
|
192
|
+
__disable_tail_calls
|
|
193
|
+
#endif /* __disable_tail_calls */
|
|
194
|
+
;
|
|
195
|
+
#define __assert_fail(assertion, file, line, function) __assert(function, file, line, assertion)
|
|
196
|
+
|
|
197
|
+
#endif /* __assert_fail */
|
|
198
|
+
|
|
199
|
+
__cold void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, unsigned line) {
|
|
200
|
+
#if MDBX_DEBUG
|
|
201
|
+
if (env && env->assert_func)
|
|
202
|
+
env->assert_func(env, msg, func, line);
|
|
203
|
+
#else
|
|
204
|
+
(void)env;
|
|
205
|
+
assert_fail(msg, func, line);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line) {
|
|
209
|
+
#endif /* MDBX_DEBUG */
|
|
210
|
+
|
|
211
|
+
if (globals.logger.ptr)
|
|
212
|
+
debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg);
|
|
213
|
+
else {
|
|
214
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
215
|
+
char *message = nullptr;
|
|
216
|
+
const int num = osal_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", msg, func ? func : "unknown", line);
|
|
217
|
+
if (num < 1 || !message)
|
|
218
|
+
message = "<troubles with assertion-message preparation>";
|
|
219
|
+
OutputDebugStringA(message);
|
|
220
|
+
#else
|
|
221
|
+
__assert_fail(msg, "mdbx", line, func);
|
|
222
|
+
#endif
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
while (1) {
|
|
226
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
227
|
+
#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG)
|
|
228
|
+
_CrtDbgReport(_CRT_ASSERT, func ? func : "unknown", line, "libmdbx", "assertion failed: %s", msg);
|
|
229
|
+
#else
|
|
230
|
+
if (IsDebuggerPresent())
|
|
231
|
+
DebugBreak();
|
|
232
|
+
#endif
|
|
233
|
+
FatalExit(STATUS_ASSERTION_FAILURE);
|
|
234
|
+
#else
|
|
235
|
+
abort();
|
|
236
|
+
#endif
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
__cold void mdbx_panic(const char *fmt, ...) {
|
|
241
|
+
va_list ap;
|
|
242
|
+
va_start(ap, fmt);
|
|
243
|
+
|
|
244
|
+
char *message = nullptr;
|
|
245
|
+
const int num = osal_vasprintf(&message, fmt, ap);
|
|
246
|
+
va_end(ap);
|
|
247
|
+
const char *const const_message =
|
|
248
|
+
unlikely(num < 1 || !message) ? "<troubles with panic-message preparation>" : message;
|
|
249
|
+
|
|
250
|
+
if (globals.logger.ptr)
|
|
251
|
+
debug_log(MDBX_LOG_FATAL, "panic", 0, "%s", const_message);
|
|
252
|
+
|
|
253
|
+
while (1) {
|
|
254
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
255
|
+
#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG)
|
|
256
|
+
_CrtDbgReport(_CRT_ASSERT, "mdbx.c", 0, "libmdbx", "panic: %s", const_message);
|
|
257
|
+
#else
|
|
258
|
+
OutputDebugStringA("\r\nMDBX-PANIC: ");
|
|
259
|
+
OutputDebugStringA(const_message);
|
|
260
|
+
if (IsDebuggerPresent())
|
|
261
|
+
DebugBreak();
|
|
262
|
+
#endif
|
|
263
|
+
FatalExit(ERROR_UNHANDLED_ERROR);
|
|
264
|
+
#else
|
|
265
|
+
__assert_fail(const_message, "mdbx", 0, "panic");
|
|
266
|
+
abort();
|
|
267
|
+
#endif
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/*----------------------------------------------------------------------------*/
|
|
272
|
+
|
|
273
|
+
#ifndef osal_vasprintf
|
|
274
|
+
MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap) {
|
|
275
|
+
va_list ones;
|
|
276
|
+
va_copy(ones, ap);
|
|
277
|
+
const int needed = vsnprintf(nullptr, 0, fmt, ones);
|
|
278
|
+
va_end(ones);
|
|
279
|
+
|
|
280
|
+
if (unlikely(needed < 0 || needed >= INT_MAX)) {
|
|
281
|
+
*strp = nullptr;
|
|
282
|
+
return needed;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
*strp = osal_malloc(needed + (size_t)1);
|
|
286
|
+
if (unlikely(*strp == nullptr)) {
|
|
287
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
288
|
+
SetLastError(MDBX_ENOMEM);
|
|
289
|
+
#else
|
|
290
|
+
errno = MDBX_ENOMEM;
|
|
291
|
+
#endif
|
|
292
|
+
return -1;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const int actual = vsnprintf(*strp, needed + (size_t)1, fmt, ap);
|
|
296
|
+
assert(actual == needed);
|
|
297
|
+
if (unlikely(actual < 0)) {
|
|
298
|
+
osal_free(*strp);
|
|
299
|
+
*strp = nullptr;
|
|
300
|
+
}
|
|
301
|
+
return actual;
|
|
302
|
+
}
|
|
303
|
+
#endif /* osal_vasprintf */
|
|
304
|
+
|
|
305
|
+
#ifndef osal_asprintf
|
|
306
|
+
MDBX_INTERNAL int osal_asprintf(char **strp, const char *fmt, ...) {
|
|
307
|
+
va_list ap;
|
|
308
|
+
va_start(ap, fmt);
|
|
309
|
+
const int rc = osal_vasprintf(strp, fmt, ap);
|
|
310
|
+
va_end(ap);
|
|
311
|
+
return rc;
|
|
312
|
+
}
|
|
313
|
+
#endif /* osal_asprintf */
|
|
314
|
+
|
|
315
|
+
#ifndef osal_memalign_alloc
|
|
316
|
+
MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result) {
|
|
317
|
+
assert(is_powerof2(alignment) && alignment >= sizeof(void *));
|
|
318
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
319
|
+
(void)alignment;
|
|
320
|
+
*result = VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
|
|
321
|
+
return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */;
|
|
322
|
+
#elif defined(_ISOC11_SOURCE)
|
|
323
|
+
*result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment));
|
|
324
|
+
return *result ? MDBX_SUCCESS : errno;
|
|
325
|
+
#elif _POSIX_VERSION >= 200112L && (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17)
|
|
326
|
+
*result = nullptr;
|
|
327
|
+
return posix_memalign(result, alignment, bytes);
|
|
328
|
+
#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L
|
|
329
|
+
*result = memalign(alignment, bytes);
|
|
330
|
+
return *result ? MDBX_SUCCESS : errno;
|
|
331
|
+
#else
|
|
332
|
+
#error FIXME
|
|
333
|
+
#endif
|
|
334
|
+
}
|
|
335
|
+
#endif /* osal_memalign_alloc */
|
|
336
|
+
|
|
337
|
+
#ifndef osal_memalign_free
|
|
338
|
+
MDBX_INTERNAL void osal_memalign_free(void *ptr) {
|
|
339
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
340
|
+
VirtualFree(ptr, 0, MEM_RELEASE);
|
|
341
|
+
#else
|
|
342
|
+
osal_free(ptr);
|
|
343
|
+
#endif
|
|
344
|
+
}
|
|
345
|
+
#endif /* osal_memalign_free */
|
|
346
|
+
|
|
347
|
+
#ifndef osal_strdup
|
|
348
|
+
char *osal_strdup(const char *str) {
|
|
349
|
+
if (!str)
|
|
350
|
+
return nullptr;
|
|
351
|
+
size_t bytes = strlen(str) + 1;
|
|
352
|
+
char *dup = osal_malloc(bytes);
|
|
353
|
+
if (dup)
|
|
354
|
+
memcpy(dup, str, bytes);
|
|
355
|
+
return dup;
|
|
356
|
+
}
|
|
357
|
+
#endif /* osal_strdup */
|
|
358
|
+
|
|
359
|
+
/*----------------------------------------------------------------------------*/
|
|
360
|
+
|
|
361
|
+
MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair) {
|
|
362
|
+
int rc;
|
|
363
|
+
memset(condpair, 0, sizeof(osal_condpair_t));
|
|
364
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
365
|
+
if (!(condpair->mutex = CreateMutexW(nullptr, FALSE, nullptr))) {
|
|
366
|
+
rc = (int)GetLastError();
|
|
367
|
+
goto bailout_mutex;
|
|
368
|
+
}
|
|
369
|
+
if (!(condpair->event[0] = CreateEventW(nullptr, FALSE, FALSE, nullptr))) {
|
|
370
|
+
rc = (int)GetLastError();
|
|
371
|
+
goto bailout_event;
|
|
372
|
+
}
|
|
373
|
+
if ((condpair->event[1] = CreateEventW(nullptr, FALSE, FALSE, nullptr)))
|
|
374
|
+
return MDBX_SUCCESS;
|
|
375
|
+
|
|
376
|
+
rc = (int)GetLastError();
|
|
377
|
+
(void)CloseHandle(condpair->event[0]);
|
|
378
|
+
bailout_event:
|
|
379
|
+
(void)CloseHandle(condpair->mutex);
|
|
380
|
+
#else
|
|
381
|
+
rc = pthread_mutex_init(&condpair->mutex, nullptr);
|
|
382
|
+
if (unlikely(rc != 0))
|
|
383
|
+
goto bailout_mutex;
|
|
384
|
+
rc = pthread_cond_init(&condpair->cond[0], nullptr);
|
|
385
|
+
if (unlikely(rc != 0))
|
|
386
|
+
goto bailout_cond;
|
|
387
|
+
rc = pthread_cond_init(&condpair->cond[1], nullptr);
|
|
388
|
+
if (likely(rc == 0))
|
|
389
|
+
return MDBX_SUCCESS;
|
|
390
|
+
|
|
391
|
+
(void)pthread_cond_destroy(&condpair->cond[0]);
|
|
392
|
+
bailout_cond:
|
|
393
|
+
(void)pthread_mutex_destroy(&condpair->mutex);
|
|
394
|
+
#endif
|
|
395
|
+
bailout_mutex:
|
|
396
|
+
memset(condpair, 0, sizeof(osal_condpair_t));
|
|
397
|
+
return rc;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair) {
|
|
401
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
402
|
+
int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
|
|
403
|
+
rc = CloseHandle(condpair->event[0]) ? rc : (int)GetLastError();
|
|
404
|
+
rc = CloseHandle(condpair->event[1]) ? rc : (int)GetLastError();
|
|
405
|
+
#else
|
|
406
|
+
int err, rc = pthread_mutex_destroy(&condpair->mutex);
|
|
407
|
+
rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc;
|
|
408
|
+
rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc;
|
|
409
|
+
#endif
|
|
410
|
+
memset(condpair, 0, sizeof(osal_condpair_t));
|
|
411
|
+
return rc;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair) {
|
|
415
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
416
|
+
DWORD code = WaitForSingleObject(condpair->mutex, INFINITE);
|
|
417
|
+
return waitstatus2errcode(code);
|
|
418
|
+
#else
|
|
419
|
+
return osal_pthread_mutex_lock(&condpair->mutex);
|
|
420
|
+
#endif
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair) {
|
|
424
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
425
|
+
return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError();
|
|
426
|
+
#else
|
|
427
|
+
return pthread_mutex_unlock(&condpair->mutex);
|
|
428
|
+
#endif
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part) {
|
|
432
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
433
|
+
return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : (int)GetLastError();
|
|
434
|
+
#else
|
|
435
|
+
return pthread_cond_signal(&condpair->cond[part]);
|
|
436
|
+
#endif
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part) {
|
|
440
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
441
|
+
DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], INFINITE, FALSE);
|
|
442
|
+
if (code == WAIT_OBJECT_0) {
|
|
443
|
+
code = WaitForSingleObject(condpair->mutex, INFINITE);
|
|
444
|
+
if (code == WAIT_OBJECT_0)
|
|
445
|
+
return MDBX_SUCCESS;
|
|
446
|
+
}
|
|
447
|
+
return waitstatus2errcode(code);
|
|
448
|
+
#else
|
|
449
|
+
return pthread_cond_wait(&condpair->cond[part], &condpair->mutex);
|
|
450
|
+
#endif
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/*----------------------------------------------------------------------------*/
|
|
454
|
+
|
|
455
|
+
MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex) {
|
|
456
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
457
|
+
InitializeCriticalSection(fastmutex);
|
|
458
|
+
return MDBX_SUCCESS;
|
|
459
|
+
#elif MDBX_DEBUG
|
|
460
|
+
pthread_mutexattr_t ma;
|
|
461
|
+
int rc = pthread_mutexattr_init(&ma);
|
|
462
|
+
if (likely(!rc)) {
|
|
463
|
+
rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
|
|
464
|
+
if (likely(!rc) || rc == ENOTSUP)
|
|
465
|
+
rc = pthread_mutex_init(fastmutex, &ma);
|
|
466
|
+
pthread_mutexattr_destroy(&ma);
|
|
467
|
+
}
|
|
468
|
+
return rc;
|
|
469
|
+
#else
|
|
470
|
+
return pthread_mutex_init(fastmutex, nullptr);
|
|
471
|
+
#endif
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex) {
|
|
475
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
476
|
+
DeleteCriticalSection(fastmutex);
|
|
477
|
+
return MDBX_SUCCESS;
|
|
478
|
+
#else
|
|
479
|
+
return pthread_mutex_destroy(fastmutex);
|
|
480
|
+
#endif
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex) {
|
|
484
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
485
|
+
__try {
|
|
486
|
+
EnterCriticalSection(fastmutex);
|
|
487
|
+
} __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
|
|
488
|
+
? EXCEPTION_EXECUTE_HANDLER
|
|
489
|
+
: EXCEPTION_CONTINUE_SEARCH) {
|
|
490
|
+
return MDBX_EDEADLK;
|
|
491
|
+
}
|
|
492
|
+
return MDBX_SUCCESS;
|
|
493
|
+
#else
|
|
494
|
+
return osal_pthread_mutex_lock(fastmutex);
|
|
495
|
+
#endif
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex) {
|
|
499
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
500
|
+
LeaveCriticalSection(fastmutex);
|
|
501
|
+
return MDBX_SUCCESS;
|
|
502
|
+
#else
|
|
503
|
+
return pthread_mutex_unlock(fastmutex);
|
|
504
|
+
#endif
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/*----------------------------------------------------------------------------*/
|
|
508
|
+
|
|
509
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
510
|
+
|
|
511
|
+
MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst) {
|
|
512
|
+
const size_t dst_wlen = MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, nullptr, 0);
|
|
513
|
+
wchar_t *dst = *pdst;
|
|
514
|
+
int rc = ERROR_INVALID_NAME;
|
|
515
|
+
if (unlikely(dst_wlen < 2 || dst_wlen > /* MAX_PATH */ INT16_MAX))
|
|
516
|
+
goto bailout;
|
|
517
|
+
|
|
518
|
+
dst = osal_realloc(dst, dst_wlen * sizeof(wchar_t));
|
|
519
|
+
rc = MDBX_ENOMEM;
|
|
520
|
+
if (unlikely(!dst))
|
|
521
|
+
goto bailout;
|
|
522
|
+
|
|
523
|
+
*pdst = dst;
|
|
524
|
+
if (likely(dst_wlen == (size_t)MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, dst, (int)dst_wlen)))
|
|
525
|
+
return MDBX_SUCCESS;
|
|
526
|
+
|
|
527
|
+
rc = ERROR_INVALID_NAME;
|
|
528
|
+
bailout:
|
|
529
|
+
if (*pdst) {
|
|
530
|
+
osal_free(*pdst);
|
|
531
|
+
*pdst = nullptr;
|
|
532
|
+
}
|
|
533
|
+
return rc;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
#endif /* Windows */
|
|
537
|
+
|
|
538
|
+
/*----------------------------------------------------------------------------*/
|
|
539
|
+
|
|
540
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
541
|
+
#define ior_alignment_mask (ior->pagesize - 1)
|
|
542
|
+
#define ior_WriteFile_flag 1
|
|
543
|
+
#define OSAL_IOV_MAX (4096 / sizeof(ior_sgv_element))
|
|
544
|
+
|
|
545
|
+
static void ior_put_event(osal_ioring_t *ior, HANDLE event) {
|
|
546
|
+
assert(event && event != INVALID_HANDLE_VALUE && event != ior);
|
|
547
|
+
assert(ior->event_stack < ior->allocated);
|
|
548
|
+
ior->event_pool[ior->event_stack] = event;
|
|
549
|
+
ior->event_stack += 1;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
static HANDLE ior_get_event(osal_ioring_t *ior) {
|
|
553
|
+
assert(ior->event_stack <= ior->allocated);
|
|
554
|
+
if (ior->event_stack > 0) {
|
|
555
|
+
ior->event_stack -= 1;
|
|
556
|
+
assert(ior->event_pool[ior->event_stack] != 0);
|
|
557
|
+
return ior->event_pool[ior->event_stack];
|
|
558
|
+
}
|
|
559
|
+
return CreateEventW(nullptr, true, false, nullptr);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
static void WINAPI ior_wocr(DWORD err, DWORD bytes, OVERLAPPED *ov) {
|
|
563
|
+
osal_ioring_t *ior = ov->hEvent;
|
|
564
|
+
ov->Internal = err;
|
|
565
|
+
ov->InternalHigh = bytes;
|
|
566
|
+
if (++ior->async_completed >= ior->async_waiting)
|
|
567
|
+
SetEvent(ior->async_done);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
#elif MDBX_HAVE_PWRITEV
|
|
571
|
+
#if defined(_SC_IOV_MAX)
|
|
572
|
+
static size_t osal_iov_max;
|
|
573
|
+
#define OSAL_IOV_MAX osal_iov_max
|
|
574
|
+
#else
|
|
575
|
+
#define OSAL_IOV_MAX IOV_MAX
|
|
576
|
+
#endif
|
|
577
|
+
#else
|
|
578
|
+
#undef OSAL_IOV_MAX
|
|
579
|
+
#endif /* OSAL_IOV_MAX */
|
|
580
|
+
|
|
581
|
+
MDBX_INTERNAL int osal_ioring_create(osal_ioring_t *ior
|
|
582
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
583
|
+
,
|
|
584
|
+
bool enable_direct, mdbx_filehandle_t overlapped_fd
|
|
585
|
+
#endif /* Windows */
|
|
586
|
+
) {
|
|
587
|
+
memset(ior, 0, sizeof(osal_ioring_t));
|
|
588
|
+
|
|
589
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
590
|
+
ior->overlapped_fd = overlapped_fd;
|
|
591
|
+
ior->direct = enable_direct && overlapped_fd;
|
|
592
|
+
ior->pagesize = globals.sys_pagesize;
|
|
593
|
+
ior->pagesize_ln2 = globals.sys_pagesize_ln2;
|
|
594
|
+
ior->async_done = ior_get_event(ior);
|
|
595
|
+
if (!ior->async_done)
|
|
596
|
+
return GetLastError();
|
|
597
|
+
#endif /* !Windows */
|
|
598
|
+
|
|
599
|
+
#if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX)
|
|
600
|
+
assert(osal_iov_max > 0);
|
|
601
|
+
#endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */
|
|
602
|
+
|
|
603
|
+
ior->boundary = ptr_disp(ior->pool, ior->allocated);
|
|
604
|
+
return MDBX_SUCCESS;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
static inline size_t ior_offset(const ior_item_t *item) {
|
|
608
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
609
|
+
return item->ov.Offset |
|
|
610
|
+
(size_t)((sizeof(size_t) > sizeof(item->ov.Offset)) ? (uint64_t)item->ov.OffsetHigh << 32 : 0);
|
|
611
|
+
#else
|
|
612
|
+
return item->offset;
|
|
613
|
+
#endif /* !Windows */
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
static inline ior_item_t *ior_next(ior_item_t *item, size_t sgvcnt) {
|
|
617
|
+
#if defined(ior_sgv_element)
|
|
618
|
+
assert(sgvcnt > 0);
|
|
619
|
+
return (ior_item_t *)ptr_disp(item, sizeof(ior_item_t) - sizeof(ior_sgv_element) + sizeof(ior_sgv_element) * sgvcnt);
|
|
620
|
+
#else
|
|
621
|
+
assert(sgvcnt == 1);
|
|
622
|
+
(void)sgvcnt;
|
|
623
|
+
return item + 1;
|
|
624
|
+
#endif
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, void *data, const size_t bytes) {
|
|
628
|
+
assert(bytes && data);
|
|
629
|
+
assert(bytes % MDBX_MIN_PAGESIZE == 0 && bytes <= MAX_WRITE);
|
|
630
|
+
assert(offset % MDBX_MIN_PAGESIZE == 0 && offset + (uint64_t)bytes <= MAX_MAPSIZE);
|
|
631
|
+
|
|
632
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
633
|
+
const unsigned segments = (unsigned)(bytes >> ior->pagesize_ln2);
|
|
634
|
+
const bool use_gather = ior->direct && ior->overlapped_fd && ior->slots_left >= segments;
|
|
635
|
+
#endif /* Windows */
|
|
636
|
+
|
|
637
|
+
ior_item_t *item = ior->pool;
|
|
638
|
+
if (likely(ior->last)) {
|
|
639
|
+
item = ior->last;
|
|
640
|
+
if (unlikely(ior_offset(item) + ior_last_bytes(ior, item) == offset) &&
|
|
641
|
+
likely(ior_last_bytes(ior, item) + bytes <= MAX_WRITE)) {
|
|
642
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
643
|
+
if (use_gather &&
|
|
644
|
+
((bytes | (uintptr_t)data | ior->last_bytes | (uintptr_t)(uint64_t)item->sgv[0].Buffer) &
|
|
645
|
+
ior_alignment_mask) == 0 &&
|
|
646
|
+
ior->last_sgvcnt + (size_t)segments < OSAL_IOV_MAX) {
|
|
647
|
+
assert(ior->overlapped_fd);
|
|
648
|
+
assert((item->single.iov_len & ior_WriteFile_flag) == 0);
|
|
649
|
+
assert(item->sgv[ior->last_sgvcnt].Buffer == 0);
|
|
650
|
+
ior->last_bytes += bytes;
|
|
651
|
+
size_t i = 0;
|
|
652
|
+
do {
|
|
653
|
+
item->sgv[ior->last_sgvcnt + i].Buffer = PtrToPtr64(data);
|
|
654
|
+
data = ptr_disp(data, ior->pagesize);
|
|
655
|
+
} while (++i < segments);
|
|
656
|
+
ior->slots_left -= segments;
|
|
657
|
+
item->sgv[ior->last_sgvcnt += segments].Buffer = 0;
|
|
658
|
+
assert((item->single.iov_len & ior_WriteFile_flag) == 0);
|
|
659
|
+
return MDBX_SUCCESS;
|
|
660
|
+
}
|
|
661
|
+
const void *end = ptr_disp(item->single.iov_base, item->single.iov_len - ior_WriteFile_flag);
|
|
662
|
+
if (unlikely(end == data)) {
|
|
663
|
+
assert((item->single.iov_len & ior_WriteFile_flag) != 0);
|
|
664
|
+
item->single.iov_len += bytes;
|
|
665
|
+
return MDBX_SUCCESS;
|
|
666
|
+
}
|
|
667
|
+
#elif MDBX_HAVE_PWRITEV
|
|
668
|
+
assert((int)item->sgvcnt > 0);
|
|
669
|
+
const void *end = ptr_disp(item->sgv[item->sgvcnt - 1].iov_base, item->sgv[item->sgvcnt - 1].iov_len);
|
|
670
|
+
if (unlikely(end == data)) {
|
|
671
|
+
item->sgv[item->sgvcnt - 1].iov_len += bytes;
|
|
672
|
+
ior->last_bytes += bytes;
|
|
673
|
+
return MDBX_SUCCESS;
|
|
674
|
+
}
|
|
675
|
+
if (likely(item->sgvcnt < OSAL_IOV_MAX)) {
|
|
676
|
+
if (unlikely(ior->slots_left < 1))
|
|
677
|
+
return MDBX_RESULT_TRUE;
|
|
678
|
+
item->sgv[item->sgvcnt].iov_base = data;
|
|
679
|
+
item->sgv[item->sgvcnt].iov_len = bytes;
|
|
680
|
+
ior->last_bytes += bytes;
|
|
681
|
+
item->sgvcnt += 1;
|
|
682
|
+
ior->slots_left -= 1;
|
|
683
|
+
return MDBX_SUCCESS;
|
|
684
|
+
}
|
|
685
|
+
#else
|
|
686
|
+
const void *end = ptr_disp(item->single.iov_base, item->single.iov_len);
|
|
687
|
+
if (unlikely(end == data)) {
|
|
688
|
+
item->single.iov_len += bytes;
|
|
689
|
+
return MDBX_SUCCESS;
|
|
690
|
+
}
|
|
691
|
+
#endif
|
|
692
|
+
}
|
|
693
|
+
item = ior_next(item, ior_last_sgvcnt(ior, item));
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
if (unlikely(ior->slots_left < 1))
|
|
697
|
+
return MDBX_RESULT_TRUE;
|
|
698
|
+
|
|
699
|
+
unsigned slots_used = 1;
|
|
700
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
701
|
+
item->ov.Internal = item->ov.InternalHigh = 0;
|
|
702
|
+
item->ov.Offset = (DWORD)offset;
|
|
703
|
+
item->ov.OffsetHigh = HIGH_DWORD(offset);
|
|
704
|
+
item->ov.hEvent = 0;
|
|
705
|
+
if (!use_gather || ((bytes | (uintptr_t)(data)) & ior_alignment_mask) != 0 || segments > OSAL_IOV_MAX) {
|
|
706
|
+
/* WriteFile() */
|
|
707
|
+
item->single.iov_base = data;
|
|
708
|
+
item->single.iov_len = bytes + ior_WriteFile_flag;
|
|
709
|
+
assert((item->single.iov_len & ior_WriteFile_flag) != 0);
|
|
710
|
+
} else {
|
|
711
|
+
/* WriteFileGather() */
|
|
712
|
+
assert(ior->overlapped_fd);
|
|
713
|
+
item->sgv[0].Buffer = PtrToPtr64(data);
|
|
714
|
+
for (size_t i = 1; i < segments; ++i) {
|
|
715
|
+
data = ptr_disp(data, ior->pagesize);
|
|
716
|
+
item->sgv[i].Buffer = PtrToPtr64(data);
|
|
717
|
+
}
|
|
718
|
+
item->sgv[slots_used = segments].Buffer = 0;
|
|
719
|
+
assert((item->single.iov_len & ior_WriteFile_flag) == 0);
|
|
720
|
+
}
|
|
721
|
+
ior->last_bytes = bytes;
|
|
722
|
+
ior_last_sgvcnt(ior, item) = slots_used;
|
|
723
|
+
#elif MDBX_HAVE_PWRITEV
|
|
724
|
+
item->offset = offset;
|
|
725
|
+
item->sgv[0].iov_base = data;
|
|
726
|
+
item->sgv[0].iov_len = bytes;
|
|
727
|
+
ior->last_bytes = bytes;
|
|
728
|
+
ior_last_sgvcnt(ior, item) = slots_used;
|
|
729
|
+
#else
|
|
730
|
+
item->offset = offset;
|
|
731
|
+
item->single.iov_base = data;
|
|
732
|
+
item->single.iov_len = bytes;
|
|
733
|
+
#endif /* !Windows */
|
|
734
|
+
ior->slots_left -= slots_used;
|
|
735
|
+
ior->last = item;
|
|
736
|
+
return MDBX_SUCCESS;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx,
|
|
740
|
+
void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)) {
|
|
741
|
+
for (ior_item_t *item = ior->pool; item <= ior->last;) {
|
|
742
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
743
|
+
size_t offset = ior_offset(item);
|
|
744
|
+
char *data = item->single.iov_base;
|
|
745
|
+
size_t bytes = item->single.iov_len - ior_WriteFile_flag;
|
|
746
|
+
size_t i = 1;
|
|
747
|
+
if (bytes & ior_WriteFile_flag) {
|
|
748
|
+
data = Ptr64ToPtr(item->sgv[0].Buffer);
|
|
749
|
+
bytes = ior->pagesize;
|
|
750
|
+
/* Zap: Reading invalid data from 'item->sgv' */
|
|
751
|
+
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
|
|
752
|
+
while (item->sgv[i].Buffer) {
|
|
753
|
+
if (data + ior->pagesize != item->sgv[i].Buffer) {
|
|
754
|
+
callback(ctx, offset, data, bytes);
|
|
755
|
+
offset += bytes;
|
|
756
|
+
data = Ptr64ToPtr(item->sgv[i].Buffer);
|
|
757
|
+
bytes = 0;
|
|
758
|
+
}
|
|
759
|
+
bytes += ior->pagesize;
|
|
760
|
+
++i;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
assert(bytes < MAX_WRITE);
|
|
764
|
+
callback(ctx, offset, data, bytes);
|
|
765
|
+
#elif MDBX_HAVE_PWRITEV
|
|
766
|
+
assert(item->sgvcnt > 0);
|
|
767
|
+
size_t offset = item->offset;
|
|
768
|
+
size_t i = 0;
|
|
769
|
+
do {
|
|
770
|
+
callback(ctx, offset, item->sgv[i].iov_base, item->sgv[i].iov_len);
|
|
771
|
+
offset += item->sgv[i].iov_len;
|
|
772
|
+
} while (++i != item->sgvcnt);
|
|
773
|
+
#else
|
|
774
|
+
const size_t i = 1;
|
|
775
|
+
callback(ctx, item->offset, item->single.iov_base, item->single.iov_len);
|
|
776
|
+
#endif
|
|
777
|
+
item = ior_next(item, i);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) {
|
|
782
|
+
osal_ioring_write_result_t r = {MDBX_SUCCESS, 0};
|
|
783
|
+
|
|
784
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
785
|
+
HANDLE *const end_wait_for = ior->event_pool + ior->allocated +
|
|
786
|
+
/* был выделен один дополнительный элемент для async_done */ 1;
|
|
787
|
+
HANDLE *wait_for = end_wait_for;
|
|
788
|
+
LONG async_started = 0;
|
|
789
|
+
for (ior_item_t *item = ior->pool; item <= ior->last;) {
|
|
790
|
+
item->ov.Internal = STATUS_PENDING;
|
|
791
|
+
size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag;
|
|
792
|
+
r.wops += 1;
|
|
793
|
+
if (bytes & ior_WriteFile_flag) {
|
|
794
|
+
assert(ior->overlapped_fd && fd == ior->overlapped_fd);
|
|
795
|
+
bytes = ior->pagesize;
|
|
796
|
+
/* Zap: Reading invalid data from 'item->sgv' */
|
|
797
|
+
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
|
|
798
|
+
while (item->sgv[i].Buffer) {
|
|
799
|
+
bytes += ior->pagesize;
|
|
800
|
+
++i;
|
|
801
|
+
}
|
|
802
|
+
assert(bytes < MAX_WRITE);
|
|
803
|
+
item->ov.hEvent = ior_get_event(ior);
|
|
804
|
+
if (unlikely(!item->ov.hEvent)) {
|
|
805
|
+
bailout_geterr:
|
|
806
|
+
r.err = GetLastError();
|
|
807
|
+
bailout_rc:
|
|
808
|
+
assert(r.err != MDBX_SUCCESS);
|
|
809
|
+
CancelIo(fd);
|
|
810
|
+
return r;
|
|
811
|
+
}
|
|
812
|
+
if (WriteFileGather(fd, item->sgv, (DWORD)bytes, nullptr, &item->ov)) {
|
|
813
|
+
assert(item->ov.Internal == 0 && WaitForSingleObject(item->ov.hEvent, 0) == WAIT_OBJECT_0);
|
|
814
|
+
ior_put_event(ior, item->ov.hEvent);
|
|
815
|
+
item->ov.hEvent = 0;
|
|
816
|
+
} else {
|
|
817
|
+
r.err = (int)GetLastError();
|
|
818
|
+
if (unlikely(r.err != ERROR_IO_PENDING)) {
|
|
819
|
+
void *data = Ptr64ToPtr(item->sgv[0].Buffer);
|
|
820
|
+
ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
|
|
821
|
+
" offset %" PRId64 ", err %d",
|
|
822
|
+
"WriteFileGather", fd, __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno,
|
|
823
|
+
bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err);
|
|
824
|
+
goto bailout_rc;
|
|
825
|
+
}
|
|
826
|
+
assert(wait_for > ior->event_pool + ior->event_stack);
|
|
827
|
+
*--wait_for = item->ov.hEvent;
|
|
828
|
+
}
|
|
829
|
+
} else if (fd == ior->overlapped_fd) {
|
|
830
|
+
assert(bytes < MAX_WRITE);
|
|
831
|
+
retry:
|
|
832
|
+
item->ov.hEvent = ior;
|
|
833
|
+
if (WriteFileEx(fd, item->single.iov_base, (DWORD)bytes, &item->ov, ior_wocr)) {
|
|
834
|
+
async_started += 1;
|
|
835
|
+
} else {
|
|
836
|
+
r.err = (int)GetLastError();
|
|
837
|
+
switch (r.err) {
|
|
838
|
+
default:
|
|
839
|
+
ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
|
|
840
|
+
" offset %" PRId64 ", err %d",
|
|
841
|
+
"WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
|
|
842
|
+
((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
|
|
843
|
+
r.err);
|
|
844
|
+
goto bailout_rc;
|
|
845
|
+
case ERROR_NOT_FOUND:
|
|
846
|
+
case ERROR_USER_MAPPED_FILE:
|
|
847
|
+
case ERROR_LOCK_VIOLATION:
|
|
848
|
+
WARNING("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
|
|
849
|
+
" offset %" PRId64 ", err %d",
|
|
850
|
+
"WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
|
|
851
|
+
((page_t *)item->single.iov_base)->pgno, bytes,
|
|
852
|
+
item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err);
|
|
853
|
+
SleepEx(0, true);
|
|
854
|
+
goto retry;
|
|
855
|
+
case ERROR_INVALID_USER_BUFFER:
|
|
856
|
+
case ERROR_NOT_ENOUGH_MEMORY:
|
|
857
|
+
if (SleepEx(0, true) == WAIT_IO_COMPLETION)
|
|
858
|
+
goto retry;
|
|
859
|
+
goto bailout_rc;
|
|
860
|
+
case ERROR_IO_PENDING:
|
|
861
|
+
async_started += 1;
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
} else {
|
|
865
|
+
assert(bytes < MAX_WRITE);
|
|
866
|
+
DWORD written = 0;
|
|
867
|
+
if (!WriteFile(fd, item->single.iov_base, (DWORD)bytes, &written, &item->ov)) {
|
|
868
|
+
r.err = (int)GetLastError();
|
|
869
|
+
ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu,"
|
|
870
|
+
" offset %" PRId64 ", err %d",
|
|
871
|
+
"WriteFile", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base,
|
|
872
|
+
((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
|
|
873
|
+
r.err);
|
|
874
|
+
goto bailout_rc;
|
|
875
|
+
} else if (unlikely(written != bytes)) {
|
|
876
|
+
r.err = ERROR_WRITE_FAULT;
|
|
877
|
+
goto bailout_rc;
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
item = ior_next(item, i);
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
assert(ior->async_waiting > ior->async_completed && ior->async_waiting == INT_MAX);
|
|
884
|
+
ior->async_waiting = async_started;
|
|
885
|
+
if (async_started > ior->async_completed && end_wait_for == wait_for) {
|
|
886
|
+
assert(wait_for > ior->event_pool + ior->event_stack);
|
|
887
|
+
*--wait_for = ior->async_done;
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
const size_t pending_count = end_wait_for - wait_for;
|
|
891
|
+
if (pending_count) {
|
|
892
|
+
/* Ждем до MAXIMUM_WAIT_OBJECTS (64) последних хендлов, а после избирательно
|
|
893
|
+
* ждем посредством GetOverlappedResult(), если какие-то более ранние
|
|
894
|
+
* элементы еще не завершены. В целом, так получается меньше системных
|
|
895
|
+
* вызовов, т.е. меньше накладных расходов. Однако, не факт что эта экономия
|
|
896
|
+
* не будет перекрыта неэффективностью реализации
|
|
897
|
+
* WaitForMultipleObjectsEx(), но тогда это проблемы на стороне M$. */
|
|
898
|
+
DWORD madness;
|
|
899
|
+
do
|
|
900
|
+
madness = WaitForMultipleObjectsEx(
|
|
901
|
+
(pending_count < MAXIMUM_WAIT_OBJECTS) ? (DWORD)pending_count : MAXIMUM_WAIT_OBJECTS, wait_for, true,
|
|
902
|
+
/* сутки */ 86400000ul, true);
|
|
903
|
+
while (madness == WAIT_IO_COMPLETION);
|
|
904
|
+
STATIC_ASSERT(WAIT_OBJECT_0 == 0);
|
|
905
|
+
if (/* madness >= WAIT_OBJECT_0 && */
|
|
906
|
+
madness < WAIT_OBJECT_0 + MAXIMUM_WAIT_OBJECTS)
|
|
907
|
+
r.err = MDBX_SUCCESS;
|
|
908
|
+
else if (madness >= WAIT_ABANDONED_0 && madness < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) {
|
|
909
|
+
r.err = ERROR_ABANDONED_WAIT_0;
|
|
910
|
+
goto bailout_rc;
|
|
911
|
+
} else if (madness == WAIT_TIMEOUT) {
|
|
912
|
+
r.err = WAIT_TIMEOUT;
|
|
913
|
+
goto bailout_rc;
|
|
914
|
+
} else {
|
|
915
|
+
r.err = /* madness == WAIT_FAILED */ MDBX_PROBLEM;
|
|
916
|
+
goto bailout_rc;
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
assert(ior->async_waiting == ior->async_completed);
|
|
920
|
+
for (ior_item_t *item = ior->pool; item <= ior->last;) {
|
|
921
|
+
size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag;
|
|
922
|
+
void *data = item->single.iov_base;
|
|
923
|
+
if (bytes & ior_WriteFile_flag) {
|
|
924
|
+
data = Ptr64ToPtr(item->sgv[0].Buffer);
|
|
925
|
+
bytes = ior->pagesize;
|
|
926
|
+
/* Zap: Reading invalid data from 'item->sgv' */
|
|
927
|
+
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
|
|
928
|
+
while (item->sgv[i].Buffer) {
|
|
929
|
+
bytes += ior->pagesize;
|
|
930
|
+
++i;
|
|
931
|
+
}
|
|
932
|
+
if (!HasOverlappedIoCompleted(&item->ov)) {
|
|
933
|
+
DWORD written = 0;
|
|
934
|
+
if (unlikely(!GetOverlappedResult(fd, &item->ov, &written, true))) {
|
|
935
|
+
ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu,"
|
|
936
|
+
" offset %" PRId64 ", err %d",
|
|
937
|
+
"GetOverlappedResult", __Wpedantic_format_voidptr(item), item - ior->pool, data,
|
|
938
|
+
((page_t *)data)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32),
|
|
939
|
+
(int)GetLastError());
|
|
940
|
+
goto bailout_geterr;
|
|
941
|
+
}
|
|
942
|
+
assert(MDBX_SUCCESS == item->ov.Internal);
|
|
943
|
+
assert(written == item->ov.InternalHigh);
|
|
944
|
+
}
|
|
945
|
+
} else {
|
|
946
|
+
assert(HasOverlappedIoCompleted(&item->ov));
|
|
947
|
+
}
|
|
948
|
+
assert(item->ov.Internal != ERROR_IO_PENDING);
|
|
949
|
+
if (unlikely(item->ov.Internal != MDBX_SUCCESS)) {
|
|
950
|
+
DWORD written = 0;
|
|
951
|
+
r.err = (int)item->ov.Internal;
|
|
952
|
+
if ((r.err & 0x80000000) && GetOverlappedResult(nullptr, &item->ov, &written, true))
|
|
953
|
+
r.err = (int)GetLastError();
|
|
954
|
+
ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu,"
|
|
955
|
+
" offset %" PRId64 ", err %d",
|
|
956
|
+
"Result", __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno, bytes,
|
|
957
|
+
item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), (int)GetLastError());
|
|
958
|
+
goto bailout_rc;
|
|
959
|
+
}
|
|
960
|
+
if (unlikely(item->ov.InternalHigh != bytes)) {
|
|
961
|
+
r.err = ERROR_WRITE_FAULT;
|
|
962
|
+
goto bailout_rc;
|
|
963
|
+
}
|
|
964
|
+
item = ior_next(item, i);
|
|
965
|
+
}
|
|
966
|
+
assert(ior->async_waiting == ior->async_completed);
|
|
967
|
+
} else {
|
|
968
|
+
assert(r.err == MDBX_SUCCESS);
|
|
969
|
+
}
|
|
970
|
+
assert(ior->async_waiting == ior->async_completed);
|
|
971
|
+
|
|
972
|
+
#else
|
|
973
|
+
STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
|
|
974
|
+
for (ior_item_t *item = ior->pool; item <= ior->last;) {
|
|
975
|
+
#if MDBX_HAVE_PWRITEV
|
|
976
|
+
assert(item->sgvcnt > 0);
|
|
977
|
+
if (item->sgvcnt == 1)
|
|
978
|
+
r.err = osal_pwrite(fd, item->sgv[0].iov_base, item->sgv[0].iov_len, item->offset);
|
|
979
|
+
else
|
|
980
|
+
r.err = osal_pwritev(fd, item->sgv, item->sgvcnt, item->offset);
|
|
981
|
+
|
|
982
|
+
// TODO: io_uring_prep_write(sqe, fd, ...);
|
|
983
|
+
|
|
984
|
+
item = ior_next(item, item->sgvcnt);
|
|
985
|
+
#else
|
|
986
|
+
r.err = osal_pwrite(fd, item->single.iov_base, item->single.iov_len, item->offset);
|
|
987
|
+
item = ior_next(item, 1);
|
|
988
|
+
#endif
|
|
989
|
+
r.wops += 1;
|
|
990
|
+
if (unlikely(r.err != MDBX_SUCCESS))
|
|
991
|
+
break;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
// TODO: io_uring_submit(&ring)
|
|
995
|
+
// TODO: err = io_uring_wait_cqe(&ring, &cqe);
|
|
996
|
+
// TODO: io_uring_cqe_seen(&ring, cqe);
|
|
997
|
+
|
|
998
|
+
#endif /* !Windows */
|
|
999
|
+
return r;
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *ior) {
|
|
1003
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1004
|
+
if (ior->last) {
|
|
1005
|
+
for (ior_item_t *item = ior->pool; item <= ior->last;) {
|
|
1006
|
+
if (!HasOverlappedIoCompleted(&item->ov)) {
|
|
1007
|
+
assert(ior->overlapped_fd);
|
|
1008
|
+
CancelIoEx(ior->overlapped_fd, &item->ov);
|
|
1009
|
+
}
|
|
1010
|
+
if (item->ov.hEvent && item->ov.hEvent != ior)
|
|
1011
|
+
ior_put_event(ior, item->ov.hEvent);
|
|
1012
|
+
size_t i = 1;
|
|
1013
|
+
if ((item->single.iov_len & ior_WriteFile_flag) == 0) {
|
|
1014
|
+
/* Zap: Reading invalid data from 'item->sgv' */
|
|
1015
|
+
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385);
|
|
1016
|
+
while (item->sgv[i].Buffer)
|
|
1017
|
+
++i;
|
|
1018
|
+
}
|
|
1019
|
+
item = ior_next(item, i);
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
ior->async_waiting = INT_MAX;
|
|
1023
|
+
ior->async_completed = 0;
|
|
1024
|
+
ResetEvent(ior->async_done);
|
|
1025
|
+
#endif /* !Windows */
|
|
1026
|
+
ior->slots_left = ior->allocated;
|
|
1027
|
+
ior->last = nullptr;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
static void ior_cleanup(osal_ioring_t *ior, const size_t since) {
|
|
1031
|
+
osal_ioring_reset(ior);
|
|
1032
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1033
|
+
for (size_t i = since; i < ior->event_stack; ++i) {
|
|
1034
|
+
/* Zap: Using uninitialized memory '**ior.event_pool' */
|
|
1035
|
+
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6001);
|
|
1036
|
+
CloseHandle(ior->event_pool[i]);
|
|
1037
|
+
}
|
|
1038
|
+
ior->event_stack = 0;
|
|
1039
|
+
#else
|
|
1040
|
+
(void)since;
|
|
1041
|
+
#endif /* Windows */
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *ior, size_t items) {
|
|
1045
|
+
assert(items > 0 && items < INT_MAX / sizeof(ior_item_t));
|
|
1046
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1047
|
+
if (ior->state & IOR_STATE_LOCKED)
|
|
1048
|
+
return MDBX_SUCCESS;
|
|
1049
|
+
const bool useSetFileIoOverlappedRange = ior->overlapped_fd && imports.SetFileIoOverlappedRange && items > 42;
|
|
1050
|
+
const size_t ceiling =
|
|
1051
|
+
useSetFileIoOverlappedRange ? ((items < 65536 / 2 / sizeof(ior_item_t)) ? 65536 : 65536 * 4) : 1024;
|
|
1052
|
+
const size_t bytes = ceil_powerof2(sizeof(ior_item_t) * items, ceiling);
|
|
1053
|
+
items = bytes / sizeof(ior_item_t);
|
|
1054
|
+
#endif /* Windows */
|
|
1055
|
+
|
|
1056
|
+
if (items != ior->allocated) {
|
|
1057
|
+
assert(items >= osal_ioring_used(ior));
|
|
1058
|
+
if (items < ior->allocated)
|
|
1059
|
+
ior_cleanup(ior, items);
|
|
1060
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1061
|
+
void *ptr = osal_realloc(ior->event_pool, (items + /* extra for waiting the async_done */ 1) * sizeof(HANDLE));
|
|
1062
|
+
if (unlikely(!ptr))
|
|
1063
|
+
return MDBX_ENOMEM;
|
|
1064
|
+
ior->event_pool = ptr;
|
|
1065
|
+
|
|
1066
|
+
int err = osal_memalign_alloc(ceiling, bytes, &ptr);
|
|
1067
|
+
if (unlikely(err != MDBX_SUCCESS))
|
|
1068
|
+
return err;
|
|
1069
|
+
if (ior->pool) {
|
|
1070
|
+
memcpy(ptr, ior->pool, ior->allocated * sizeof(ior_item_t));
|
|
1071
|
+
osal_memalign_free(ior->pool);
|
|
1072
|
+
}
|
|
1073
|
+
#else
|
|
1074
|
+
void *ptr = osal_realloc(ior->pool, sizeof(ior_item_t) * items);
|
|
1075
|
+
if (unlikely(!ptr))
|
|
1076
|
+
return MDBX_ENOMEM;
|
|
1077
|
+
#endif
|
|
1078
|
+
ior->pool = ptr;
|
|
1079
|
+
|
|
1080
|
+
if (items > ior->allocated)
|
|
1081
|
+
memset(ior->pool + ior->allocated, 0, sizeof(ior_item_t) * (items - ior->allocated));
|
|
1082
|
+
ior->allocated = (unsigned)items;
|
|
1083
|
+
ior->boundary = ptr_disp(ior->pool, ior->allocated);
|
|
1084
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1085
|
+
if (useSetFileIoOverlappedRange) {
|
|
1086
|
+
if (imports.SetFileIoOverlappedRange(ior->overlapped_fd, ptr, (ULONG)bytes))
|
|
1087
|
+
ior->state += IOR_STATE_LOCKED;
|
|
1088
|
+
else
|
|
1089
|
+
return GetLastError();
|
|
1090
|
+
}
|
|
1091
|
+
#endif /* Windows */
|
|
1092
|
+
}
|
|
1093
|
+
return MDBX_SUCCESS;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *ior) {
|
|
1097
|
+
if (ior->allocated)
|
|
1098
|
+
ior_cleanup(ior, 0);
|
|
1099
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1100
|
+
osal_memalign_free(ior->pool);
|
|
1101
|
+
osal_free(ior->event_pool);
|
|
1102
|
+
CloseHandle(ior->async_done);
|
|
1103
|
+
if (ior->overlapped_fd)
|
|
1104
|
+
CloseHandle(ior->overlapped_fd);
|
|
1105
|
+
#else
|
|
1106
|
+
osal_free(ior->pool);
|
|
1107
|
+
#endif
|
|
1108
|
+
memset(ior, 0, sizeof(osal_ioring_t));
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
/*----------------------------------------------------------------------------*/
|
|
1112
|
+
|
|
1113
|
+
MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname) {
|
|
1114
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1115
|
+
return DeleteFileW(pathname) ? MDBX_SUCCESS : (int)GetLastError();
|
|
1116
|
+
#else
|
|
1117
|
+
return unlink(pathname) ? errno : MDBX_SUCCESS;
|
|
1118
|
+
#endif
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
#if !(defined(_WIN32) || defined(_WIN64))
|
|
1122
|
+
static bool is_valid_fd(int fd) { return !(isatty(fd) < 0 && errno == EBADF); }
|
|
1123
|
+
#endif /*! Windows */
|
|
1124
|
+
|
|
1125
|
+
MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname) {
|
|
1126
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1127
|
+
return RemoveDirectoryW(pathname) ? MDBX_SUCCESS : (int)GetLastError();
|
|
1128
|
+
#else
|
|
1129
|
+
return rmdir(pathname) ? errno : MDBX_SUCCESS;
|
|
1130
|
+
#endif
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname) {
|
|
1134
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1135
|
+
if (GetFileAttributesW(pathname) != INVALID_FILE_ATTRIBUTES)
|
|
1136
|
+
return MDBX_RESULT_TRUE;
|
|
1137
|
+
int err = GetLastError();
|
|
1138
|
+
return (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) ? MDBX_RESULT_FALSE : err;
|
|
1139
|
+
#else
|
|
1140
|
+
if (access(pathname, F_OK) == 0)
|
|
1141
|
+
return MDBX_RESULT_TRUE;
|
|
1142
|
+
int err = errno;
|
|
1143
|
+
return (err == ENOENT || err == ENOTDIR) ? MDBX_RESULT_FALSE : err;
|
|
1144
|
+
#endif
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len) {
|
|
1148
|
+
const pathchar_t *ext = nullptr;
|
|
1149
|
+
for (size_t i = 0; i < len && pathname[i]; i++)
|
|
1150
|
+
if (pathname[i] == '.')
|
|
1151
|
+
ext = pathname + i;
|
|
1152
|
+
else if (osal_isdirsep(pathname[i]))
|
|
1153
|
+
ext = nullptr;
|
|
1154
|
+
return (pathchar_t *)ext;
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len) {
|
|
1158
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1159
|
+
for (size_t i = 0; i < len; ++i) {
|
|
1160
|
+
pathchar_t a = l[i];
|
|
1161
|
+
pathchar_t b = r[i];
|
|
1162
|
+
a = (a == '\\') ? '/' : a;
|
|
1163
|
+
b = (b == '\\') ? '/' : b;
|
|
1164
|
+
if (a != b)
|
|
1165
|
+
return false;
|
|
1166
|
+
}
|
|
1167
|
+
return true;
|
|
1168
|
+
#else
|
|
1169
|
+
return memcmp(l, r, len * sizeof(pathchar_t)) == 0;
|
|
1170
|
+
#endif
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env,
|
|
1174
|
+
const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits) {
|
|
1175
|
+
*fd = INVALID_HANDLE_VALUE;
|
|
1176
|
+
|
|
1177
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1178
|
+
DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING;
|
|
1179
|
+
DWORD FlagsAndAttributes = FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED;
|
|
1180
|
+
DWORD DesiredAccess = FILE_READ_ATTRIBUTES;
|
|
1181
|
+
DWORD ShareMode = (env->flags & MDBX_EXCLUSIVE) ? 0 : (FILE_SHARE_READ | FILE_SHARE_WRITE);
|
|
1182
|
+
|
|
1183
|
+
switch (purpose) {
|
|
1184
|
+
default:
|
|
1185
|
+
return ERROR_INVALID_PARAMETER;
|
|
1186
|
+
case MDBX_OPEN_LCK:
|
|
1187
|
+
CreationDisposition = OPEN_ALWAYS;
|
|
1188
|
+
DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
|
|
1189
|
+
FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY;
|
|
1190
|
+
break;
|
|
1191
|
+
case MDBX_OPEN_DXB_READ:
|
|
1192
|
+
CreationDisposition = OPEN_EXISTING;
|
|
1193
|
+
DesiredAccess |= GENERIC_READ;
|
|
1194
|
+
ShareMode |= FILE_SHARE_READ;
|
|
1195
|
+
break;
|
|
1196
|
+
case MDBX_OPEN_DXB_LAZY:
|
|
1197
|
+
DesiredAccess |= GENERIC_READ | GENERIC_WRITE;
|
|
1198
|
+
break;
|
|
1199
|
+
case MDBX_OPEN_DXB_OVERLAPPED_DIRECT:
|
|
1200
|
+
FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING;
|
|
1201
|
+
/* fall through */
|
|
1202
|
+
__fallthrough;
|
|
1203
|
+
case MDBX_OPEN_DXB_OVERLAPPED:
|
|
1204
|
+
FlagsAndAttributes |= FILE_FLAG_OVERLAPPED;
|
|
1205
|
+
/* fall through */
|
|
1206
|
+
__fallthrough;
|
|
1207
|
+
case MDBX_OPEN_DXB_DSYNC:
|
|
1208
|
+
CreationDisposition = OPEN_EXISTING;
|
|
1209
|
+
DesiredAccess |= GENERIC_WRITE | GENERIC_READ;
|
|
1210
|
+
FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH;
|
|
1211
|
+
break;
|
|
1212
|
+
case MDBX_OPEN_COPY:
|
|
1213
|
+
CreationDisposition = CREATE_NEW;
|
|
1214
|
+
ShareMode = 0;
|
|
1215
|
+
DesiredAccess |= GENERIC_WRITE;
|
|
1216
|
+
if (env->ps >= globals.sys_pagesize)
|
|
1217
|
+
FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING;
|
|
1218
|
+
break;
|
|
1219
|
+
case MDBX_OPEN_DELETE:
|
|
1220
|
+
CreationDisposition = OPEN_EXISTING;
|
|
1221
|
+
ShareMode |= FILE_SHARE_DELETE;
|
|
1222
|
+
DesiredAccess = FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE;
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
*fd = CreateFileW(pathname, DesiredAccess, ShareMode, nullptr, CreationDisposition, FlagsAndAttributes, nullptr);
|
|
1227
|
+
if (*fd == INVALID_HANDLE_VALUE) {
|
|
1228
|
+
int err = (int)GetLastError();
|
|
1229
|
+
if (err == ERROR_ACCESS_DENIED && purpose == MDBX_OPEN_LCK) {
|
|
1230
|
+
if (GetFileAttributesW(pathname) == INVALID_FILE_ATTRIBUTES && GetLastError() == ERROR_FILE_NOT_FOUND)
|
|
1231
|
+
err = ERROR_FILE_NOT_FOUND;
|
|
1232
|
+
}
|
|
1233
|
+
return err;
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
BY_HANDLE_FILE_INFORMATION info;
|
|
1237
|
+
if (!GetFileInformationByHandle(*fd, &info)) {
|
|
1238
|
+
int err = (int)GetLastError();
|
|
1239
|
+
CloseHandle(*fd);
|
|
1240
|
+
*fd = INVALID_HANDLE_VALUE;
|
|
1241
|
+
return err;
|
|
1242
|
+
}
|
|
1243
|
+
const DWORD AttributesDiff =
|
|
1244
|
+
(info.dwFileAttributes ^ FlagsAndAttributes) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED |
|
|
1245
|
+
FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED);
|
|
1246
|
+
if (AttributesDiff)
|
|
1247
|
+
(void)SetFileAttributesW(pathname, info.dwFileAttributes ^ AttributesDiff);
|
|
1248
|
+
|
|
1249
|
+
#else
|
|
1250
|
+
int flags = unix_mode_bits ? O_CREAT : 0;
|
|
1251
|
+
switch (purpose) {
|
|
1252
|
+
default:
|
|
1253
|
+
return EINVAL;
|
|
1254
|
+
case MDBX_OPEN_LCK:
|
|
1255
|
+
flags |= O_RDWR;
|
|
1256
|
+
break;
|
|
1257
|
+
case MDBX_OPEN_DXB_READ:
|
|
1258
|
+
flags = O_RDONLY;
|
|
1259
|
+
break;
|
|
1260
|
+
case MDBX_OPEN_DXB_LAZY:
|
|
1261
|
+
flags |= O_RDWR;
|
|
1262
|
+
break;
|
|
1263
|
+
case MDBX_OPEN_COPY:
|
|
1264
|
+
flags = O_CREAT | O_WRONLY | O_EXCL;
|
|
1265
|
+
break;
|
|
1266
|
+
case MDBX_OPEN_DXB_DSYNC:
|
|
1267
|
+
flags |= O_WRONLY;
|
|
1268
|
+
#if defined(O_DSYNC)
|
|
1269
|
+
flags |= O_DSYNC;
|
|
1270
|
+
#elif defined(O_SYNC)
|
|
1271
|
+
flags |= O_SYNC;
|
|
1272
|
+
#elif defined(O_FSYNC)
|
|
1273
|
+
flags |= O_FSYNC;
|
|
1274
|
+
#endif
|
|
1275
|
+
break;
|
|
1276
|
+
case MDBX_OPEN_DELETE:
|
|
1277
|
+
flags = O_RDWR;
|
|
1278
|
+
break;
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
const bool direct_nocache_for_copy = env->ps >= globals.sys_pagesize && purpose == MDBX_OPEN_COPY;
|
|
1282
|
+
if (direct_nocache_for_copy) {
|
|
1283
|
+
#if defined(O_DIRECT)
|
|
1284
|
+
flags |= O_DIRECT;
|
|
1285
|
+
#endif /* O_DIRECT */
|
|
1286
|
+
#if defined(O_NOCACHE)
|
|
1287
|
+
flags |= O_NOCACHE;
|
|
1288
|
+
#endif /* O_NOCACHE */
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
#ifdef O_CLOEXEC
|
|
1292
|
+
flags |= O_CLOEXEC;
|
|
1293
|
+
#endif /* O_CLOEXEC */
|
|
1294
|
+
|
|
1295
|
+
/* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */
|
|
1296
|
+
#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
|
|
1297
|
+
int stub_fd0 = -1, stub_fd1 = -1, stub_fd2 = -1;
|
|
1298
|
+
static const char dev_null[] = "/dev/null";
|
|
1299
|
+
if (!is_valid_fd(STDIN_FILENO)) {
|
|
1300
|
+
WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN", STDIN_FILENO, dev_null);
|
|
1301
|
+
stub_fd0 = open(dev_null, O_RDONLY | O_NOCTTY);
|
|
1302
|
+
}
|
|
1303
|
+
if (!is_valid_fd(STDOUT_FILENO)) {
|
|
1304
|
+
WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "OUT", STDOUT_FILENO, dev_null);
|
|
1305
|
+
stub_fd1 = open(dev_null, O_WRONLY | O_NOCTTY);
|
|
1306
|
+
}
|
|
1307
|
+
if (!is_valid_fd(STDERR_FILENO)) {
|
|
1308
|
+
WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "ERR", STDERR_FILENO, dev_null);
|
|
1309
|
+
stub_fd2 = open(dev_null, O_WRONLY | O_NOCTTY);
|
|
1310
|
+
}
|
|
1311
|
+
#else
|
|
1312
|
+
#error "Unexpected or unsupported UNIX or POSIX system"
|
|
1313
|
+
#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */
|
|
1314
|
+
|
|
1315
|
+
*fd = open(pathname, flags, unix_mode_bits);
|
|
1316
|
+
#if defined(O_DIRECT)
|
|
1317
|
+
if (*fd < 0 && (flags & O_DIRECT) && (errno == EINVAL || errno == EAFNOSUPPORT)) {
|
|
1318
|
+
flags &= ~(O_DIRECT | O_EXCL);
|
|
1319
|
+
*fd = open(pathname, flags, unix_mode_bits);
|
|
1320
|
+
}
|
|
1321
|
+
#endif /* O_DIRECT */
|
|
1322
|
+
|
|
1323
|
+
if (*fd < 0 && errno == EACCES && purpose == MDBX_OPEN_LCK) {
|
|
1324
|
+
struct stat unused;
|
|
1325
|
+
if (stat(pathname, &unused) == 0 || errno != ENOENT)
|
|
1326
|
+
errno = EACCES /* restore errno if file exists */;
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
/* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */
|
|
1330
|
+
#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2
|
|
1331
|
+
if (*fd == STDIN_FILENO) {
|
|
1332
|
+
WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN", STDIN_FILENO);
|
|
1333
|
+
assert(stub_fd0 == -1);
|
|
1334
|
+
*fd = dup(stub_fd0 = *fd);
|
|
1335
|
+
}
|
|
1336
|
+
if (*fd == STDOUT_FILENO) {
|
|
1337
|
+
WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT", STDOUT_FILENO);
|
|
1338
|
+
assert(stub_fd1 == -1);
|
|
1339
|
+
*fd = dup(stub_fd1 = *fd);
|
|
1340
|
+
}
|
|
1341
|
+
if (*fd == STDERR_FILENO) {
|
|
1342
|
+
WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR", STDERR_FILENO);
|
|
1343
|
+
assert(stub_fd2 == -1);
|
|
1344
|
+
*fd = dup(stub_fd2 = *fd);
|
|
1345
|
+
}
|
|
1346
|
+
const int err = errno;
|
|
1347
|
+
if (stub_fd0 != -1)
|
|
1348
|
+
close(stub_fd0);
|
|
1349
|
+
if (stub_fd1 != -1)
|
|
1350
|
+
close(stub_fd1);
|
|
1351
|
+
if (stub_fd2 != -1)
|
|
1352
|
+
close(stub_fd2);
|
|
1353
|
+
if (*fd >= STDIN_FILENO && *fd <= STDERR_FILENO) {
|
|
1354
|
+
ERROR("Rejecting the use of a FD in the range "
|
|
1355
|
+
"STDIN_FILENO/%d..STDERR_FILENO/%d to prevent database corruption",
|
|
1356
|
+
STDIN_FILENO, STDERR_FILENO);
|
|
1357
|
+
close(*fd);
|
|
1358
|
+
return EBADF;
|
|
1359
|
+
}
|
|
1360
|
+
#else
|
|
1361
|
+
#error "Unexpected or unsupported UNIX or POSIX system"
|
|
1362
|
+
#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */
|
|
1363
|
+
|
|
1364
|
+
if (*fd < 0)
|
|
1365
|
+
return err;
|
|
1366
|
+
|
|
1367
|
+
#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC)
|
|
1368
|
+
const int fd_flags = fcntl(*fd, F_GETFD);
|
|
1369
|
+
if (fd_flags != -1)
|
|
1370
|
+
(void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC);
|
|
1371
|
+
#endif /* FD_CLOEXEC && !O_CLOEXEC */
|
|
1372
|
+
|
|
1373
|
+
if (direct_nocache_for_copy) {
|
|
1374
|
+
#if defined(F_NOCACHE) && !defined(O_NOCACHE)
|
|
1375
|
+
(void)fcntl(*fd, F_NOCACHE, 1);
|
|
1376
|
+
#endif /* F_NOCACHE */
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
#endif
|
|
1380
|
+
return MDBX_SUCCESS;
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd) {
|
|
1384
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1385
|
+
return CloseHandle(fd) ? MDBX_SUCCESS : (int)GetLastError();
|
|
1386
|
+
#else
|
|
1387
|
+
assert(fd > STDERR_FILENO);
|
|
1388
|
+
return (close(fd) == 0) ? MDBX_SUCCESS : errno;
|
|
1389
|
+
#endif
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, uint64_t offset) {
|
|
1393
|
+
if (bytes > MAX_WRITE)
|
|
1394
|
+
return MDBX_EINVAL;
|
|
1395
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1396
|
+
OVERLAPPED ov;
|
|
1397
|
+
ov.hEvent = 0;
|
|
1398
|
+
ov.Offset = (DWORD)offset;
|
|
1399
|
+
ov.OffsetHigh = HIGH_DWORD(offset);
|
|
1400
|
+
|
|
1401
|
+
DWORD read = 0;
|
|
1402
|
+
if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) {
|
|
1403
|
+
int rc = (int)GetLastError();
|
|
1404
|
+
return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc;
|
|
1405
|
+
}
|
|
1406
|
+
#else
|
|
1407
|
+
STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
|
|
1408
|
+
intptr_t read = pread(fd, buf, bytes, offset);
|
|
1409
|
+
if (read < 0) {
|
|
1410
|
+
int rc = errno;
|
|
1411
|
+
return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc;
|
|
1412
|
+
}
|
|
1413
|
+
#endif
|
|
1414
|
+
return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t bytes, uint64_t offset) {
|
|
1418
|
+
while (true) {
|
|
1419
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1420
|
+
OVERLAPPED ov;
|
|
1421
|
+
ov.hEvent = 0;
|
|
1422
|
+
ov.Offset = (DWORD)offset;
|
|
1423
|
+
ov.OffsetHigh = HIGH_DWORD(offset);
|
|
1424
|
+
|
|
1425
|
+
DWORD written;
|
|
1426
|
+
if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, &ov)))
|
|
1427
|
+
return (int)GetLastError();
|
|
1428
|
+
if (likely(bytes == written))
|
|
1429
|
+
return MDBX_SUCCESS;
|
|
1430
|
+
#else
|
|
1431
|
+
STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
|
|
1432
|
+
const intptr_t written = pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset);
|
|
1433
|
+
if (likely(bytes == (size_t)written))
|
|
1434
|
+
return MDBX_SUCCESS;
|
|
1435
|
+
if (written < 0) {
|
|
1436
|
+
const int rc = errno;
|
|
1437
|
+
if (rc != EINTR)
|
|
1438
|
+
return rc;
|
|
1439
|
+
continue;
|
|
1440
|
+
}
|
|
1441
|
+
#endif
|
|
1442
|
+
bytes -= written;
|
|
1443
|
+
offset += written;
|
|
1444
|
+
buf = ptr_disp(buf, written);
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t bytes) {
|
|
1449
|
+
while (true) {
|
|
1450
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1451
|
+
DWORD written;
|
|
1452
|
+
if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, nullptr)))
|
|
1453
|
+
return (int)GetLastError();
|
|
1454
|
+
if (likely(bytes == written))
|
|
1455
|
+
return MDBX_SUCCESS;
|
|
1456
|
+
#else
|
|
1457
|
+
STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
|
|
1458
|
+
const intptr_t written = write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE);
|
|
1459
|
+
if (likely(bytes == (size_t)written))
|
|
1460
|
+
return MDBX_SUCCESS;
|
|
1461
|
+
if (written < 0) {
|
|
1462
|
+
const int rc = errno;
|
|
1463
|
+
if (rc != EINTR)
|
|
1464
|
+
return rc;
|
|
1465
|
+
continue;
|
|
1466
|
+
}
|
|
1467
|
+
#endif
|
|
1468
|
+
bytes -= written;
|
|
1469
|
+
buf = ptr_disp(buf, written);
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset) {
|
|
1474
|
+
size_t expected = 0;
|
|
1475
|
+
for (size_t i = 0; i < sgvcnt; ++i)
|
|
1476
|
+
expected += iov[i].iov_len;
|
|
1477
|
+
#if !MDBX_HAVE_PWRITEV
|
|
1478
|
+
size_t written = 0;
|
|
1479
|
+
for (size_t i = 0; i < sgvcnt; ++i) {
|
|
1480
|
+
int rc = osal_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset);
|
|
1481
|
+
if (unlikely(rc != MDBX_SUCCESS))
|
|
1482
|
+
return rc;
|
|
1483
|
+
written += iov[i].iov_len;
|
|
1484
|
+
offset += iov[i].iov_len;
|
|
1485
|
+
}
|
|
1486
|
+
return (expected == written) ? MDBX_SUCCESS : MDBX_EIO /* ERROR_WRITE_FAULT */;
|
|
1487
|
+
#else
|
|
1488
|
+
int rc;
|
|
1489
|
+
intptr_t written;
|
|
1490
|
+
do {
|
|
1491
|
+
STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
|
|
1492
|
+
written = pwritev(fd, iov, sgvcnt, offset);
|
|
1493
|
+
if (likely(expected == (size_t)written))
|
|
1494
|
+
return MDBX_SUCCESS;
|
|
1495
|
+
rc = errno;
|
|
1496
|
+
} while (rc == EINTR);
|
|
1497
|
+
return (written < 0) ? rc : MDBX_EIO /* Use which error code? */;
|
|
1498
|
+
#endif
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, enum osal_syncmode_bits mode_bits) {
|
|
1502
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1503
|
+
if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd))
|
|
1504
|
+
return (int)GetLastError();
|
|
1505
|
+
return MDBX_SUCCESS;
|
|
1506
|
+
#else
|
|
1507
|
+
|
|
1508
|
+
#if defined(__APPLE__) && MDBX_APPLE_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY
|
|
1509
|
+
if (mode_bits & MDBX_SYNC_IODQ)
|
|
1510
|
+
return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno;
|
|
1511
|
+
#endif /* MacOS */
|
|
1512
|
+
|
|
1513
|
+
/* LY: This approach is always safe and without appreciable performance
|
|
1514
|
+
* degradation, even on a kernel with fdatasync's bug.
|
|
1515
|
+
*
|
|
1516
|
+
* For more info about of a corresponding fdatasync() bug
|
|
1517
|
+
* see http://www.spinics.net/lists/linux-ext4/msg33714.html */
|
|
1518
|
+
while (1) {
|
|
1519
|
+
switch (mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_SIZE)) {
|
|
1520
|
+
case MDBX_SYNC_NONE:
|
|
1521
|
+
case MDBX_SYNC_KICK:
|
|
1522
|
+
return MDBX_SUCCESS /* nothing to do */;
|
|
1523
|
+
#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0
|
|
1524
|
+
case MDBX_SYNC_DATA:
|
|
1525
|
+
if (likely(fdatasync(fd) == 0))
|
|
1526
|
+
return MDBX_SUCCESS;
|
|
1527
|
+
break /* error */;
|
|
1528
|
+
#if defined(__linux__) || defined(__gnu_linux__)
|
|
1529
|
+
case MDBX_SYNC_SIZE:
|
|
1530
|
+
assert(globals.linux_kernel_version >= 0x03060000);
|
|
1531
|
+
return MDBX_SUCCESS;
|
|
1532
|
+
#endif /* Linux */
|
|
1533
|
+
#endif /* _POSIX_SYNCHRONIZED_IO > 0 */
|
|
1534
|
+
default:
|
|
1535
|
+
if (likely(fsync(fd) == 0))
|
|
1536
|
+
return MDBX_SUCCESS;
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
int rc = errno;
|
|
1540
|
+
if (rc != EINTR)
|
|
1541
|
+
return rc;
|
|
1542
|
+
}
|
|
1543
|
+
#endif
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
int osal_filesize(mdbx_filehandle_t fd, uint64_t *length) {
|
|
1547
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1548
|
+
BY_HANDLE_FILE_INFORMATION info;
|
|
1549
|
+
if (!GetFileInformationByHandle(fd, &info))
|
|
1550
|
+
return (int)GetLastError();
|
|
1551
|
+
*length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32;
|
|
1552
|
+
#else
|
|
1553
|
+
struct stat st;
|
|
1554
|
+
|
|
1555
|
+
STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
|
|
1556
|
+
if (fstat(fd, &st))
|
|
1557
|
+
return errno;
|
|
1558
|
+
|
|
1559
|
+
*length = st.st_size;
|
|
1560
|
+
#endif
|
|
1561
|
+
return MDBX_SUCCESS;
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd) {
|
|
1565
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1566
|
+
switch (GetFileType(fd)) {
|
|
1567
|
+
case FILE_TYPE_DISK:
|
|
1568
|
+
return MDBX_RESULT_FALSE;
|
|
1569
|
+
case FILE_TYPE_CHAR:
|
|
1570
|
+
case FILE_TYPE_PIPE:
|
|
1571
|
+
return MDBX_RESULT_TRUE;
|
|
1572
|
+
default:
|
|
1573
|
+
return (int)GetLastError();
|
|
1574
|
+
}
|
|
1575
|
+
#else
|
|
1576
|
+
struct stat info;
|
|
1577
|
+
if (fstat(fd, &info))
|
|
1578
|
+
return errno;
|
|
1579
|
+
switch (info.st_mode & S_IFMT) {
|
|
1580
|
+
case S_IFBLK:
|
|
1581
|
+
case S_IFREG:
|
|
1582
|
+
return MDBX_RESULT_FALSE;
|
|
1583
|
+
case S_IFCHR:
|
|
1584
|
+
case S_IFIFO:
|
|
1585
|
+
case S_IFSOCK:
|
|
1586
|
+
return MDBX_RESULT_TRUE;
|
|
1587
|
+
case S_IFDIR:
|
|
1588
|
+
case S_IFLNK:
|
|
1589
|
+
default:
|
|
1590
|
+
return MDBX_INCOMPATIBLE;
|
|
1591
|
+
}
|
|
1592
|
+
#endif
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length) {
|
|
1596
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1597
|
+
if (imports.SetFileInformationByHandle) {
|
|
1598
|
+
FILE_END_OF_FILE_INFO EndOfFileInfo;
|
|
1599
|
+
EndOfFileInfo.EndOfFile.QuadPart = length;
|
|
1600
|
+
return imports.SetFileInformationByHandle(fd, FileEndOfFileInfo, &EndOfFileInfo, sizeof(FILE_END_OF_FILE_INFO))
|
|
1601
|
+
? MDBX_SUCCESS
|
|
1602
|
+
: (int)GetLastError();
|
|
1603
|
+
} else {
|
|
1604
|
+
LARGE_INTEGER li;
|
|
1605
|
+
li.QuadPart = length;
|
|
1606
|
+
return (SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) && SetEndOfFile(fd)) ? MDBX_SUCCESS : (int)GetLastError();
|
|
1607
|
+
}
|
|
1608
|
+
#else
|
|
1609
|
+
STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
|
|
1610
|
+
return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno;
|
|
1611
|
+
#endif
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos) {
|
|
1615
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1616
|
+
LARGE_INTEGER li;
|
|
1617
|
+
li.QuadPart = pos;
|
|
1618
|
+
return SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) ? MDBX_SUCCESS : (int)GetLastError();
|
|
1619
|
+
#else
|
|
1620
|
+
STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems");
|
|
1621
|
+
return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS;
|
|
1622
|
+
#endif
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
/*----------------------------------------------------------------------------*/
|
|
1626
|
+
|
|
1627
|
+
MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *),
|
|
1628
|
+
void *arg) {
|
|
1629
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1630
|
+
*thread = CreateThread(nullptr, 0, start_routine, arg, 0, nullptr);
|
|
1631
|
+
return *thread ? MDBX_SUCCESS : (int)GetLastError();
|
|
1632
|
+
#else
|
|
1633
|
+
return pthread_create(thread, nullptr, start_routine, arg);
|
|
1634
|
+
#endif
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
MDBX_INTERNAL int osal_thread_join(osal_thread_t thread) {
|
|
1638
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1639
|
+
DWORD code = WaitForSingleObject(thread, INFINITE);
|
|
1640
|
+
return waitstatus2errcode(code);
|
|
1641
|
+
#else
|
|
1642
|
+
void *unused_retval = &unused_retval;
|
|
1643
|
+
return pthread_join(thread, &unused_retval);
|
|
1644
|
+
#endif
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
/*----------------------------------------------------------------------------*/
|
|
1648
|
+
|
|
1649
|
+
MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits) {
|
|
1650
|
+
if (!MDBX_MMAP_NEEDS_JOLT && mode_bits == MDBX_SYNC_NONE)
|
|
1651
|
+
return MDBX_SUCCESS;
|
|
1652
|
+
|
|
1653
|
+
void *ptr = ptr_disp(map->base, offset);
|
|
1654
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1655
|
+
if (!FlushViewOfFile(ptr, length))
|
|
1656
|
+
return (int)GetLastError();
|
|
1657
|
+
if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(map->fd))
|
|
1658
|
+
return (int)GetLastError();
|
|
1659
|
+
#else
|
|
1660
|
+
#if defined(__linux__) || defined(__gnu_linux__)
|
|
1661
|
+
/* Since Linux 2.6.19, MS_ASYNC is in fact a no-op. The kernel properly
|
|
1662
|
+
* tracks dirty pages and flushes ones as necessary. */
|
|
1663
|
+
//
|
|
1664
|
+
// However, this behavior may be changed in custom kernels,
|
|
1665
|
+
// so just leave such optimization to the libc discretion.
|
|
1666
|
+
// NOTE: The MDBX_MMAP_NEEDS_JOLT must be defined to 1 for such cases.
|
|
1667
|
+
//
|
|
1668
|
+
// assert(mdbx.linux_kernel_version > 0x02061300);
|
|
1669
|
+
// if (mode_bits <= MDBX_SYNC_KICK)
|
|
1670
|
+
// return MDBX_SUCCESS;
|
|
1671
|
+
#endif /* Linux */
|
|
1672
|
+
if (msync(ptr, length, (mode_bits & MDBX_SYNC_DATA) ? MS_SYNC : MS_ASYNC))
|
|
1673
|
+
return errno;
|
|
1674
|
+
if ((mode_bits & MDBX_SYNC_SIZE) && fsync(map->fd))
|
|
1675
|
+
return errno;
|
|
1676
|
+
#endif
|
|
1677
|
+
return MDBX_SUCCESS;
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err) {
|
|
1681
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1682
|
+
(void)pathname;
|
|
1683
|
+
(void)err;
|
|
1684
|
+
if (!imports.GetVolumeInformationByHandleW)
|
|
1685
|
+
return MDBX_ENOSYS;
|
|
1686
|
+
DWORD unused, flags;
|
|
1687
|
+
if (!imports.GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused, &flags, nullptr, 0))
|
|
1688
|
+
return (int)GetLastError();
|
|
1689
|
+
if ((flags & FILE_READ_ONLY_VOLUME) == 0)
|
|
1690
|
+
return MDBX_EACCESS;
|
|
1691
|
+
#else
|
|
1692
|
+
struct statvfs info;
|
|
1693
|
+
if (err != MDBX_ENOFILE) {
|
|
1694
|
+
if (statvfs(pathname, &info) == 0)
|
|
1695
|
+
return (info.f_flag & ST_RDONLY) ? MDBX_SUCCESS : err;
|
|
1696
|
+
if (errno != MDBX_ENOFILE)
|
|
1697
|
+
return errno;
|
|
1698
|
+
}
|
|
1699
|
+
if (fstatvfs(handle, &info))
|
|
1700
|
+
return errno;
|
|
1701
|
+
if ((info.f_flag & ST_RDONLY) == 0)
|
|
1702
|
+
return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err;
|
|
1703
|
+
#endif /* !Windows */
|
|
1704
|
+
return MDBX_SUCCESS;
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle) {
|
|
1708
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1709
|
+
(void)handle;
|
|
1710
|
+
#else
|
|
1711
|
+
struct statfs statfs_info;
|
|
1712
|
+
if (fstatfs(handle, &statfs_info))
|
|
1713
|
+
return errno;
|
|
1714
|
+
|
|
1715
|
+
#if defined(__OpenBSD__)
|
|
1716
|
+
const unsigned type = 0;
|
|
1717
|
+
#else
|
|
1718
|
+
const unsigned type = statfs_info.f_type;
|
|
1719
|
+
#endif
|
|
1720
|
+
switch (type) {
|
|
1721
|
+
case 0x28cd3d45 /* CRAMFS_MAGIC */:
|
|
1722
|
+
case 0x858458f6 /* RAMFS_MAGIC */:
|
|
1723
|
+
case 0x01021994 /* TMPFS_MAGIC */:
|
|
1724
|
+
case 0x73717368 /* SQUASHFS_MAGIC */:
|
|
1725
|
+
case 0x7275 /* ROMFS_MAGIC */:
|
|
1726
|
+
return MDBX_RESULT_TRUE;
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \
|
|
1730
|
+
defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \
|
|
1731
|
+
defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN)
|
|
1732
|
+
const char *const name = statfs_info.f_fstypename;
|
|
1733
|
+
const size_t name_len = sizeof(statfs_info.f_fstypename);
|
|
1734
|
+
#else
|
|
1735
|
+
const char *const name = "";
|
|
1736
|
+
const size_t name_len = 0;
|
|
1737
|
+
#endif
|
|
1738
|
+
if (name_len) {
|
|
1739
|
+
if (strncasecmp("tmpfs", name, 6) == 0 || strncasecmp("mfs", name, 4) == 0 || strncasecmp("ramfs", name, 6) == 0 ||
|
|
1740
|
+
strncasecmp("romfs", name, 6) == 0)
|
|
1741
|
+
return MDBX_RESULT_TRUE;
|
|
1742
|
+
}
|
|
1743
|
+
#endif /* !Windows */
|
|
1744
|
+
|
|
1745
|
+
return MDBX_RESULT_FALSE;
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags) {
|
|
1749
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
1750
|
+
if (globals.running_under_Wine && !(flags & MDBX_EXCLUSIVE))
|
|
1751
|
+
return ERROR_NOT_CAPABLE /* workaround for Wine */;
|
|
1752
|
+
|
|
1753
|
+
if (GetFileType(handle) != FILE_TYPE_DISK)
|
|
1754
|
+
return ERROR_FILE_OFFLINE;
|
|
1755
|
+
|
|
1756
|
+
if (imports.GetFileInformationByHandleEx) {
|
|
1757
|
+
FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo;
|
|
1758
|
+
if (imports.GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, &RemoteProtocolInfo,
|
|
1759
|
+
sizeof(RemoteProtocolInfo))) {
|
|
1760
|
+
if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && !(flags & MDBX_RDONLY))
|
|
1761
|
+
return ERROR_FILE_OFFLINE;
|
|
1762
|
+
if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && !(flags & MDBX_EXCLUSIVE))
|
|
1763
|
+
return MDBX_EREMOTE;
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
if (imports.NtFsControlFile) {
|
|
1768
|
+
NTSTATUS rc;
|
|
1769
|
+
struct {
|
|
1770
|
+
WOF_EXTERNAL_INFO wof_info;
|
|
1771
|
+
union {
|
|
1772
|
+
WIM_PROVIDER_EXTERNAL_INFO wim_info;
|
|
1773
|
+
FILE_PROVIDER_EXTERNAL_INFO_V1 file_info;
|
|
1774
|
+
};
|
|
1775
|
+
size_t reserved_for_microsoft_madness[42];
|
|
1776
|
+
} GetExternalBacking_OutputBuffer;
|
|
1777
|
+
IO_STATUS_BLOCK StatusBlock;
|
|
1778
|
+
rc = imports.NtFsControlFile(handle, nullptr, nullptr, nullptr, &StatusBlock, FSCTL_GET_EXTERNAL_BACKING, nullptr,
|
|
1779
|
+
0, &GetExternalBacking_OutputBuffer, sizeof(GetExternalBacking_OutputBuffer));
|
|
1780
|
+
if (NT_SUCCESS(rc)) {
|
|
1781
|
+
if (!(flags & MDBX_EXCLUSIVE))
|
|
1782
|
+
return MDBX_EREMOTE;
|
|
1783
|
+
} else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && rc != STATUS_INVALID_DEVICE_REQUEST &&
|
|
1784
|
+
rc != STATUS_NOT_SUPPORTED)
|
|
1785
|
+
return ntstatus2errcode(rc);
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
if (imports.GetVolumeInformationByHandleW && imports.GetFinalPathNameByHandleW) {
|
|
1789
|
+
WCHAR *PathBuffer = osal_malloc(sizeof(WCHAR) * INT16_MAX);
|
|
1790
|
+
if (!PathBuffer)
|
|
1791
|
+
return MDBX_ENOMEM;
|
|
1792
|
+
|
|
1793
|
+
int rc = MDBX_SUCCESS;
|
|
1794
|
+
DWORD VolumeSerialNumber, FileSystemFlags;
|
|
1795
|
+
if (!imports.GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, &VolumeSerialNumber, nullptr,
|
|
1796
|
+
&FileSystemFlags, nullptr, 0)) {
|
|
1797
|
+
rc = (int)GetLastError();
|
|
1798
|
+
goto bailout;
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
if ((flags & MDBX_RDONLY) == 0) {
|
|
1802
|
+
if (FileSystemFlags & (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | FILE_VOLUME_IS_COMPRESSED)) {
|
|
1803
|
+
rc = MDBX_EREMOTE;
|
|
1804
|
+
goto bailout;
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_NT)) {
|
|
1809
|
+
if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) {
|
|
1810
|
+
if (!(flags & MDBX_EXCLUSIVE)) {
|
|
1811
|
+
rc = MDBX_EREMOTE;
|
|
1812
|
+
goto bailout;
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
|
|
1817
|
+
if (F_ISSET(flags, MDBX_RDONLY | MDBX_EXCLUSIVE) && (FileSystemFlags & FILE_READ_ONLY_VOLUME)) {
|
|
1818
|
+
/* without-LCK (exclusive readonly) mode for DB on a read-only volume */
|
|
1819
|
+
goto bailout;
|
|
1820
|
+
}
|
|
1821
|
+
|
|
1822
|
+
if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS)) {
|
|
1823
|
+
UINT DriveType = GetDriveTypeW(PathBuffer);
|
|
1824
|
+
if (DriveType == DRIVE_NO_ROOT_DIR && _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 &&
|
|
1825
|
+
_wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) {
|
|
1826
|
+
PathBuffer[7] = 0;
|
|
1827
|
+
DriveType = GetDriveTypeW(PathBuffer + 4);
|
|
1828
|
+
}
|
|
1829
|
+
switch (DriveType) {
|
|
1830
|
+
case DRIVE_CDROM:
|
|
1831
|
+
if (flags & MDBX_RDONLY)
|
|
1832
|
+
break;
|
|
1833
|
+
// fall through
|
|
1834
|
+
case DRIVE_UNKNOWN:
|
|
1835
|
+
case DRIVE_NO_ROOT_DIR:
|
|
1836
|
+
case DRIVE_REMOTE:
|
|
1837
|
+
default:
|
|
1838
|
+
if (!(flags & MDBX_EXCLUSIVE))
|
|
1839
|
+
rc = MDBX_EREMOTE;
|
|
1840
|
+
// fall through
|
|
1841
|
+
case DRIVE_REMOVABLE:
|
|
1842
|
+
case DRIVE_FIXED:
|
|
1843
|
+
case DRIVE_RAMDISK:
|
|
1844
|
+
break;
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
bailout:
|
|
1849
|
+
osal_free(PathBuffer);
|
|
1850
|
+
return rc;
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
#else
|
|
1854
|
+
|
|
1855
|
+
struct statvfs statvfs_info;
|
|
1856
|
+
if (fstatvfs(handle, &statvfs_info))
|
|
1857
|
+
return errno;
|
|
1858
|
+
#if defined(ST_LOCAL) || defined(ST_EXPORTED)
|
|
1859
|
+
const unsigned long st_flags = statvfs_info.f_flag;
|
|
1860
|
+
#endif /* ST_LOCAL || ST_EXPORTED */
|
|
1861
|
+
|
|
1862
|
+
#if defined(__NetBSD__)
|
|
1863
|
+
const unsigned type = 0;
|
|
1864
|
+
const char *const name = statvfs_info.f_fstypename;
|
|
1865
|
+
const size_t name_len = VFS_NAMELEN;
|
|
1866
|
+
#elif defined(_AIX) || defined(__OS400__)
|
|
1867
|
+
const char *const name = statvfs_info.f_basetype;
|
|
1868
|
+
const size_t name_len = sizeof(statvfs_info.f_basetype);
|
|
1869
|
+
struct stat st;
|
|
1870
|
+
if (fstat(handle, &st))
|
|
1871
|
+
return errno;
|
|
1872
|
+
const unsigned type = st.st_vfstype;
|
|
1873
|
+
if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE))
|
|
1874
|
+
return MDBX_EREMOTE;
|
|
1875
|
+
#elif defined(FSTYPSZ) || defined(_FSTYPSZ)
|
|
1876
|
+
const unsigned type = 0;
|
|
1877
|
+
const char *const name = statvfs_info.f_basetype;
|
|
1878
|
+
const size_t name_len = sizeof(statvfs_info.f_basetype);
|
|
1879
|
+
#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ)
|
|
1880
|
+
const unsigned type = 0;
|
|
1881
|
+
struct stat st;
|
|
1882
|
+
if (fstat(handle, &st))
|
|
1883
|
+
return errno;
|
|
1884
|
+
const char *const name = st.st_fstype;
|
|
1885
|
+
const size_t name_len = strlen(name);
|
|
1886
|
+
#else
|
|
1887
|
+
struct statfs statfs_info;
|
|
1888
|
+
if (fstatfs(handle, &statfs_info))
|
|
1889
|
+
return errno;
|
|
1890
|
+
#if defined(__OpenBSD__)
|
|
1891
|
+
const unsigned type = 0;
|
|
1892
|
+
#else
|
|
1893
|
+
const unsigned type = statfs_info.f_type;
|
|
1894
|
+
#endif
|
|
1895
|
+
#if defined(MNT_LOCAL) || defined(MNT_EXPORTED)
|
|
1896
|
+
const unsigned long mnt_flags = statfs_info.f_flags;
|
|
1897
|
+
#endif /* MNT_LOCAL || MNT_EXPORTED */
|
|
1898
|
+
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \
|
|
1899
|
+
defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \
|
|
1900
|
+
defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN)
|
|
1901
|
+
const char *const name = statfs_info.f_fstypename;
|
|
1902
|
+
const size_t name_len = sizeof(statfs_info.f_fstypename);
|
|
1903
|
+
#elif defined(__ANDROID_API__) && __ANDROID_API__ < 21
|
|
1904
|
+
const char *const name = "";
|
|
1905
|
+
const unsigned name_len = 0;
|
|
1906
|
+
#else
|
|
1907
|
+
|
|
1908
|
+
const char *name = "";
|
|
1909
|
+
unsigned name_len = 0;
|
|
1910
|
+
|
|
1911
|
+
struct stat st;
|
|
1912
|
+
if (fstat(handle, &st))
|
|
1913
|
+
return errno;
|
|
1914
|
+
|
|
1915
|
+
char pathbuf[PATH_MAX];
|
|
1916
|
+
FILE *mounted = nullptr;
|
|
1917
|
+
#if defined(__linux__) || defined(__gnu_linux__)
|
|
1918
|
+
mounted = setmntent("/proc/mounts", "r");
|
|
1919
|
+
#endif /* Linux */
|
|
1920
|
+
if (!mounted)
|
|
1921
|
+
mounted = setmntent("/etc/mtab", "r");
|
|
1922
|
+
if (mounted) {
|
|
1923
|
+
const struct mntent *ent;
|
|
1924
|
+
#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \
|
|
1925
|
+
(defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19))
|
|
1926
|
+
struct mntent entbuf;
|
|
1927
|
+
const bool should_copy = false;
|
|
1928
|
+
while (nullptr != (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf))))
|
|
1929
|
+
#else
|
|
1930
|
+
const bool should_copy = true;
|
|
1931
|
+
while (nullptr != (ent = getmntent(mounted)))
|
|
1932
|
+
#endif
|
|
1933
|
+
{
|
|
1934
|
+
struct stat mnt;
|
|
1935
|
+
if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) {
|
|
1936
|
+
if (should_copy) {
|
|
1937
|
+
name = strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1);
|
|
1938
|
+
pathbuf[name_len] = 0;
|
|
1939
|
+
} else {
|
|
1940
|
+
name = ent->mnt_fsname;
|
|
1941
|
+
name_len = strlen(name);
|
|
1942
|
+
}
|
|
1943
|
+
break;
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
endmntent(mounted);
|
|
1947
|
+
}
|
|
1948
|
+
#endif /* !xBSD && !Android/Bionic */
|
|
1949
|
+
#endif
|
|
1950
|
+
|
|
1951
|
+
if (name_len) {
|
|
1952
|
+
if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || strncasecmp("cifs", name, name_len) == 0 ||
|
|
1953
|
+
strncasecmp("ncpfs", name, name_len) == 0 || strncasecmp("smbfs", name, name_len) == 0 ||
|
|
1954
|
+
strcasecmp("9P" /* WSL2 */, name) == 0 ||
|
|
1955
|
+
((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && strncasecmp("fuseblk", name, name_len) != 0)) &&
|
|
1956
|
+
!(flags & MDBX_EXCLUSIVE))
|
|
1957
|
+
return MDBX_EREMOTE;
|
|
1958
|
+
if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || strcasecmp("sshfs", name) == 0)
|
|
1959
|
+
return MDBX_EREMOTE;
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
#ifdef ST_LOCAL
|
|
1963
|
+
if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
|
|
1964
|
+
return MDBX_EREMOTE;
|
|
1965
|
+
#elif defined(MNT_LOCAL)
|
|
1966
|
+
if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE))
|
|
1967
|
+
return MDBX_EREMOTE;
|
|
1968
|
+
#endif /* ST/MNT_LOCAL */
|
|
1969
|
+
|
|
1970
|
+
#ifdef ST_EXPORTED
|
|
1971
|
+
if ((st_flags & ST_EXPORTED) != 0 && !(flags & (MDBX_RDONLY | MDBX_EXCLUSIVE))))
|
|
1972
|
+
return MDBX_RESULT_TRUE;
|
|
1973
|
+
#elif defined(MNT_EXPORTED)
|
|
1974
|
+
if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)))
|
|
1975
|
+
return MDBX_RESULT_TRUE;
|
|
1976
|
+
#endif /* ST/MNT_EXPORTED */
|
|
1977
|
+
|
|
1978
|
+
switch (type) {
|
|
1979
|
+
case 0xFF534D42 /* CIFS_MAGIC_NUMBER */:
|
|
1980
|
+
case 0x6969 /* NFS_SUPER_MAGIC */:
|
|
1981
|
+
case 0x564c /* NCP_SUPER_MAGIC */:
|
|
1982
|
+
case 0x517B /* SMB_SUPER_MAGIC */:
|
|
1983
|
+
#if defined(__digital__) || defined(__osf__) || defined(__osf)
|
|
1984
|
+
case 0x0E /* Tru64 NFS */:
|
|
1985
|
+
#endif
|
|
1986
|
+
#ifdef ST_FST_NFS
|
|
1987
|
+
case ST_FST_NFS:
|
|
1988
|
+
#endif
|
|
1989
|
+
if ((flags & MDBX_EXCLUSIVE) == 0)
|
|
1990
|
+
return MDBX_EREMOTE;
|
|
1991
|
+
case 0:
|
|
1992
|
+
default:
|
|
1993
|
+
break;
|
|
1994
|
+
}
|
|
1995
|
+
#endif /* Unix */
|
|
1996
|
+
|
|
1997
|
+
return MDBX_SUCCESS;
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
static int check_mmap_limit(const size_t limit) {
|
|
2001
|
+
const bool should_check =
|
|
2002
|
+
#if defined(__SANITIZE_ADDRESS__)
|
|
2003
|
+
true;
|
|
2004
|
+
#else
|
|
2005
|
+
RUNNING_ON_VALGRIND;
|
|
2006
|
+
#endif /* __SANITIZE_ADDRESS__ */
|
|
2007
|
+
|
|
2008
|
+
if (should_check) {
|
|
2009
|
+
intptr_t pagesize, total_ram_pages, avail_ram_pages;
|
|
2010
|
+
int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages);
|
|
2011
|
+
if (unlikely(err != MDBX_SUCCESS))
|
|
2012
|
+
return err;
|
|
2013
|
+
|
|
2014
|
+
const int log2page = log2n_powerof2(pagesize);
|
|
2015
|
+
if ((limit >> (log2page + 7)) > (size_t)total_ram_pages || (limit >> (log2page + 6)) > (size_t)avail_ram_pages) {
|
|
2016
|
+
ERROR("%s (%zu pages) is too large for available (%zu pages) or total "
|
|
2017
|
+
"(%zu pages) system RAM",
|
|
2018
|
+
"database upper size limit", limit >> log2page, avail_ram_pages, total_ram_pages);
|
|
2019
|
+
return MDBX_TOO_LARGE;
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
return MDBX_SUCCESS;
|
|
2024
|
+
}
|
|
2025
|
+
|
|
2026
|
+
MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options,
|
|
2027
|
+
const pathchar_t *pathname4logging) {
|
|
2028
|
+
assert(size <= limit);
|
|
2029
|
+
map->limit = 0;
|
|
2030
|
+
map->current = 0;
|
|
2031
|
+
map->base = nullptr;
|
|
2032
|
+
map->filesize = 0;
|
|
2033
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2034
|
+
map->section = nullptr;
|
|
2035
|
+
#endif /* Windows */
|
|
2036
|
+
|
|
2037
|
+
int err = osal_check_fs_local(map->fd, flags);
|
|
2038
|
+
if (unlikely(err != MDBX_SUCCESS)) {
|
|
2039
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2040
|
+
if (globals.running_under_Wine)
|
|
2041
|
+
NOTICE("%s", "Please use native Linux application or WSL at least, instead of trouble-full Wine!");
|
|
2042
|
+
#endif /* Windows */
|
|
2043
|
+
switch (err) {
|
|
2044
|
+
case MDBX_RESULT_TRUE:
|
|
2045
|
+
#if MDBX_ENABLE_NON_READONLY_EXPORT
|
|
2046
|
+
WARNING("%" MDBX_PRIsPATH " is exported via NFS, avoid using the file on a remote side!", pathname4logging);
|
|
2047
|
+
break;
|
|
2048
|
+
#else
|
|
2049
|
+
ERROR("%" MDBX_PRIsPATH " is exported via NFS", pathname4logging);
|
|
2050
|
+
return MDBX_EREMOTE;
|
|
2051
|
+
#endif /* MDBX_PROHIBIT_NON_READONLY_EXPORT */
|
|
2052
|
+
case MDBX_EREMOTE:
|
|
2053
|
+
ERROR("%" MDBX_PRIsPATH " is on a remote file system, the %s is required", pathname4logging, "MDBX_EXCLUSIVE");
|
|
2054
|
+
__fallthrough /* fall through */;
|
|
2055
|
+
default:
|
|
2056
|
+
return err;
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2060
|
+
err = check_mmap_limit(limit);
|
|
2061
|
+
if (unlikely(err != MDBX_SUCCESS))
|
|
2062
|
+
return err;
|
|
2063
|
+
|
|
2064
|
+
if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_TRUNCATE) != 0) {
|
|
2065
|
+
err = osal_ftruncate(map->fd, size);
|
|
2066
|
+
VERBOSE("ftruncate %zu, err %d", size, err);
|
|
2067
|
+
if (err != MDBX_SUCCESS)
|
|
2068
|
+
return err;
|
|
2069
|
+
map->filesize = size;
|
|
2070
|
+
#if !(defined(_WIN32) || defined(_WIN64))
|
|
2071
|
+
map->current = size;
|
|
2072
|
+
#endif /* !Windows */
|
|
2073
|
+
} else {
|
|
2074
|
+
err = osal_filesize(map->fd, &map->filesize);
|
|
2075
|
+
VERBOSE("filesize %" PRIu64 ", err %d", map->filesize, err);
|
|
2076
|
+
if (err != MDBX_SUCCESS)
|
|
2077
|
+
return err;
|
|
2078
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2079
|
+
if (map->filesize < size) {
|
|
2080
|
+
WARNING("file size (%zu) less than requested for mapping (%zu)", (size_t)map->filesize, size);
|
|
2081
|
+
size = (size_t)map->filesize;
|
|
2082
|
+
}
|
|
2083
|
+
#else
|
|
2084
|
+
map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
|
|
2085
|
+
#endif /* !Windows */
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2089
|
+
LARGE_INTEGER SectionSize;
|
|
2090
|
+
SectionSize.QuadPart = size;
|
|
2091
|
+
err = NtCreateSection(&map->section,
|
|
2092
|
+
/* DesiredAccess */
|
|
2093
|
+
(flags & MDBX_WRITEMAP)
|
|
2094
|
+
? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE
|
|
2095
|
+
: SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
|
|
2096
|
+
/* ObjectAttributes */ nullptr,
|
|
2097
|
+
/* MaximumSize (InitialSize) */ &SectionSize,
|
|
2098
|
+
/* SectionPageProtection */
|
|
2099
|
+
(flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
|
|
2100
|
+
/* AllocationAttributes */ SEC_RESERVE, map->fd);
|
|
2101
|
+
if (!NT_SUCCESS(err))
|
|
2102
|
+
return ntstatus2errcode(err);
|
|
2103
|
+
|
|
2104
|
+
SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 : globals.running_under_Wine ? size : limit;
|
|
2105
|
+
err = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base,
|
|
2106
|
+
/* ZeroBits */ 0,
|
|
2107
|
+
/* CommitSize */ 0,
|
|
2108
|
+
/* SectionOffset */ nullptr, &ViewSize,
|
|
2109
|
+
/* InheritDisposition */ ViewUnmap,
|
|
2110
|
+
/* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
|
|
2111
|
+
/* Win32Protect */
|
|
2112
|
+
(flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
|
|
2113
|
+
if (!NT_SUCCESS(err)) {
|
|
2114
|
+
NtClose(map->section);
|
|
2115
|
+
map->section = 0;
|
|
2116
|
+
map->base = nullptr;
|
|
2117
|
+
return ntstatus2errcode(err);
|
|
2118
|
+
}
|
|
2119
|
+
assert(map->base != MAP_FAILED);
|
|
2120
|
+
|
|
2121
|
+
map->current = (size_t)SectionSize.QuadPart;
|
|
2122
|
+
map->limit = ViewSize;
|
|
2123
|
+
|
|
2124
|
+
#else /* Windows */
|
|
2125
|
+
|
|
2126
|
+
#ifndef MAP_TRYFIXED
|
|
2127
|
+
#define MAP_TRYFIXED 0
|
|
2128
|
+
#endif
|
|
2129
|
+
|
|
2130
|
+
#ifndef MAP_HASSEMAPHORE
|
|
2131
|
+
#define MAP_HASSEMAPHORE 0
|
|
2132
|
+
#endif
|
|
2133
|
+
|
|
2134
|
+
#ifndef MAP_CONCEAL
|
|
2135
|
+
#define MAP_CONCEAL 0
|
|
2136
|
+
#endif
|
|
2137
|
+
|
|
2138
|
+
#ifndef MAP_NOSYNC
|
|
2139
|
+
#define MAP_NOSYNC 0
|
|
2140
|
+
#endif
|
|
2141
|
+
|
|
2142
|
+
#ifndef MAP_FIXED_NOREPLACE
|
|
2143
|
+
#define MAP_FIXED_NOREPLACE 0
|
|
2144
|
+
#endif
|
|
2145
|
+
|
|
2146
|
+
#ifndef MAP_NORESERVE
|
|
2147
|
+
#define MAP_NORESERVE 0
|
|
2148
|
+
#endif
|
|
2149
|
+
|
|
2150
|
+
map->base = mmap(nullptr, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ,
|
|
2151
|
+
MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) |
|
|
2152
|
+
((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC : MAP_CONCEAL),
|
|
2153
|
+
map->fd, 0);
|
|
2154
|
+
|
|
2155
|
+
if (unlikely(map->base == MAP_FAILED)) {
|
|
2156
|
+
map->limit = 0;
|
|
2157
|
+
map->current = 0;
|
|
2158
|
+
map->base = nullptr;
|
|
2159
|
+
assert(errno != 0);
|
|
2160
|
+
return errno;
|
|
2161
|
+
}
|
|
2162
|
+
map->limit = limit;
|
|
2163
|
+
|
|
2164
|
+
#ifdef MADV_DONTFORK
|
|
2165
|
+
if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0))
|
|
2166
|
+
return errno;
|
|
2167
|
+
#endif /* MADV_DONTFORK */
|
|
2168
|
+
#ifdef MADV_NOHUGEPAGE
|
|
2169
|
+
(void)madvise(map->base, map->limit, MADV_NOHUGEPAGE);
|
|
2170
|
+
#endif /* MADV_NOHUGEPAGE */
|
|
2171
|
+
|
|
2172
|
+
#endif /* ! Windows */
|
|
2173
|
+
|
|
2174
|
+
VALGRIND_MAKE_MEM_DEFINED(map->base, map->current);
|
|
2175
|
+
MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->current);
|
|
2176
|
+
return MDBX_SUCCESS;
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2179
|
+
MDBX_INTERNAL int osal_munmap(osal_mmap_t *map) {
|
|
2180
|
+
VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
|
|
2181
|
+
/* Unpoisoning is required for ASAN to avoid false-positive diagnostic
|
|
2182
|
+
* when this memory will re-used by malloc or another mmapping.
|
|
2183
|
+
* See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */
|
|
2184
|
+
MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base,
|
|
2185
|
+
(map->filesize && map->filesize < map->limit) ? map->filesize : map->limit);
|
|
2186
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2187
|
+
if (map->section)
|
|
2188
|
+
NtClose(map->section);
|
|
2189
|
+
NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->base);
|
|
2190
|
+
if (!NT_SUCCESS(rc))
|
|
2191
|
+
ntstatus2errcode(rc);
|
|
2192
|
+
#else
|
|
2193
|
+
if (unlikely(munmap(map->base, map->limit))) {
|
|
2194
|
+
assert(errno != 0);
|
|
2195
|
+
return errno;
|
|
2196
|
+
}
|
|
2197
|
+
#endif /* ! Windows */
|
|
2198
|
+
|
|
2199
|
+
map->limit = 0;
|
|
2200
|
+
map->current = 0;
|
|
2201
|
+
map->base = nullptr;
|
|
2202
|
+
return MDBX_SUCCESS;
|
|
2203
|
+
}
|
|
2204
|
+
|
|
2205
|
+
MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit) {
|
|
2206
|
+
int rc = osal_filesize(map->fd, &map->filesize);
|
|
2207
|
+
VERBOSE("flags 0x%x, size %zu, limit %zu, filesize %" PRIu64, flags, size, limit, map->filesize);
|
|
2208
|
+
assert(size <= limit);
|
|
2209
|
+
if (rc != MDBX_SUCCESS) {
|
|
2210
|
+
map->filesize = 0;
|
|
2211
|
+
return rc;
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2215
|
+
assert(size != map->current || limit != map->limit || size < map->filesize);
|
|
2216
|
+
|
|
2217
|
+
NTSTATUS status;
|
|
2218
|
+
LARGE_INTEGER SectionSize;
|
|
2219
|
+
int err;
|
|
2220
|
+
|
|
2221
|
+
if (limit == map->limit && size > map->current) {
|
|
2222
|
+
if ((flags & MDBX_RDONLY) && map->filesize >= size) {
|
|
2223
|
+
map->current = size;
|
|
2224
|
+
return MDBX_SUCCESS;
|
|
2225
|
+
} else if (!(flags & MDBX_RDONLY) &&
|
|
2226
|
+
/* workaround for Wine */ imports.NtExtendSection) {
|
|
2227
|
+
/* growth rw-section */
|
|
2228
|
+
SectionSize.QuadPart = size;
|
|
2229
|
+
status = imports.NtExtendSection(map->section, &SectionSize);
|
|
2230
|
+
if (!NT_SUCCESS(status))
|
|
2231
|
+
return ntstatus2errcode(status);
|
|
2232
|
+
map->current = size;
|
|
2233
|
+
if (map->filesize < size)
|
|
2234
|
+
map->filesize = size;
|
|
2235
|
+
return MDBX_SUCCESS;
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
|
|
2239
|
+
if (limit > map->limit) {
|
|
2240
|
+
err = check_mmap_limit(limit);
|
|
2241
|
+
if (unlikely(err != MDBX_SUCCESS))
|
|
2242
|
+
return err;
|
|
2243
|
+
|
|
2244
|
+
/* check ability of address space for growth before unmap */
|
|
2245
|
+
PVOID BaseAddress = (PBYTE)map->base + map->limit;
|
|
2246
|
+
SIZE_T RegionSize = limit - map->limit;
|
|
2247
|
+
status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, &RegionSize, MEM_RESERVE, PAGE_NOACCESS);
|
|
2248
|
+
if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
|
|
2249
|
+
return MDBX_UNABLE_EXTEND_MAPSIZE;
|
|
2250
|
+
if (!NT_SUCCESS(status))
|
|
2251
|
+
return ntstatus2errcode(status);
|
|
2252
|
+
|
|
2253
|
+
status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, MEM_RELEASE);
|
|
2254
|
+
if (!NT_SUCCESS(status))
|
|
2255
|
+
return ntstatus2errcode(status);
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
/* Windows unable:
|
|
2259
|
+
* - shrink a mapped file;
|
|
2260
|
+
* - change size of mapped view;
|
|
2261
|
+
* - extend read-only mapping;
|
|
2262
|
+
* Therefore we should unmap/map entire section. */
|
|
2263
|
+
if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) {
|
|
2264
|
+
if (size <= map->current && limit == map->limit)
|
|
2265
|
+
return MDBX_SUCCESS;
|
|
2266
|
+
return MDBX_EPERM;
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
/* Unpoisoning is required for ASAN to avoid false-positive diagnostic
|
|
2270
|
+
* when this memory will re-used by malloc or another mmapping.
|
|
2271
|
+
* See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */
|
|
2272
|
+
MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->limit);
|
|
2273
|
+
status = NtUnmapViewOfSection(GetCurrentProcess(), map->base);
|
|
2274
|
+
if (!NT_SUCCESS(status))
|
|
2275
|
+
return ntstatus2errcode(status);
|
|
2276
|
+
status = NtClose(map->section);
|
|
2277
|
+
map->section = nullptr;
|
|
2278
|
+
PVOID ReservedAddress = nullptr;
|
|
2279
|
+
SIZE_T ReservedSize = limit;
|
|
2280
|
+
|
|
2281
|
+
if (!NT_SUCCESS(status)) {
|
|
2282
|
+
bailout_ntstatus:
|
|
2283
|
+
err = ntstatus2errcode(status);
|
|
2284
|
+
map->base = nullptr;
|
|
2285
|
+
map->current = map->limit = 0;
|
|
2286
|
+
if (ReservedAddress) {
|
|
2287
|
+
ReservedSize = 0;
|
|
2288
|
+
status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE);
|
|
2289
|
+
assert(NT_SUCCESS(status));
|
|
2290
|
+
(void)status;
|
|
2291
|
+
}
|
|
2292
|
+
return err;
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2295
|
+
retry_file_and_section:
|
|
2296
|
+
/* resizing of the file may take a while,
|
|
2297
|
+
* therefore we reserve address space to avoid occupy it by other threads */
|
|
2298
|
+
ReservedAddress = map->base;
|
|
2299
|
+
status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, &ReservedSize, MEM_RESERVE, PAGE_NOACCESS);
|
|
2300
|
+
if (!NT_SUCCESS(status)) {
|
|
2301
|
+
ReservedAddress = nullptr;
|
|
2302
|
+
if (status != (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
|
|
2303
|
+
goto bailout_ntstatus /* no way to recovery */;
|
|
2304
|
+
|
|
2305
|
+
if (flags & MDBX_MRESIZE_MAY_MOVE)
|
|
2306
|
+
/* the base address could be changed */
|
|
2307
|
+
map->base = nullptr;
|
|
2308
|
+
}
|
|
2309
|
+
|
|
2310
|
+
if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) {
|
|
2311
|
+
err = osal_ftruncate(map->fd, size);
|
|
2312
|
+
if (err == MDBX_SUCCESS)
|
|
2313
|
+
map->filesize = size;
|
|
2314
|
+
/* ignore error, because Windows unable shrink file
|
|
2315
|
+
* that already mapped (by another process) */
|
|
2316
|
+
}
|
|
2317
|
+
|
|
2318
|
+
SectionSize.QuadPart = size;
|
|
2319
|
+
status = NtCreateSection(&map->section,
|
|
2320
|
+
/* DesiredAccess */
|
|
2321
|
+
(flags & MDBX_WRITEMAP)
|
|
2322
|
+
? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE
|
|
2323
|
+
: SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
|
|
2324
|
+
/* ObjectAttributes */ nullptr,
|
|
2325
|
+
/* MaximumSize (InitialSize) */ &SectionSize,
|
|
2326
|
+
/* SectionPageProtection */
|
|
2327
|
+
(flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
|
|
2328
|
+
/* AllocationAttributes */ SEC_RESERVE, map->fd);
|
|
2329
|
+
|
|
2330
|
+
if (!NT_SUCCESS(status))
|
|
2331
|
+
goto bailout_ntstatus;
|
|
2332
|
+
|
|
2333
|
+
if (ReservedAddress) {
|
|
2334
|
+
/* release reserved address space */
|
|
2335
|
+
ReservedSize = 0;
|
|
2336
|
+
status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE);
|
|
2337
|
+
ReservedAddress = nullptr;
|
|
2338
|
+
if (!NT_SUCCESS(status))
|
|
2339
|
+
goto bailout_ntstatus;
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2342
|
+
retry_mapview:;
|
|
2343
|
+
SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit;
|
|
2344
|
+
status = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base,
|
|
2345
|
+
/* ZeroBits */ 0,
|
|
2346
|
+
/* CommitSize */ 0,
|
|
2347
|
+
/* SectionOffset */ nullptr, &ViewSize,
|
|
2348
|
+
/* InheritDisposition */ ViewUnmap,
|
|
2349
|
+
/* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
|
|
2350
|
+
/* Win32Protect */
|
|
2351
|
+
(flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
|
|
2352
|
+
|
|
2353
|
+
if (!NT_SUCCESS(status)) {
|
|
2354
|
+
if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && map->base &&
|
|
2355
|
+
(flags & MDBX_MRESIZE_MAY_MOVE) != 0) {
|
|
2356
|
+
/* try remap at another base address */
|
|
2357
|
+
map->base = nullptr;
|
|
2358
|
+
goto retry_mapview;
|
|
2359
|
+
}
|
|
2360
|
+
NtClose(map->section);
|
|
2361
|
+
map->section = nullptr;
|
|
2362
|
+
|
|
2363
|
+
if (map->base && (size != map->current || limit != map->limit)) {
|
|
2364
|
+
/* try remap with previously size and limit,
|
|
2365
|
+
* but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */
|
|
2366
|
+
rc = (limit > map->limit) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM;
|
|
2367
|
+
size = map->current;
|
|
2368
|
+
ReservedSize = limit = map->limit;
|
|
2369
|
+
goto retry_file_and_section;
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
/* no way to recovery */
|
|
2373
|
+
goto bailout_ntstatus;
|
|
2374
|
+
}
|
|
2375
|
+
assert(map->base != MAP_FAILED);
|
|
2376
|
+
|
|
2377
|
+
map->current = (size_t)SectionSize.QuadPart;
|
|
2378
|
+
map->limit = ViewSize;
|
|
2379
|
+
|
|
2380
|
+
#else /* Windows */
|
|
2381
|
+
|
|
2382
|
+
if (flags & MDBX_RDONLY) {
|
|
2383
|
+
if (size > map->filesize)
|
|
2384
|
+
rc = MDBX_UNABLE_EXTEND_MAPSIZE;
|
|
2385
|
+
else if (size < map->filesize && map->filesize > limit)
|
|
2386
|
+
rc = MDBX_EPERM;
|
|
2387
|
+
map->current = (map->filesize > limit) ? limit : (size_t)map->filesize;
|
|
2388
|
+
} else {
|
|
2389
|
+
if (size > map->filesize || (size < map->filesize && (flags & txn_shrink_allowed))) {
|
|
2390
|
+
rc = osal_ftruncate(map->fd, size);
|
|
2391
|
+
VERBOSE("ftruncate %zu, err %d", size, rc);
|
|
2392
|
+
if (rc != MDBX_SUCCESS)
|
|
2393
|
+
return rc;
|
|
2394
|
+
map->filesize = size;
|
|
2395
|
+
}
|
|
2396
|
+
|
|
2397
|
+
if (map->current > size) {
|
|
2398
|
+
/* Clearing asan's bitmask for the region which released in shrinking,
|
|
2399
|
+
* since:
|
|
2400
|
+
* - after the shrinking we will get an exception when accessing
|
|
2401
|
+
* this region and (therefore) do not need the help of ASAN.
|
|
2402
|
+
* - this allows us to clear the mask only within the file size
|
|
2403
|
+
* when closing the mapping. */
|
|
2404
|
+
MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr_disp(map->base, size),
|
|
2405
|
+
((map->current < map->limit) ? map->current : map->limit) - size);
|
|
2406
|
+
}
|
|
2407
|
+
map->current = (size < map->limit) ? size : map->limit;
|
|
2408
|
+
}
|
|
2409
|
+
|
|
2410
|
+
if (limit == map->limit)
|
|
2411
|
+
return rc;
|
|
2412
|
+
|
|
2413
|
+
if (limit < map->limit) {
|
|
2414
|
+
/* unmap an excess at end of mapping. */
|
|
2415
|
+
// coverity[offset_free : FALSE]
|
|
2416
|
+
if (unlikely(munmap(ptr_disp(map->base, limit), map->limit - limit))) {
|
|
2417
|
+
assert(errno != 0);
|
|
2418
|
+
return errno;
|
|
2419
|
+
}
|
|
2420
|
+
map->limit = limit;
|
|
2421
|
+
return rc;
|
|
2422
|
+
}
|
|
2423
|
+
|
|
2424
|
+
int err = check_mmap_limit(limit);
|
|
2425
|
+
if (unlikely(err != MDBX_SUCCESS))
|
|
2426
|
+
return err;
|
|
2427
|
+
|
|
2428
|
+
assert(limit > map->limit);
|
|
2429
|
+
void *ptr = MAP_FAILED;
|
|
2430
|
+
|
|
2431
|
+
#if (defined(__linux__) || defined(__gnu_linux__)) && defined(_GNU_SOURCE)
|
|
2432
|
+
ptr = mremap(map->base, map->limit, limit,
|
|
2433
|
+
#if defined(MREMAP_MAYMOVE)
|
|
2434
|
+
(flags & MDBX_MRESIZE_MAY_MOVE) ? MREMAP_MAYMOVE :
|
|
2435
|
+
#endif /* MREMAP_MAYMOVE */
|
|
2436
|
+
0);
|
|
2437
|
+
if (ptr == MAP_FAILED) {
|
|
2438
|
+
err = errno;
|
|
2439
|
+
assert(err != 0);
|
|
2440
|
+
switch (err) {
|
|
2441
|
+
default:
|
|
2442
|
+
return err;
|
|
2443
|
+
case 0 /* paranoia */:
|
|
2444
|
+
case EAGAIN:
|
|
2445
|
+
case ENOMEM:
|
|
2446
|
+
return MDBX_UNABLE_EXTEND_MAPSIZE;
|
|
2447
|
+
case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */:
|
|
2448
|
+
break;
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
#endif /* Linux & _GNU_SOURCE */
|
|
2452
|
+
|
|
2453
|
+
const unsigned mmap_flags =
|
|
2454
|
+
MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0);
|
|
2455
|
+
const unsigned mmap_prot = (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ;
|
|
2456
|
+
|
|
2457
|
+
if (ptr == MAP_FAILED) {
|
|
2458
|
+
/* Try to mmap additional space beyond the end of mapping. */
|
|
2459
|
+
ptr = mmap(ptr_disp(map->base, map->limit), limit - map->limit, mmap_prot, mmap_flags | MAP_FIXED_NOREPLACE,
|
|
2460
|
+
map->fd, map->limit);
|
|
2461
|
+
if (ptr == ptr_disp(map->base, map->limit))
|
|
2462
|
+
/* успешно прилепили отображение в конец */
|
|
2463
|
+
ptr = map->base;
|
|
2464
|
+
else if (ptr != MAP_FAILED) {
|
|
2465
|
+
/* the desired address is busy, unmap unsuitable one */
|
|
2466
|
+
if (unlikely(munmap(ptr, limit - map->limit))) {
|
|
2467
|
+
assert(errno != 0);
|
|
2468
|
+
return errno;
|
|
2469
|
+
}
|
|
2470
|
+
ptr = MAP_FAILED;
|
|
2471
|
+
} else {
|
|
2472
|
+
err = errno;
|
|
2473
|
+
assert(err != 0);
|
|
2474
|
+
switch (err) {
|
|
2475
|
+
default:
|
|
2476
|
+
return err;
|
|
2477
|
+
case 0 /* paranoia */:
|
|
2478
|
+
case EAGAIN:
|
|
2479
|
+
case ENOMEM:
|
|
2480
|
+
return MDBX_UNABLE_EXTEND_MAPSIZE;
|
|
2481
|
+
case EEXIST: /* address busy */
|
|
2482
|
+
case EINVAL: /* kernel don't support MAP_FIXED_NOREPLACE */
|
|
2483
|
+
break;
|
|
2484
|
+
}
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2487
|
+
|
|
2488
|
+
if (ptr == MAP_FAILED) {
|
|
2489
|
+
/* unmap and map again whole region */
|
|
2490
|
+
if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) {
|
|
2491
|
+
/* TODO: Perhaps here it is worth to implement suspend/resume threads
|
|
2492
|
+
* and perform unmap/map as like for Windows. */
|
|
2493
|
+
return MDBX_UNABLE_EXTEND_MAPSIZE;
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
if (unlikely(munmap(map->base, map->limit))) {
|
|
2497
|
+
assert(errno != 0);
|
|
2498
|
+
return errno;
|
|
2499
|
+
}
|
|
2500
|
+
|
|
2501
|
+
// coverity[pass_freed_arg : FALSE]
|
|
2502
|
+
ptr = mmap(map->base, limit, mmap_prot,
|
|
2503
|
+
(flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags
|
|
2504
|
+
: mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED),
|
|
2505
|
+
map->fd, 0);
|
|
2506
|
+
if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) &&
|
|
2507
|
+
!(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
|
|
2508
|
+
// coverity[pass_freed_arg : FALSE]
|
|
2509
|
+
ptr = mmap(map->base, limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0);
|
|
2510
|
+
|
|
2511
|
+
if (unlikely(ptr == MAP_FAILED)) {
|
|
2512
|
+
/* try to restore prev mapping */
|
|
2513
|
+
// coverity[pass_freed_arg : FALSE]
|
|
2514
|
+
ptr = mmap(map->base, map->limit, mmap_prot,
|
|
2515
|
+
(flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags
|
|
2516
|
+
: mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED),
|
|
2517
|
+
map->fd, 0);
|
|
2518
|
+
if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) &&
|
|
2519
|
+
!(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL)
|
|
2520
|
+
// coverity[pass_freed_arg : FALSE]
|
|
2521
|
+
ptr = mmap(map->base, map->limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0);
|
|
2522
|
+
if (unlikely(ptr == MAP_FAILED)) {
|
|
2523
|
+
VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
|
|
2524
|
+
/* Unpoisoning is required for ASAN to avoid false-positive diagnostic
|
|
2525
|
+
* when this memory will re-used by malloc or another mmapping.
|
|
2526
|
+
* See
|
|
2527
|
+
* https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203
|
|
2528
|
+
*/
|
|
2529
|
+
MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit);
|
|
2530
|
+
map->limit = 0;
|
|
2531
|
+
map->current = 0;
|
|
2532
|
+
map->base = nullptr;
|
|
2533
|
+
assert(errno != 0);
|
|
2534
|
+
return errno;
|
|
2535
|
+
}
|
|
2536
|
+
rc = MDBX_UNABLE_EXTEND_MAPSIZE;
|
|
2537
|
+
limit = map->limit;
|
|
2538
|
+
}
|
|
2539
|
+
}
|
|
2540
|
+
|
|
2541
|
+
assert(ptr && ptr != MAP_FAILED);
|
|
2542
|
+
if (map->base != ptr) {
|
|
2543
|
+
VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current);
|
|
2544
|
+
/* Unpoisoning is required for ASAN to avoid false-positive diagnostic
|
|
2545
|
+
* when this memory will re-used by malloc or another mmapping.
|
|
2546
|
+
* See
|
|
2547
|
+
* https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203
|
|
2548
|
+
*/
|
|
2549
|
+
MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit);
|
|
2550
|
+
|
|
2551
|
+
VALGRIND_MAKE_MEM_DEFINED(ptr, map->current);
|
|
2552
|
+
MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr, map->current);
|
|
2553
|
+
map->base = ptr;
|
|
2554
|
+
}
|
|
2555
|
+
map->limit = limit;
|
|
2556
|
+
map->current = size;
|
|
2557
|
+
|
|
2558
|
+
#ifdef MADV_DONTFORK
|
|
2559
|
+
if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0)) {
|
|
2560
|
+
assert(errno != 0);
|
|
2561
|
+
return errno;
|
|
2562
|
+
}
|
|
2563
|
+
#endif /* MADV_DONTFORK */
|
|
2564
|
+
#ifdef MADV_NOHUGEPAGE
|
|
2565
|
+
(void)madvise(map->base, map->limit, MADV_NOHUGEPAGE);
|
|
2566
|
+
#endif /* MADV_NOHUGEPAGE */
|
|
2567
|
+
|
|
2568
|
+
#endif /* POSIX / Windows */
|
|
2569
|
+
|
|
2570
|
+
/* Zap: Redundant code */
|
|
2571
|
+
MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6287);
|
|
2572
|
+
assert(rc != MDBX_SUCCESS || (map->base != nullptr && map->base != MAP_FAILED && map->current == size &&
|
|
2573
|
+
map->limit == limit && map->filesize >= size));
|
|
2574
|
+
return rc;
|
|
2575
|
+
}
|
|
2576
|
+
|
|
2577
|
+
/*----------------------------------------------------------------------------*/
|
|
2578
|
+
|
|
2579
|
+
__cold MDBX_INTERNAL void osal_jitter(bool tiny) {
|
|
2580
|
+
for (;;) {
|
|
2581
|
+
#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__)
|
|
2582
|
+
unsigned salt = 5296013u * (unsigned)__rdtsc();
|
|
2583
|
+
salt ^= salt >> 11;
|
|
2584
|
+
salt *= 25810541u;
|
|
2585
|
+
#elif (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT
|
|
2586
|
+
static ULONG state;
|
|
2587
|
+
const unsigned salt = (unsigned)RtlRandomEx(&state);
|
|
2588
|
+
#else
|
|
2589
|
+
const unsigned salt = rand();
|
|
2590
|
+
#endif
|
|
2591
|
+
|
|
2592
|
+
const int coin = salt % (tiny ? 29u : 43u);
|
|
2593
|
+
if (coin < 43 / 3)
|
|
2594
|
+
break;
|
|
2595
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2596
|
+
if (coin < 43 * 2 / 3)
|
|
2597
|
+
SwitchToThread();
|
|
2598
|
+
else {
|
|
2599
|
+
static HANDLE timer;
|
|
2600
|
+
if (!timer)
|
|
2601
|
+
timer = CreateWaitableTimer(NULL, TRUE, NULL);
|
|
2602
|
+
|
|
2603
|
+
LARGE_INTEGER ft;
|
|
2604
|
+
ft.QuadPart = coin * (int64_t)-10; // Convert to 100 nanosecond interval,
|
|
2605
|
+
// negative value indicates relative time.
|
|
2606
|
+
SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0);
|
|
2607
|
+
WaitForSingleObject(timer, INFINITE);
|
|
2608
|
+
// CloseHandle(timer);
|
|
2609
|
+
break;
|
|
2610
|
+
}
|
|
2611
|
+
#else
|
|
2612
|
+
sched_yield();
|
|
2613
|
+
if (coin > 43 * 2 / 3)
|
|
2614
|
+
usleep(coin);
|
|
2615
|
+
#endif
|
|
2616
|
+
}
|
|
2617
|
+
}
|
|
2618
|
+
|
|
2619
|
+
/*----------------------------------------------------------------------------*/
|
|
2620
|
+
|
|
2621
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2622
|
+
static LARGE_INTEGER performance_frequency;
|
|
2623
|
+
#elif defined(__APPLE__) || defined(__MACH__)
|
|
2624
|
+
#include <mach/mach_time.h>
|
|
2625
|
+
static uint64_t ratio_16dot16_to_monotine;
|
|
2626
|
+
#elif defined(__linux__) || defined(__gnu_linux__)
|
|
2627
|
+
static clockid_t posix_clockid;
|
|
2628
|
+
__cold static clockid_t choice_monoclock(void) {
|
|
2629
|
+
struct timespec probe;
|
|
2630
|
+
#if defined(CLOCK_BOOTTIME)
|
|
2631
|
+
if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0)
|
|
2632
|
+
return CLOCK_BOOTTIME;
|
|
2633
|
+
#elif defined(CLOCK_MONOTONIC_RAW)
|
|
2634
|
+
if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0)
|
|
2635
|
+
return CLOCK_MONOTONIC_RAW;
|
|
2636
|
+
#elif defined(CLOCK_MONOTONIC_COARSE)
|
|
2637
|
+
if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0)
|
|
2638
|
+
return CLOCK_MONOTONIC_COARSE;
|
|
2639
|
+
#endif
|
|
2640
|
+
return CLOCK_MONOTONIC;
|
|
2641
|
+
}
|
|
2642
|
+
#elif defined(CLOCK_MONOTONIC)
|
|
2643
|
+
#define posix_clockid CLOCK_MONOTONIC
|
|
2644
|
+
#else
|
|
2645
|
+
#define posix_clockid CLOCK_REALTIME
|
|
2646
|
+
#endif
|
|
2647
|
+
|
|
2648
|
+
MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16) {
|
|
2649
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2650
|
+
const uint64_t ratio = performance_frequency.QuadPart;
|
|
2651
|
+
#elif defined(__APPLE__) || defined(__MACH__)
|
|
2652
|
+
const uint64_t ratio = ratio_16dot16_to_monotine;
|
|
2653
|
+
#else
|
|
2654
|
+
const uint64_t ratio = UINT64_C(1000000000);
|
|
2655
|
+
#endif
|
|
2656
|
+
const uint64_t ret = (ratio * seconds_16dot16 + 32768) >> 16;
|
|
2657
|
+
return likely(ret || seconds_16dot16 == 0) ? ret : /* fix underflow */ 1;
|
|
2658
|
+
}
|
|
2659
|
+
|
|
2660
|
+
static uint64_t monotime_limit;
|
|
2661
|
+
MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime) {
|
|
2662
|
+
if (unlikely(monotime > monotime_limit))
|
|
2663
|
+
return UINT32_MAX;
|
|
2664
|
+
|
|
2665
|
+
const uint32_t ret =
|
|
2666
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2667
|
+
(uint32_t)((monotime << 16) / performance_frequency.QuadPart);
|
|
2668
|
+
#elif defined(__APPLE__) || defined(__MACH__)
|
|
2669
|
+
(uint32_t)((monotime << 16) / ratio_16dot16_to_monotine);
|
|
2670
|
+
#else
|
|
2671
|
+
(uint32_t)((monotime << 7) / 1953125);
|
|
2672
|
+
#endif
|
|
2673
|
+
return ret;
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
MDBX_INTERNAL uint64_t osal_monotime(void) {
|
|
2677
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2678
|
+
LARGE_INTEGER counter;
|
|
2679
|
+
if (QueryPerformanceCounter(&counter))
|
|
2680
|
+
return counter.QuadPart;
|
|
2681
|
+
#elif defined(__APPLE__) || defined(__MACH__)
|
|
2682
|
+
return mach_absolute_time();
|
|
2683
|
+
#else
|
|
2684
|
+
struct timespec ts;
|
|
2685
|
+
if (likely(clock_gettime(posix_clockid, &ts) == 0))
|
|
2686
|
+
return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
|
|
2687
|
+
#endif
|
|
2688
|
+
return 0;
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults) {
|
|
2692
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2693
|
+
if (optional_page_faults) {
|
|
2694
|
+
PROCESS_MEMORY_COUNTERS pmc;
|
|
2695
|
+
*optional_page_faults = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) ? pmc.PageFaultCount : 0;
|
|
2696
|
+
}
|
|
2697
|
+
FILETIME unused, usermode;
|
|
2698
|
+
if (GetThreadTimes(GetCurrentThread(),
|
|
2699
|
+
/* CreationTime */ &unused,
|
|
2700
|
+
/* ExitTime */ &unused,
|
|
2701
|
+
/* KernelTime */ &unused,
|
|
2702
|
+
/* UserTime */ &usermode)) {
|
|
2703
|
+
/* one second = 10_000_000 * 100ns = 78125 * (1 << 7) * 100ns;
|
|
2704
|
+
* result = (h * f / 10_000_000) << 32) + l * f / 10_000_000 =
|
|
2705
|
+
* = ((h * f) >> 7) / 78125) << 32) + ((l * f) >> 7) / 78125;
|
|
2706
|
+
* 1) {h, l} *= f;
|
|
2707
|
+
* 2) {h, l} >>= 7;
|
|
2708
|
+
* 3) result = ((h / 78125) << 32) + l / 78125; */
|
|
2709
|
+
uint64_t l = usermode.dwLowDateTime * performance_frequency.QuadPart;
|
|
2710
|
+
uint64_t h = usermode.dwHighDateTime * performance_frequency.QuadPart;
|
|
2711
|
+
l = h << (64 - 7) | l >> 7;
|
|
2712
|
+
h = h >> 7;
|
|
2713
|
+
return ((h / 78125) << 32) + l / 78125;
|
|
2714
|
+
}
|
|
2715
|
+
#elif defined(RUSAGE_THREAD) || defined(RUSAGE_LWP)
|
|
2716
|
+
#ifndef RUSAGE_THREAD
|
|
2717
|
+
#define RUSAGE_THREAD RUSAGE_LWP /* Solaris */
|
|
2718
|
+
#endif
|
|
2719
|
+
struct rusage usage;
|
|
2720
|
+
if (getrusage(RUSAGE_THREAD, &usage) == 0) {
|
|
2721
|
+
if (optional_page_faults)
|
|
2722
|
+
*optional_page_faults = usage.ru_majflt;
|
|
2723
|
+
return usage.ru_utime.tv_sec * UINT64_C(1000000000) + usage.ru_utime.tv_usec * 1000u;
|
|
2724
|
+
}
|
|
2725
|
+
if (optional_page_faults)
|
|
2726
|
+
*optional_page_faults = 0;
|
|
2727
|
+
#elif defined(CLOCK_THREAD_CPUTIME_ID)
|
|
2728
|
+
if (optional_page_faults)
|
|
2729
|
+
*optional_page_faults = 0;
|
|
2730
|
+
struct timespec ts;
|
|
2731
|
+
if (likely(clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0))
|
|
2732
|
+
return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec;
|
|
2733
|
+
#else
|
|
2734
|
+
/* FIXME */
|
|
2735
|
+
if (optional_page_faults)
|
|
2736
|
+
*optional_page_faults = 0;
|
|
2737
|
+
#endif
|
|
2738
|
+
return 0;
|
|
2739
|
+
}
|
|
2740
|
+
|
|
2741
|
+
/*----------------------------------------------------------------------------*/
|
|
2742
|
+
|
|
2743
|
+
static void bootid_shake(bin128_t *p) {
|
|
2744
|
+
/* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */
|
|
2745
|
+
const uint32_t e = p->a - (p->b << 23 | p->b >> 9);
|
|
2746
|
+
p->a = p->b ^ (p->c << 16 | p->c >> 16);
|
|
2747
|
+
p->b = p->c + (p->d << 11 | p->d >> 21);
|
|
2748
|
+
p->c = p->d + e;
|
|
2749
|
+
p->d = e + p->a;
|
|
2750
|
+
}
|
|
2751
|
+
|
|
2752
|
+
__cold static void bootid_collect(bin128_t *p, const void *s, size_t n) {
|
|
2753
|
+
p->y += UINT64_C(64526882297375213);
|
|
2754
|
+
bootid_shake(p);
|
|
2755
|
+
for (size_t i = 0; i < n; ++i) {
|
|
2756
|
+
bootid_shake(p);
|
|
2757
|
+
p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i];
|
|
2758
|
+
bootid_shake(p);
|
|
2759
|
+
p->y += 14621231;
|
|
2760
|
+
}
|
|
2761
|
+
bootid_shake(p);
|
|
2762
|
+
|
|
2763
|
+
/* minor non-linear tomfoolery */
|
|
2764
|
+
const unsigned z = p->x % 61 + 1;
|
|
2765
|
+
p->y = p->y << z | p->y >> (64 - z);
|
|
2766
|
+
bootid_shake(p);
|
|
2767
|
+
bootid_shake(p);
|
|
2768
|
+
const unsigned q = p->x % 59 + 1;
|
|
2769
|
+
p->y = p->y << q | p->y >> (64 - q);
|
|
2770
|
+
bootid_shake(p);
|
|
2771
|
+
bootid_shake(p);
|
|
2772
|
+
bootid_shake(p);
|
|
2773
|
+
}
|
|
2774
|
+
|
|
2775
|
+
static size_t hamming_weight(size_t v) {
|
|
2776
|
+
const size_t m1 = (size_t)UINT64_C(0x5555555555555555);
|
|
2777
|
+
const size_t m2 = (size_t)UINT64_C(0x3333333333333333);
|
|
2778
|
+
const size_t m4 = (size_t)UINT64_C(0x0f0f0f0f0f0f0f0f);
|
|
2779
|
+
const size_t h01 = (size_t)UINT64_C(0x0101010101010101);
|
|
2780
|
+
v -= (v >> 1) & m1;
|
|
2781
|
+
v = (v & m2) + ((v >> 2) & m2);
|
|
2782
|
+
v = (v + (v >> 4)) & m4;
|
|
2783
|
+
return (v * h01) >> (sizeof(v) * 8 - 8);
|
|
2784
|
+
}
|
|
2785
|
+
|
|
2786
|
+
static inline size_t hw64(uint64_t v) {
|
|
2787
|
+
size_t r = hamming_weight((size_t)v);
|
|
2788
|
+
if (sizeof(v) > sizeof(r))
|
|
2789
|
+
r += hamming_weight((size_t)(v >> sizeof(r) * 4 >> sizeof(r) * 4));
|
|
2790
|
+
return r;
|
|
2791
|
+
}
|
|
2792
|
+
|
|
2793
|
+
static bool check_uuid(bin128_t uuid) {
|
|
2794
|
+
size_t hw = hw64(uuid.x) + hw64(uuid.y) + hw64(uuid.x ^ uuid.y);
|
|
2795
|
+
return (hw >> 6) == 1;
|
|
2796
|
+
}
|
|
2797
|
+
|
|
2798
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2799
|
+
|
|
2800
|
+
__cold static uint64_t windows_systemtime_ms() {
|
|
2801
|
+
FILETIME ft;
|
|
2802
|
+
GetSystemTimeAsFileTime(&ft);
|
|
2803
|
+
return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul;
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
__cold static uint64_t windows_bootime(void) {
|
|
2807
|
+
unsigned confirmed = 0;
|
|
2808
|
+
uint64_t boottime = 0;
|
|
2809
|
+
uint64_t up0 = imports.GetTickCount64();
|
|
2810
|
+
uint64_t st0 = windows_systemtime_ms();
|
|
2811
|
+
for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) {
|
|
2812
|
+
YieldProcessor();
|
|
2813
|
+
const uint64_t up1 = imports.GetTickCount64();
|
|
2814
|
+
const uint64_t st1 = windows_systemtime_ms();
|
|
2815
|
+
if (st1 > fuse && st1 == st0 && up1 == up0) {
|
|
2816
|
+
uint64_t diff = st1 - up1;
|
|
2817
|
+
if (boottime == diff) {
|
|
2818
|
+
if (++confirmed > 4)
|
|
2819
|
+
return boottime;
|
|
2820
|
+
} else {
|
|
2821
|
+
confirmed = 0;
|
|
2822
|
+
boottime = diff;
|
|
2823
|
+
}
|
|
2824
|
+
fuse = st1;
|
|
2825
|
+
Sleep(1);
|
|
2826
|
+
}
|
|
2827
|
+
st0 = st1;
|
|
2828
|
+
up0 = up1;
|
|
2829
|
+
}
|
|
2830
|
+
return 0;
|
|
2831
|
+
}
|
|
2832
|
+
|
|
2833
|
+
__cold static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue, PVOID pvData, LPDWORD pcbData) {
|
|
2834
|
+
LSTATUS rc;
|
|
2835
|
+
if (!imports.RegGetValueA) {
|
|
2836
|
+
/* an old Windows 2000/XP */
|
|
2837
|
+
HKEY hSubKey;
|
|
2838
|
+
rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey);
|
|
2839
|
+
if (rc == ERROR_SUCCESS) {
|
|
2840
|
+
rc = RegQueryValueExA(hSubKey, lpValue, nullptr, nullptr, pvData, pcbData);
|
|
2841
|
+
RegCloseKey(hSubKey);
|
|
2842
|
+
}
|
|
2843
|
+
return rc;
|
|
2844
|
+
}
|
|
2845
|
+
|
|
2846
|
+
rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, nullptr, pvData, pcbData);
|
|
2847
|
+
if (rc != ERROR_FILE_NOT_FOUND)
|
|
2848
|
+
return rc;
|
|
2849
|
+
|
|
2850
|
+
rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, nullptr,
|
|
2851
|
+
pvData, pcbData);
|
|
2852
|
+
if (rc != ERROR_FILE_NOT_FOUND)
|
|
2853
|
+
return rc;
|
|
2854
|
+
return imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, nullptr,
|
|
2855
|
+
pvData, pcbData);
|
|
2856
|
+
}
|
|
2857
|
+
#endif
|
|
2858
|
+
|
|
2859
|
+
MDBX_MAYBE_UNUSED __cold static bool bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) {
|
|
2860
|
+
if (n > 31) {
|
|
2861
|
+
unsigned bits = 0;
|
|
2862
|
+
for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ {
|
|
2863
|
+
uint8_t c = ((const uint8_t *)p)[i];
|
|
2864
|
+
if (c >= '0' && c <= '9')
|
|
2865
|
+
c -= '0';
|
|
2866
|
+
else if (c >= 'a' && c <= 'f')
|
|
2867
|
+
c -= 'a' - 10;
|
|
2868
|
+
else if (c >= 'A' && c <= 'F')
|
|
2869
|
+
c -= 'A' - 10;
|
|
2870
|
+
else
|
|
2871
|
+
continue;
|
|
2872
|
+
assert(c <= 15);
|
|
2873
|
+
c ^= s->y >> 60;
|
|
2874
|
+
s->y = s->y << 4 | s->x >> 60;
|
|
2875
|
+
s->x = s->x << 4 | c;
|
|
2876
|
+
bits += 4;
|
|
2877
|
+
}
|
|
2878
|
+
if (bits > 42 * 3)
|
|
2879
|
+
/* UUID parsed successfully */
|
|
2880
|
+
return true;
|
|
2881
|
+
}
|
|
2882
|
+
|
|
2883
|
+
if (n > 15) /* is enough handle it as a binary? */ {
|
|
2884
|
+
if (n == sizeof(bin128_t)) {
|
|
2885
|
+
bin128_t aligned;
|
|
2886
|
+
memcpy(&aligned, p, sizeof(bin128_t));
|
|
2887
|
+
s->x += aligned.x;
|
|
2888
|
+
s->y += aligned.y;
|
|
2889
|
+
} else
|
|
2890
|
+
bootid_collect(s, p, n);
|
|
2891
|
+
return check_uuid(*s);
|
|
2892
|
+
}
|
|
2893
|
+
|
|
2894
|
+
if (n)
|
|
2895
|
+
bootid_collect(s, p, n);
|
|
2896
|
+
return false;
|
|
2897
|
+
}
|
|
2898
|
+
|
|
2899
|
+
#if defined(__linux__) || defined(__gnu_linux__)
|
|
2900
|
+
|
|
2901
|
+
__cold static bool is_inside_lxc(void) {
|
|
2902
|
+
bool inside_lxc = false;
|
|
2903
|
+
FILE *mounted = setmntent("/proc/mounts", "r");
|
|
2904
|
+
if (mounted) {
|
|
2905
|
+
const struct mntent *ent;
|
|
2906
|
+
while (nullptr != (ent = getmntent(mounted))) {
|
|
2907
|
+
if (strcmp(ent->mnt_fsname, "lxcfs") == 0 && strncmp(ent->mnt_dir, "/proc/", 6) == 0) {
|
|
2908
|
+
inside_lxc = true;
|
|
2909
|
+
break;
|
|
2910
|
+
}
|
|
2911
|
+
}
|
|
2912
|
+
endmntent(mounted);
|
|
2913
|
+
}
|
|
2914
|
+
return inside_lxc;
|
|
2915
|
+
}
|
|
2916
|
+
|
|
2917
|
+
__cold static bool proc_read_uuid(const char *path, bin128_t *target) {
|
|
2918
|
+
const int fd = open(path, O_RDONLY | O_NOFOLLOW);
|
|
2919
|
+
if (fd != -1) {
|
|
2920
|
+
struct statfs fs;
|
|
2921
|
+
char buf[42];
|
|
2922
|
+
const ssize_t len = (fstatfs(fd, &fs) == 0 &&
|
|
2923
|
+
(fs.f_type == /* procfs */ 0x9FA0 || (fs.f_type == /* tmpfs */ 0x1021994 && is_inside_lxc())))
|
|
2924
|
+
? read(fd, buf, sizeof(buf))
|
|
2925
|
+
: -1;
|
|
2926
|
+
const int err = close(fd);
|
|
2927
|
+
assert(err == 0);
|
|
2928
|
+
(void)err;
|
|
2929
|
+
if (len > 0)
|
|
2930
|
+
return bootid_parse_uuid(target, buf, len);
|
|
2931
|
+
}
|
|
2932
|
+
return false;
|
|
2933
|
+
}
|
|
2934
|
+
#endif /* Linux */
|
|
2935
|
+
|
|
2936
|
+
__cold static bin128_t osal_bootid(void) {
|
|
2937
|
+
bin128_t uuid = {{0, 0}};
|
|
2938
|
+
bool got_machineid = false, got_boottime = false, got_bootseq = false;
|
|
2939
|
+
|
|
2940
|
+
#if defined(__linux__) || defined(__gnu_linux__)
|
|
2941
|
+
if (proc_read_uuid("/proc/sys/kernel/random/boot_id", &uuid))
|
|
2942
|
+
return uuid;
|
|
2943
|
+
#endif /* Linux */
|
|
2944
|
+
|
|
2945
|
+
#if defined(__APPLE__) || defined(__MACH__)
|
|
2946
|
+
{
|
|
2947
|
+
char buf[42];
|
|
2948
|
+
size_t len = sizeof(buf);
|
|
2949
|
+
if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && bootid_parse_uuid(&uuid, buf, len))
|
|
2950
|
+
return uuid;
|
|
2951
|
+
|
|
2952
|
+
#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED > 1050
|
|
2953
|
+
uuid_t hostuuid;
|
|
2954
|
+
struct timespec wait = {0, 1000000000u / 42};
|
|
2955
|
+
if (!gethostuuid(hostuuid, &wait))
|
|
2956
|
+
got_machineid = bootid_parse_uuid(&uuid, hostuuid, sizeof(hostuuid));
|
|
2957
|
+
#endif /* > 10.5 */
|
|
2958
|
+
|
|
2959
|
+
struct timeval boottime;
|
|
2960
|
+
len = sizeof(boottime);
|
|
2961
|
+
if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && len == sizeof(boottime) && boottime.tv_sec)
|
|
2962
|
+
got_boottime = true;
|
|
2963
|
+
}
|
|
2964
|
+
#endif /* Apple/Darwin */
|
|
2965
|
+
|
|
2966
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
2967
|
+
{
|
|
2968
|
+
union buf {
|
|
2969
|
+
DWORD BootId;
|
|
2970
|
+
DWORD BaseTime;
|
|
2971
|
+
SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo;
|
|
2972
|
+
struct {
|
|
2973
|
+
LARGE_INTEGER BootTime;
|
|
2974
|
+
LARGE_INTEGER CurrentTime;
|
|
2975
|
+
LARGE_INTEGER TimeZoneBias;
|
|
2976
|
+
ULONG TimeZoneId;
|
|
2977
|
+
ULONG Reserved;
|
|
2978
|
+
ULONGLONG BootTimeBias;
|
|
2979
|
+
ULONGLONG SleepTimeBias;
|
|
2980
|
+
} SysTimeOfDayInfoHacked;
|
|
2981
|
+
wchar_t MachineGuid[42];
|
|
2982
|
+
char DigitalProductId[248];
|
|
2983
|
+
} buf;
|
|
2984
|
+
|
|
2985
|
+
static const char HKLM_MicrosoftCryptography[] = "SOFTWARE\\Microsoft\\Cryptography";
|
|
2986
|
+
DWORD len = sizeof(buf);
|
|
2987
|
+
/* Windows is madness and must die */
|
|
2988
|
+
if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, "MachineGuid", &buf.MachineGuid, &len) ==
|
|
2989
|
+
ERROR_SUCCESS &&
|
|
2990
|
+
len < sizeof(buf))
|
|
2991
|
+
got_machineid = bootid_parse_uuid(&uuid, &buf.MachineGuid, len);
|
|
2992
|
+
|
|
2993
|
+
if (!got_machineid) {
|
|
2994
|
+
/* again, Windows is madness */
|
|
2995
|
+
static const char HKLM_WindowsNT[] = "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion";
|
|
2996
|
+
static const char HKLM_WindowsNT_DPK[] = "SOFTWARE\\Microsoft\\Windows "
|
|
2997
|
+
"NT\\CurrentVersion\\DefaultProductKey";
|
|
2998
|
+
static const char HKLM_WindowsNT_DPK2[] = "SOFTWARE\\Microsoft\\Windows "
|
|
2999
|
+
"NT\\CurrentVersion\\DefaultProductKey2";
|
|
3000
|
+
|
|
3001
|
+
len = sizeof(buf);
|
|
3002
|
+
if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, "DigitalProductId", &buf.DigitalProductId, &len) ==
|
|
3003
|
+
ERROR_SUCCESS &&
|
|
3004
|
+
len > 42 && len < sizeof(buf)) {
|
|
3005
|
+
bootid_collect(&uuid, &buf.DigitalProductId, len);
|
|
3006
|
+
got_machineid = true;
|
|
3007
|
+
}
|
|
3008
|
+
len = sizeof(buf);
|
|
3009
|
+
if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, "DigitalProductId", &buf.DigitalProductId, &len) ==
|
|
3010
|
+
ERROR_SUCCESS &&
|
|
3011
|
+
len > 42 && len < sizeof(buf)) {
|
|
3012
|
+
bootid_collect(&uuid, &buf.DigitalProductId, len);
|
|
3013
|
+
got_machineid = true;
|
|
3014
|
+
}
|
|
3015
|
+
len = sizeof(buf);
|
|
3016
|
+
if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, "DigitalProductId", &buf.DigitalProductId, &len) ==
|
|
3017
|
+
ERROR_SUCCESS &&
|
|
3018
|
+
len > 42 && len < sizeof(buf)) {
|
|
3019
|
+
bootid_collect(&uuid, &buf.DigitalProductId, len);
|
|
3020
|
+
got_machineid = true;
|
|
3021
|
+
}
|
|
3022
|
+
}
|
|
3023
|
+
|
|
3024
|
+
static const char HKLM_PrefetcherParams[] = "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory "
|
|
3025
|
+
"Management\\PrefetchParameters";
|
|
3026
|
+
len = sizeof(buf);
|
|
3027
|
+
if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", &buf.BootId, &len) == ERROR_SUCCESS &&
|
|
3028
|
+
len > 1 && len < sizeof(buf)) {
|
|
3029
|
+
bootid_collect(&uuid, &buf.BootId, len);
|
|
3030
|
+
got_bootseq = true;
|
|
3031
|
+
}
|
|
3032
|
+
|
|
3033
|
+
len = sizeof(buf);
|
|
3034
|
+
if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", &buf.BaseTime, &len) == ERROR_SUCCESS &&
|
|
3035
|
+
len >= sizeof(buf.BaseTime) && buf.BaseTime) {
|
|
3036
|
+
bootid_collect(&uuid, &buf.BaseTime, len);
|
|
3037
|
+
got_boottime = true;
|
|
3038
|
+
}
|
|
3039
|
+
|
|
3040
|
+
/* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */
|
|
3041
|
+
NTSTATUS status = NtQuerySystemInformation(0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo,
|
|
3042
|
+
sizeof(buf.SysTimeOfDayInfo), &len);
|
|
3043
|
+
if (NT_SUCCESS(status) &&
|
|
3044
|
+
len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTimeBias) +
|
|
3045
|
+
sizeof(buf.SysTimeOfDayInfoHacked.BootTimeBias) &&
|
|
3046
|
+
buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) {
|
|
3047
|
+
const uint64_t UnbiasedBootTime =
|
|
3048
|
+
buf.SysTimeOfDayInfoHacked.BootTime.QuadPart - buf.SysTimeOfDayInfoHacked.BootTimeBias;
|
|
3049
|
+
if (UnbiasedBootTime) {
|
|
3050
|
+
bootid_collect(&uuid, &UnbiasedBootTime, sizeof(UnbiasedBootTime));
|
|
3051
|
+
got_boottime = true;
|
|
3052
|
+
}
|
|
3053
|
+
}
|
|
3054
|
+
|
|
3055
|
+
if (!got_boottime) {
|
|
3056
|
+
uint64_t boottime = windows_bootime();
|
|
3057
|
+
if (boottime) {
|
|
3058
|
+
bootid_collect(&uuid, &boottime, sizeof(boottime));
|
|
3059
|
+
got_boottime = true;
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
#endif /* Windows */
|
|
3064
|
+
|
|
3065
|
+
#if defined(CTL_HW) && defined(HW_UUID)
|
|
3066
|
+
if (!got_machineid) {
|
|
3067
|
+
static const int mib[] = {CTL_HW, HW_UUID};
|
|
3068
|
+
char buf[42];
|
|
3069
|
+
size_t len = sizeof(buf);
|
|
3070
|
+
if (sysctl(
|
|
3071
|
+
#ifdef SYSCTL_LEGACY_NONCONST_MIB
|
|
3072
|
+
(int *)
|
|
3073
|
+
#endif
|
|
3074
|
+
mib,
|
|
3075
|
+
ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0)
|
|
3076
|
+
got_machineid = bootid_parse_uuid(&uuid, buf, len);
|
|
3077
|
+
}
|
|
3078
|
+
#endif /* CTL_HW && HW_UUID */
|
|
3079
|
+
|
|
3080
|
+
#if defined(CTL_KERN) && defined(KERN_HOSTUUID)
|
|
3081
|
+
if (!got_machineid) {
|
|
3082
|
+
static const int mib[] = {CTL_KERN, KERN_HOSTUUID};
|
|
3083
|
+
char buf[42];
|
|
3084
|
+
size_t len = sizeof(buf);
|
|
3085
|
+
if (sysctl(
|
|
3086
|
+
#ifdef SYSCTL_LEGACY_NONCONST_MIB
|
|
3087
|
+
(int *)
|
|
3088
|
+
#endif
|
|
3089
|
+
mib,
|
|
3090
|
+
ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0)
|
|
3091
|
+
got_machineid = bootid_parse_uuid(&uuid, buf, len);
|
|
3092
|
+
}
|
|
3093
|
+
#endif /* CTL_KERN && KERN_HOSTUUID */
|
|
3094
|
+
|
|
3095
|
+
#if defined(__NetBSD__)
|
|
3096
|
+
if (!got_machineid) {
|
|
3097
|
+
char buf[42];
|
|
3098
|
+
size_t len = sizeof(buf);
|
|
3099
|
+
if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, nullptr, 0) == 0)
|
|
3100
|
+
got_machineid = bootid_parse_uuid(&uuid, buf, len);
|
|
3101
|
+
}
|
|
3102
|
+
#endif /* __NetBSD__ */
|
|
3103
|
+
|
|
3104
|
+
#if !(defined(_WIN32) || defined(_WIN64))
|
|
3105
|
+
if (!got_machineid) {
|
|
3106
|
+
int fd = open("/etc/machine-id", O_RDONLY);
|
|
3107
|
+
if (fd == -1)
|
|
3108
|
+
fd = open("/var/lib/dbus/machine-id", O_RDONLY);
|
|
3109
|
+
if (fd != -1) {
|
|
3110
|
+
char buf[42];
|
|
3111
|
+
const ssize_t len = read(fd, buf, sizeof(buf));
|
|
3112
|
+
const int err = close(fd);
|
|
3113
|
+
assert(err == 0);
|
|
3114
|
+
(void)err;
|
|
3115
|
+
if (len > 0)
|
|
3116
|
+
got_machineid = bootid_parse_uuid(&uuid, buf, len);
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
#endif /* !Windows */
|
|
3120
|
+
|
|
3121
|
+
#if _XOPEN_SOURCE_EXTENDED
|
|
3122
|
+
if (!got_machineid) {
|
|
3123
|
+
const long hostid = gethostid();
|
|
3124
|
+
if (hostid != 0 && hostid != -1) {
|
|
3125
|
+
bootid_collect(&uuid, &hostid, sizeof(hostid));
|
|
3126
|
+
got_machineid = true;
|
|
3127
|
+
}
|
|
3128
|
+
}
|
|
3129
|
+
#endif /* _XOPEN_SOURCE_EXTENDED */
|
|
3130
|
+
|
|
3131
|
+
if (!got_machineid) {
|
|
3132
|
+
lack:
|
|
3133
|
+
uuid.x = uuid.y = 0;
|
|
3134
|
+
return uuid;
|
|
3135
|
+
}
|
|
3136
|
+
|
|
3137
|
+
/*--------------------------------------------------------------------------*/
|
|
3138
|
+
|
|
3139
|
+
#if defined(CTL_KERN) && defined(KERN_BOOTTIME)
|
|
3140
|
+
if (!got_boottime) {
|
|
3141
|
+
static const int mib[] = {CTL_KERN, KERN_BOOTTIME};
|
|
3142
|
+
struct timeval boottime;
|
|
3143
|
+
size_t len = sizeof(boottime);
|
|
3144
|
+
if (sysctl(
|
|
3145
|
+
#ifdef SYSCTL_LEGACY_NONCONST_MIB
|
|
3146
|
+
(int *)
|
|
3147
|
+
#endif
|
|
3148
|
+
mib,
|
|
3149
|
+
ARRAY_LENGTH(mib), &boottime, &len, nullptr, 0) == 0 &&
|
|
3150
|
+
len == sizeof(boottime) && boottime.tv_sec) {
|
|
3151
|
+
bootid_collect(&uuid, &boottime, len);
|
|
3152
|
+
got_boottime = true;
|
|
3153
|
+
}
|
|
3154
|
+
}
|
|
3155
|
+
#endif /* CTL_KERN && KERN_BOOTTIME */
|
|
3156
|
+
|
|
3157
|
+
#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
|
|
3158
|
+
if (!got_boottime) {
|
|
3159
|
+
kstat_ctl_t *kc = kstat_open();
|
|
3160
|
+
if (kc) {
|
|
3161
|
+
kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc");
|
|
3162
|
+
if (kp && kstat_read(kc, kp, 0) != -1) {
|
|
3163
|
+
kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time");
|
|
3164
|
+
if (kn) {
|
|
3165
|
+
switch (kn->data_type) {
|
|
3166
|
+
case KSTAT_DATA_INT32:
|
|
3167
|
+
case KSTAT_DATA_UINT32:
|
|
3168
|
+
bootid_collect(&uuid, &kn->value, sizeof(int32_t));
|
|
3169
|
+
got_boottime = true;
|
|
3170
|
+
case KSTAT_DATA_INT64:
|
|
3171
|
+
case KSTAT_DATA_UINT64:
|
|
3172
|
+
bootid_collect(&uuid, &kn->value, sizeof(int64_t));
|
|
3173
|
+
got_boottime = true;
|
|
3174
|
+
}
|
|
3175
|
+
}
|
|
3176
|
+
}
|
|
3177
|
+
kstat_close(kc);
|
|
3178
|
+
}
|
|
3179
|
+
}
|
|
3180
|
+
#endif /* SunOS / Solaris */
|
|
3181
|
+
|
|
3182
|
+
#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME)
|
|
3183
|
+
if (!got_boottime) {
|
|
3184
|
+
setutxent();
|
|
3185
|
+
const struct utmpx id = {.ut_type = BOOT_TIME};
|
|
3186
|
+
const struct utmpx *entry = getutxid(&id);
|
|
3187
|
+
if (entry) {
|
|
3188
|
+
bootid_collect(&uuid, entry, sizeof(*entry));
|
|
3189
|
+
got_boottime = true;
|
|
3190
|
+
while (unlikely((entry = getutxid(&id)) != nullptr)) {
|
|
3191
|
+
/* have multiple reboot records, assuming we can distinguish next
|
|
3192
|
+
* bootsession even if RTC is wrong or absent */
|
|
3193
|
+
bootid_collect(&uuid, entry, sizeof(*entry));
|
|
3194
|
+
got_bootseq = true;
|
|
3195
|
+
}
|
|
3196
|
+
}
|
|
3197
|
+
endutxent();
|
|
3198
|
+
}
|
|
3199
|
+
#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */
|
|
3200
|
+
|
|
3201
|
+
if (!got_bootseq) {
|
|
3202
|
+
if (!got_boottime || !MDBX_TRUST_RTC)
|
|
3203
|
+
goto lack;
|
|
3204
|
+
|
|
3205
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
3206
|
+
FILETIME now;
|
|
3207
|
+
GetSystemTimeAsFileTime(&now);
|
|
3208
|
+
if (0x1CCCCCC > now.dwHighDateTime)
|
|
3209
|
+
#else
|
|
3210
|
+
struct timespec mono, real;
|
|
3211
|
+
if (clock_gettime(CLOCK_MONOTONIC, &mono) || clock_gettime(CLOCK_REALTIME, &real) ||
|
|
3212
|
+
/* wrong time, RTC is mad or absent */
|
|
3213
|
+
1555555555l > real.tv_sec ||
|
|
3214
|
+
/* seems no adjustment by RTC/NTP, i.e. a fake time */
|
|
3215
|
+
real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || (real.tv_sec - mono.tv_sec) % 900u == 0)
|
|
3216
|
+
#endif
|
|
3217
|
+
goto lack;
|
|
3218
|
+
}
|
|
3219
|
+
|
|
3220
|
+
return uuid;
|
|
3221
|
+
}
|
|
3222
|
+
|
|
3223
|
+
__cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, intptr_t *avail_pages) {
|
|
3224
|
+
if (!page_size && !total_pages && !avail_pages)
|
|
3225
|
+
return LOG_IFERR(MDBX_EINVAL);
|
|
3226
|
+
if (total_pages)
|
|
3227
|
+
*total_pages = -1;
|
|
3228
|
+
if (avail_pages)
|
|
3229
|
+
*avail_pages = -1;
|
|
3230
|
+
|
|
3231
|
+
const intptr_t pagesize = globals.sys_pagesize;
|
|
3232
|
+
if (page_size)
|
|
3233
|
+
*page_size = pagesize;
|
|
3234
|
+
if (unlikely(pagesize < MDBX_MIN_PAGESIZE || !is_powerof2(pagesize)))
|
|
3235
|
+
return LOG_IFERR(MDBX_INCOMPATIBLE);
|
|
3236
|
+
|
|
3237
|
+
MDBX_MAYBE_UNUSED const int log2page = log2n_powerof2(pagesize);
|
|
3238
|
+
assert(pagesize == (INT64_C(1) << log2page));
|
|
3239
|
+
(void)log2page;
|
|
3240
|
+
|
|
3241
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
3242
|
+
MEMORYSTATUSEX info;
|
|
3243
|
+
memset(&info, 0, sizeof(info));
|
|
3244
|
+
info.dwLength = sizeof(info);
|
|
3245
|
+
if (!GlobalMemoryStatusEx(&info))
|
|
3246
|
+
return LOG_IFERR((int)GetLastError());
|
|
3247
|
+
#endif
|
|
3248
|
+
|
|
3249
|
+
if (total_pages) {
|
|
3250
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
3251
|
+
const intptr_t total_ram_pages = (intptr_t)(info.ullTotalPhys >> log2page);
|
|
3252
|
+
#elif defined(_SC_PHYS_PAGES)
|
|
3253
|
+
const intptr_t total_ram_pages = sysconf(_SC_PHYS_PAGES);
|
|
3254
|
+
if (total_ram_pages == -1)
|
|
3255
|
+
return LOG_IFERR(errno);
|
|
3256
|
+
#elif defined(_SC_AIX_REALMEM)
|
|
3257
|
+
const intptr_t total_ram_Kb = sysconf(_SC_AIX_REALMEM);
|
|
3258
|
+
if (total_ram_Kb == -1)
|
|
3259
|
+
return LOG_IFERR(errno);
|
|
3260
|
+
const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page;
|
|
3261
|
+
#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || defined(HW_PHYSMEM)
|
|
3262
|
+
size_t ram, len = sizeof(ram);
|
|
3263
|
+
static const int mib[] = {CTL_HW,
|
|
3264
|
+
#if defined(HW_USERMEM)
|
|
3265
|
+
HW_USERMEM
|
|
3266
|
+
#elif defined(HW_PHYSMEM64)
|
|
3267
|
+
HW_PHYSMEM64
|
|
3268
|
+
#elif defined(HW_MEMSIZE)
|
|
3269
|
+
HW_MEMSIZE
|
|
3270
|
+
#else
|
|
3271
|
+
HW_PHYSMEM
|
|
3272
|
+
#endif
|
|
3273
|
+
};
|
|
3274
|
+
if (sysctl(
|
|
3275
|
+
#ifdef SYSCTL_LEGACY_NONCONST_MIB
|
|
3276
|
+
(int *)
|
|
3277
|
+
#endif
|
|
3278
|
+
mib,
|
|
3279
|
+
ARRAY_LENGTH(mib), &ram, &len, nullptr, 0) != 0)
|
|
3280
|
+
return LOG_IFERR(errno);
|
|
3281
|
+
if (len != sizeof(ram))
|
|
3282
|
+
return LOG_IFERR(MDBX_ENOSYS);
|
|
3283
|
+
const intptr_t total_ram_pages = (intptr_t)(ram >> log2page);
|
|
3284
|
+
#else
|
|
3285
|
+
#error "FIXME: Get User-accessible or physical RAM"
|
|
3286
|
+
#endif
|
|
3287
|
+
*total_pages = total_ram_pages;
|
|
3288
|
+
if (total_ram_pages < 1)
|
|
3289
|
+
return LOG_IFERR(MDBX_ENOSYS);
|
|
3290
|
+
}
|
|
3291
|
+
|
|
3292
|
+
if (avail_pages) {
|
|
3293
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
3294
|
+
const intptr_t avail_ram_pages = (intptr_t)(info.ullAvailPhys >> log2page);
|
|
3295
|
+
#elif defined(_SC_AVPHYS_PAGES)
|
|
3296
|
+
const intptr_t avail_ram_pages = sysconf(_SC_AVPHYS_PAGES);
|
|
3297
|
+
if (avail_ram_pages == -1)
|
|
3298
|
+
return LOG_IFERR(errno);
|
|
3299
|
+
#elif defined(__MACH__)
|
|
3300
|
+
mach_msg_type_number_t count = HOST_VM_INFO_COUNT;
|
|
3301
|
+
vm_statistics_data_t vmstat;
|
|
3302
|
+
mach_port_t mport = mach_host_self();
|
|
3303
|
+
kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t)&vmstat, &count);
|
|
3304
|
+
mach_port_deallocate(mach_task_self(), mport);
|
|
3305
|
+
if (unlikely(kerr != KERN_SUCCESS))
|
|
3306
|
+
return LOG_IFERR(MDBX_ENOSYS);
|
|
3307
|
+
const intptr_t avail_ram_pages = vmstat.free_count;
|
|
3308
|
+
#elif defined(VM_TOTAL) || defined(VM_METER)
|
|
3309
|
+
struct vmtotal info;
|
|
3310
|
+
size_t len = sizeof(info);
|
|
3311
|
+
static const int mib[] = {CTL_VM,
|
|
3312
|
+
#if defined(VM_TOTAL)
|
|
3313
|
+
VM_TOTAL
|
|
3314
|
+
#elif defined(VM_METER)
|
|
3315
|
+
VM_METER
|
|
3316
|
+
#endif
|
|
3317
|
+
};
|
|
3318
|
+
if (sysctl(
|
|
3319
|
+
#ifdef SYSCTL_LEGACY_NONCONST_MIB
|
|
3320
|
+
(int *)
|
|
3321
|
+
#endif
|
|
3322
|
+
mib,
|
|
3323
|
+
ARRAY_LENGTH(mib), &info, &len, nullptr, 0) != 0)
|
|
3324
|
+
return LOG_IFERR(errno);
|
|
3325
|
+
if (len != sizeof(info))
|
|
3326
|
+
return LOG_IFERR(MDBX_ENOSYS);
|
|
3327
|
+
const intptr_t avail_ram_pages = info.t_free;
|
|
3328
|
+
#else
|
|
3329
|
+
#error "FIXME: Get Available RAM"
|
|
3330
|
+
#endif
|
|
3331
|
+
*avail_pages = avail_ram_pages;
|
|
3332
|
+
if (avail_ram_pages < 1)
|
|
3333
|
+
return LOG_IFERR(MDBX_ENOSYS);
|
|
3334
|
+
}
|
|
3335
|
+
|
|
3336
|
+
return MDBX_SUCCESS;
|
|
3337
|
+
}
|
|
3338
|
+
|
|
3339
|
+
/*----------------------------------------------------------------------------*/
|
|
3340
|
+
|
|
3341
|
+
#ifdef __FreeBSD__
|
|
3342
|
+
#include <sys/uuid.h>
|
|
3343
|
+
#endif /* FreeBSD */
|
|
3344
|
+
|
|
3345
|
+
#ifdef __IPHONE_OS_VERSION_MIN_REQUIRED
|
|
3346
|
+
#include <CommonCrypto/CommonRandom.h>
|
|
3347
|
+
#elif __GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \
|
|
3348
|
+
defined(__DragonFly__) || defined(__APPLE__) || __has_include(<sys/random.h>)
|
|
3349
|
+
#include <sys/random.h>
|
|
3350
|
+
#endif /* sys/random.h */
|
|
3351
|
+
|
|
3352
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
3353
|
+
#include <wincrypt.h>
|
|
3354
|
+
#endif /* Windows */
|
|
3355
|
+
|
|
3356
|
+
MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *env) {
|
|
3357
|
+
struct {
|
|
3358
|
+
uint64_t begin, end, cputime;
|
|
3359
|
+
uintptr_t thread, pid;
|
|
3360
|
+
const void *x, *y;
|
|
3361
|
+
bin128_t (*z)(const MDBX_env *env);
|
|
3362
|
+
} salt;
|
|
3363
|
+
|
|
3364
|
+
salt.begin = osal_monotime();
|
|
3365
|
+
bin128_t uuid = {{0, 0}};
|
|
3366
|
+
|
|
3367
|
+
#if defined(__linux__) || defined(__gnu_linux__)
|
|
3368
|
+
if (proc_read_uuid("/proc/sys/kernel/random/uuid", &uuid) && check_uuid(uuid))
|
|
3369
|
+
return uuid;
|
|
3370
|
+
#endif /* Linux */
|
|
3371
|
+
|
|
3372
|
+
#ifdef __FreeBSD__
|
|
3373
|
+
STATIC_ASSERT(sizeof(uuid) == sizeof(struct uuid));
|
|
3374
|
+
if (uuidgen((struct uuid *)&uuid, 1) == 0 && check_uuid(uuid))
|
|
3375
|
+
return uuid;
|
|
3376
|
+
#endif /* FreeBSD */
|
|
3377
|
+
|
|
3378
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
3379
|
+
if (imports.CoCreateGuid && imports.CoCreateGuid(&uuid) == 0 && check_uuid(uuid))
|
|
3380
|
+
return uuid;
|
|
3381
|
+
|
|
3382
|
+
HCRYPTPROV hCryptProv = 0;
|
|
3383
|
+
if (CryptAcquireContextW(&hCryptProv, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) {
|
|
3384
|
+
const BOOL ok = CryptGenRandom(hCryptProv, sizeof(uuid), (unsigned char *)&uuid);
|
|
3385
|
+
CryptReleaseContext(hCryptProv, 0);
|
|
3386
|
+
if (ok && check_uuid(uuid))
|
|
3387
|
+
return uuid;
|
|
3388
|
+
}
|
|
3389
|
+
#elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && defined(__IPHONE_8_0)
|
|
3390
|
+
#if __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_8_0
|
|
3391
|
+
if (CCRandomGenerateBytes(&uuid, sizeof(uuid)) == kCCSuccess && check_uuid(uuid))
|
|
3392
|
+
return uuid;
|
|
3393
|
+
#endif /* iOS >= 8.x */
|
|
3394
|
+
#else
|
|
3395
|
+
const int fd = open("/dev/urandom", O_RDONLY);
|
|
3396
|
+
if (fd != -1) {
|
|
3397
|
+
const ssize_t len = read(fd, &uuid, sizeof(uuid));
|
|
3398
|
+
const int err = close(fd);
|
|
3399
|
+
assert(err == 0);
|
|
3400
|
+
(void)err;
|
|
3401
|
+
if (len == sizeof(uuid) && check_uuid(uuid))
|
|
3402
|
+
return uuid;
|
|
3403
|
+
}
|
|
3404
|
+
#if (__GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \
|
|
3405
|
+
defined(__DragonFly__)) && \
|
|
3406
|
+
!defined(__APPLE__) && !defined(__ANDROID_API__)
|
|
3407
|
+
if (getrandom(&uuid, sizeof(uuid), 0) == sizeof(uuid) && check_uuid(uuid))
|
|
3408
|
+
return uuid;
|
|
3409
|
+
#elif defined(__OpenBSD__) || (defined(__sun) && defined(__SVR4)) || \
|
|
3410
|
+
(defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200)
|
|
3411
|
+
if (getentropy(&uuid, sizeof(uuid)) == 0 && check_uuid(uuid))
|
|
3412
|
+
return uuid;
|
|
3413
|
+
#endif /* getrandom() / getentropy() */
|
|
3414
|
+
#endif /* !Windows */
|
|
3415
|
+
|
|
3416
|
+
uuid = globals.bootid;
|
|
3417
|
+
bootid_collect(&uuid, env, sizeof(*env));
|
|
3418
|
+
salt.thread = osal_thread_self();
|
|
3419
|
+
salt.pid = osal_getpid();
|
|
3420
|
+
salt.x = &salt;
|
|
3421
|
+
salt.y = env;
|
|
3422
|
+
salt.z = &osal_guid;
|
|
3423
|
+
do {
|
|
3424
|
+
salt.cputime = osal_cputime(nullptr);
|
|
3425
|
+
salt.end = osal_monotime();
|
|
3426
|
+
bootid_collect(&uuid, &salt, sizeof(salt));
|
|
3427
|
+
} while (!check_uuid(uuid));
|
|
3428
|
+
return uuid;
|
|
3429
|
+
}
|
|
3430
|
+
|
|
3431
|
+
/*--------------------------------------------------------------------------*/
|
|
3432
|
+
|
|
3433
|
+
void osal_ctor(void) {
|
|
3434
|
+
#if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX)
|
|
3435
|
+
osal_iov_max = sysconf(_SC_IOV_MAX);
|
|
3436
|
+
if (RUNNING_ON_VALGRIND && osal_iov_max > 64)
|
|
3437
|
+
/* чтобы не описывать все 1024 исключения в valgrind_suppress.txt */
|
|
3438
|
+
osal_iov_max = 64;
|
|
3439
|
+
#endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */
|
|
3440
|
+
|
|
3441
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
3442
|
+
SYSTEM_INFO si;
|
|
3443
|
+
GetSystemInfo(&si);
|
|
3444
|
+
globals.sys_pagesize = si.dwPageSize;
|
|
3445
|
+
globals.sys_allocation_granularity = si.dwAllocationGranularity;
|
|
3446
|
+
#else
|
|
3447
|
+
globals.sys_pagesize = sysconf(_SC_PAGE_SIZE);
|
|
3448
|
+
globals.sys_allocation_granularity = (MDBX_WORDBITS > 32) ? 65536 : 4096;
|
|
3449
|
+
globals.sys_allocation_granularity = (globals.sys_allocation_granularity > globals.sys_pagesize)
|
|
3450
|
+
? globals.sys_allocation_granularity
|
|
3451
|
+
: globals.sys_pagesize;
|
|
3452
|
+
#endif
|
|
3453
|
+
assert(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0);
|
|
3454
|
+
assert(globals.sys_allocation_granularity >= globals.sys_pagesize &&
|
|
3455
|
+
globals.sys_allocation_granularity % globals.sys_pagesize == 0);
|
|
3456
|
+
globals.sys_pagesize_ln2 = log2n_powerof2(globals.sys_pagesize);
|
|
3457
|
+
|
|
3458
|
+
#if defined(__linux__) || defined(__gnu_linux__)
|
|
3459
|
+
posix_clockid = choice_monoclock();
|
|
3460
|
+
#endif
|
|
3461
|
+
|
|
3462
|
+
#if defined(_WIN32) || defined(_WIN64)
|
|
3463
|
+
QueryPerformanceFrequency(&performance_frequency);
|
|
3464
|
+
#elif defined(__APPLE__) || defined(__MACH__)
|
|
3465
|
+
mach_timebase_info_data_t ti;
|
|
3466
|
+
mach_timebase_info(&ti);
|
|
3467
|
+
ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer;
|
|
3468
|
+
#endif
|
|
3469
|
+
monotime_limit = osal_16dot16_to_monotime(UINT32_MAX - 1);
|
|
3470
|
+
|
|
3471
|
+
uint32_t proba = UINT32_MAX;
|
|
3472
|
+
while (true) {
|
|
3473
|
+
unsigned time_conversion_checkup = osal_monotime_to_16dot16(osal_16dot16_to_monotime(proba));
|
|
3474
|
+
unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba;
|
|
3475
|
+
unsigned one_less = (proba > 0) ? proba - 1 : proba;
|
|
3476
|
+
ENSURE(nullptr, time_conversion_checkup >= one_less && time_conversion_checkup <= one_more);
|
|
3477
|
+
if (proba == 0)
|
|
3478
|
+
break;
|
|
3479
|
+
proba >>= 1;
|
|
3480
|
+
}
|
|
3481
|
+
|
|
3482
|
+
globals.bootid = osal_bootid();
|
|
3483
|
+
}
|
|
3484
|
+
|
|
3485
|
+
void osal_dtor(void) {}
|