@delali/narsil-react-native 0.0.0-stage → 0.3.1-next.151

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/LICENSE +201 -0
  2. package/NarsilReactNative.podspec +57 -0
  3. package/README.md +46 -2
  4. package/android/CMakeLists.txt +48 -0
  5. package/android/build.gradle +151 -0
  6. package/android/gradle.properties +5 -0
  7. package/android/src/main/AndroidManifest.xml +2 -0
  8. package/android/src/main/cpp/AndroidEnvironment.cpp +44 -0
  9. package/android/src/main/cpp/AndroidEnvironment.hpp +11 -0
  10. package/android/src/main/cpp/cpp-adapter.cpp +11 -0
  11. package/android/src/main/java/com/margelo/nitro/narsil/NarsilAndroidEnvironment.kt +29 -0
  12. package/android/src/main/java/com/margelo/nitro/narsil/NarsilReactNativePackage.kt +22 -0
  13. package/app.plugin.js +25 -0
  14. package/cpp/AttachedField.cpp +234 -0
  15. package/cpp/AttachedField.hpp +61 -0
  16. package/cpp/Compression.cpp +113 -0
  17. package/cpp/Compression.hpp +16 -0
  18. package/cpp/GraphHeldAlone.cpp +29 -0
  19. package/cpp/GraphHeldAlone.hpp +25 -0
  20. package/cpp/HostMemory.cpp +123 -0
  21. package/cpp/HostMemory.hpp +53 -0
  22. package/cpp/HybridNarsilCoreImpl.cpp +390 -0
  23. package/cpp/HybridNarsilCoreImpl.hpp +68 -0
  24. package/cpp/HybridNarsilFilesImpl.cpp +245 -0
  25. package/cpp/HybridNarsilFilesImpl.hpp +34 -0
  26. package/cpp/HybridNarsilHostImpl.cpp +126 -0
  27. package/cpp/HybridNarsilHostImpl.hpp +42 -0
  28. package/cpp/NativeAtomics.cpp +110 -0
  29. package/cpp/NativeAtomics.hpp +11 -0
  30. package/cpp/ProcessInfo.cpp +83 -0
  31. package/cpp/ProcessInfo.hpp +11 -0
  32. package/cpp/ThreadPriority.cpp +77 -0
  33. package/cpp/ThreadPriority.hpp +8 -0
  34. package/cpp/WorkletRuntimeDispatcher.cpp +61 -0
  35. package/cpp/WorkletRuntimeDispatcher.hpp +9 -0
  36. package/cpp/core/include/narsil_core.h +208 -0
  37. package/cpp/core/src/connect.c +213 -0
  38. package/cpp/core/src/distance.c +103 -0
  39. package/cpp/core/src/files.c +99 -0
  40. package/cpp/core/src/heap.c +84 -0
  41. package/cpp/core/src/internal.h +284 -0
  42. package/cpp/core/src/kernels_code.c +209 -0
  43. package/cpp/core/src/kernels_float.c +197 -0
  44. package/cpp/core/src/kernels_pair.c +133 -0
  45. package/cpp/core/src/last_links.c +145 -0
  46. package/cpp/core/src/links.c +195 -0
  47. package/cpp/core/src/locks.c +168 -0
  48. package/cpp/core/src/nearest_links.c +147 -0
  49. package/cpp/core/src/place.c +215 -0
  50. package/cpp/core/src/quantize.c +226 -0
  51. package/cpp/core/src/records.c +97 -0
  52. package/cpp/core/src/remove.c +205 -0
  53. package/cpp/core/src/vectors.c +139 -0
  54. package/cpp/core/src/walk.c +231 -0
  55. package/cpp/core/src/workspace.c +167 -0
  56. package/ios/PrivacyInfo.xcprivacy +23 -0
  57. package/lib/client.d.ts +19 -0
  58. package/lib/client.js +41 -0
  59. package/lib/constants.d.ts +9 -0
  60. package/lib/constants.js +9 -0
  61. package/lib/engine/byte-files.d.ts +5 -0
  62. package/lib/engine/byte-files.js +97 -0
  63. package/lib/engine/core.d.ts +3 -0
  64. package/lib/engine/core.js +113 -0
  65. package/lib/engine/files.d.ts +3 -0
  66. package/lib/engine/files.js +188 -0
  67. package/lib/engine/host.d.ts +13 -0
  68. package/lib/engine/host.js +102 -0
  69. package/lib/engine/install.d.ts +4 -0
  70. package/lib/engine/install.js +29 -0
  71. package/lib/engine/native-error.d.ts +1 -0
  72. package/lib/engine/native-error.js +12 -0
  73. package/lib/engine/stand-ins/atomics.d.ts +5 -0
  74. package/lib/engine/stand-ins/atomics.js +236 -0
  75. package/lib/engine/stand-ins/crypto.d.ts +2 -0
  76. package/lib/engine/stand-ins/crypto.js +65 -0
  77. package/lib/engine/stand-ins/process.d.ts +2 -0
  78. package/lib/engine/stand-ins/process.js +22 -0
  79. package/lib/engine/stand-ins/streams.d.ts +1 -0
  80. package/lib/engine/stand-ins/streams.js +31 -0
  81. package/lib/engine/stand-ins/structured-clone.d.ts +1 -0
  82. package/lib/engine/stand-ins/structured-clone.js +121 -0
  83. package/lib/engine/streamed.d.ts +6 -0
  84. package/lib/engine/streamed.js +29 -0
  85. package/lib/engine/transfer.d.ts +3 -0
  86. package/lib/engine/transfer.js +77 -0
  87. package/lib/errors.d.ts +25 -0
  88. package/lib/errors.js +59 -0
  89. package/lib/index.d.ts +3 -0
  90. package/lib/index.js +2 -0
  91. package/lib/methods.d.ts +5 -0
  92. package/lib/methods.js +47 -0
  93. package/lib/native-objects.d.ts +2 -0
  94. package/lib/native-objects.js +24 -0
  95. package/lib/specs/core.nitro.d.ts +79 -0
  96. package/lib/specs/core.nitro.js +1 -0
  97. package/lib/specs/files.nitro.d.ts +36 -0
  98. package/lib/specs/files.nitro.js +1 -0
  99. package/lib/specs/host.nitro.d.ts +24 -0
  100. package/lib/specs/host.nitro.js +1 -0
  101. package/lib/transferred.d.ts +15 -0
  102. package/lib/transferred.js +6 -0
  103. package/lib/types.d.ts +136 -0
  104. package/lib/types.js +1 -0
  105. package/lib/worklets/app-turns.d.ts +1 -0
  106. package/lib/worklets/app-turns.js +44 -0
  107. package/lib/worklets/client.d.ts +30 -0
  108. package/lib/worklets/client.js +144 -0
  109. package/lib/worklets/index.d.ts +4 -0
  110. package/lib/worklets/index.js +2 -0
  111. package/lib/worklets/instances.d.ts +15 -0
  112. package/lib/worklets/instances.js +194 -0
  113. package/lib/worklets/json-part.d.ts +1 -0
  114. package/lib/worklets/json-part.js +42 -0
  115. package/lib/worklets/runtime.d.ts +25 -0
  116. package/lib/worklets/runtime.js +80 -0
  117. package/lib/worklets/transfer-queue.d.ts +7 -0
  118. package/lib/worklets/transfer-queue.js +79 -0
  119. package/lib/worklets/types.d.ts +26 -0
  120. package/lib/worklets/types.js +1 -0
  121. package/nitro.json +32 -0
  122. package/nitrogen/generated/.gitattributes +1 -0
  123. package/nitrogen/generated/android/NarsilReactNative+autolinking.cmake +85 -0
  124. package/nitrogen/generated/android/NarsilReactNative+autolinking.gradle +27 -0
  125. package/nitrogen/generated/android/NarsilReactNativeOnLoad.cpp +69 -0
  126. package/nitrogen/generated/android/NarsilReactNativeOnLoad.hpp +34 -0
  127. package/nitrogen/generated/android/kotlin/com/margelo/nitro/narsil/NarsilReactNativeOnLoad.kt +35 -0
  128. package/nitrogen/generated/ios/NarsilReactNative+autolinking.rb +62 -0
  129. package/nitrogen/generated/ios/NarsilReactNative-Swift-Cxx-Bridge.cpp +17 -0
  130. package/nitrogen/generated/ios/NarsilReactNative-Swift-Cxx-Bridge.hpp +27 -0
  131. package/nitrogen/generated/ios/NarsilReactNative-Swift-Cxx-Umbrella.hpp +38 -0
  132. package/nitrogen/generated/ios/NarsilReactNativeAutolinking.mm +55 -0
  133. package/nitrogen/generated/ios/NarsilReactNativeAutolinking.swift +16 -0
  134. package/nitrogen/generated/shared/c++/DirectoryEntry.hpp +91 -0
  135. package/nitrogen/generated/shared/c++/FileStatus.hpp +103 -0
  136. package/nitrogen/generated/shared/c++/GraphMemory.hpp +120 -0
  137. package/nitrogen/generated/shared/c++/HybridNarsilAttachedGraphSpec.cpp +21 -0
  138. package/nitrogen/generated/shared/c++/HybridNarsilAttachedGraphSpec.hpp +62 -0
  139. package/nitrogen/generated/shared/c++/HybridNarsilAttachedStoreSpec.cpp +21 -0
  140. package/nitrogen/generated/shared/c++/HybridNarsilAttachedStoreSpec.hpp +62 -0
  141. package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.cpp +35 -0
  142. package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.hpp +100 -0
  143. package/nitrogen/generated/shared/c++/HybridNarsilFilesSpec.cpp +37 -0
  144. package/nitrogen/generated/shared/c++/HybridNarsilFilesSpec.hpp +86 -0
  145. package/nitrogen/generated/shared/c++/HybridNarsilHostSpec.cpp +34 -0
  146. package/nitrogen/generated/shared/c++/HybridNarsilHostSpec.hpp +78 -0
  147. package/nitrogen/generated/shared/c++/MemoryView.hpp +91 -0
  148. package/nitrogen/generated/shared/c++/NativeConnectOutcome.hpp +87 -0
  149. package/nitrogen/generated/shared/c++/NativeScores.hpp +87 -0
  150. package/nitrogen/generated/shared/c++/NativeSearchHits.hpp +91 -0
  151. package/nitrogen/generated/shared/c++/ProcessMemory.hpp +87 -0
  152. package/nitrogen/generated/shared/c++/StoreMemory.hpp +142 -0
  153. package/package.json +87 -3
  154. package/react-native.config.js +8 -0
  155. package/src/client.ts +44 -0
  156. package/src/constants.ts +9 -0
  157. package/src/engine/byte-files.ts +99 -0
  158. package/src/engine/core.ts +162 -0
  159. package/src/engine/files.ts +186 -0
  160. package/src/engine/host.ts +111 -0
  161. package/src/engine/install.ts +31 -0
  162. package/src/engine/native-error.ts +10 -0
  163. package/src/engine/stand-ins/atomics.ts +288 -0
  164. package/src/engine/stand-ins/crypto.ts +81 -0
  165. package/src/engine/stand-ins/process.ts +30 -0
  166. package/src/engine/stand-ins/streams.ts +45 -0
  167. package/src/engine/stand-ins/structured-clone.ts +118 -0
  168. package/src/engine/streamed.ts +40 -0
  169. package/src/engine/transfer.ts +72 -0
  170. package/src/errors.ts +73 -0
  171. package/src/index.ts +3 -0
  172. package/src/methods.ts +52 -0
  173. package/src/native-objects.ts +30 -0
  174. package/src/specs/core.nitro.ts +140 -0
  175. package/src/specs/files.nitro.ts +42 -0
  176. package/src/specs/host.nitro.ts +30 -0
  177. package/src/transferred.ts +18 -0
  178. package/src/types.ts +164 -0
  179. package/src/worklets/app-turns.ts +48 -0
  180. package/src/worklets/client.ts +165 -0
  181. package/src/worklets/index.ts +4 -0
  182. package/src/worklets/instances.ts +231 -0
  183. package/src/worklets/json-part.ts +38 -0
  184. package/src/worklets/runtime.ts +190 -0
  185. package/src/worklets/transfer-queue.ts +98 -0
  186. package/src/worklets/types.ts +37 -0
@@ -0,0 +1,77 @@
1
+ #include "ThreadPriority.hpp"
2
+
3
+ #include <cmath>
4
+
5
+ #if defined(__APPLE__)
6
+ #include <pthread.h>
7
+ #include <pthread/qos.h>
8
+ #elif defined(__ANDROID__)
9
+ #include <sys/resource.h>
10
+ #include <unistd.h>
11
+ #endif
12
+
13
+ namespace margelo::nitro::narsil {
14
+
15
+ namespace {
16
+
17
+ bool isWholeNumber(double value) { return std::isfinite(value) && std::trunc(value) == value; }
18
+
19
+ }
20
+
21
+ #if defined(__APPLE__)
22
+
23
+ namespace {
24
+
25
+ constexpr int HIGHEST_ENGINE_SERVICE_CLASS = static_cast<int>(QOS_CLASS_USER_INITIATED);
26
+
27
+ bool isServiceClass(int value) {
28
+ switch (value) {
29
+ case QOS_CLASS_USER_INTERACTIVE:
30
+ case QOS_CLASS_USER_INITIATED:
31
+ case QOS_CLASS_DEFAULT:
32
+ case QOS_CLASS_UTILITY:
33
+ case QOS_CLASS_BACKGROUND:
34
+ return true;
35
+ default:
36
+ return false;
37
+ }
38
+ }
39
+
40
+ }
41
+
42
+ double priorityOfThisThread() { return static_cast<double>(qos_class_self()); }
43
+
44
+ bool adoptAppThreadPriority(double appThreadPriority) {
45
+ if (!isWholeNumber(appThreadPriority)) return false;
46
+ int serviceClass = static_cast<int>(appThreadPriority);
47
+ if (!isServiceClass(serviceClass)) return false;
48
+ if (serviceClass > HIGHEST_ENGINE_SERVICE_CLASS) serviceClass = HIGHEST_ENGINE_SERVICE_CLASS;
49
+ return pthread_set_qos_class_self_np(static_cast<qos_class_t>(serviceClass), 0) == 0;
50
+ }
51
+
52
+ #elif defined(__ANDROID__)
53
+
54
+ namespace {
55
+
56
+ constexpr int HIGHEST_NICE = -20;
57
+ constexpr int LOWEST_NICE = 19;
58
+
59
+ }
60
+
61
+ double priorityOfThisThread() { return static_cast<double>(getpriority(PRIO_PROCESS, static_cast<id_t>(gettid()))); }
62
+
63
+ bool adoptAppThreadPriority(double appThreadPriority) {
64
+ if (!isWholeNumber(appThreadPriority)) return false;
65
+ if (appThreadPriority < HIGHEST_NICE || appThreadPriority > LOWEST_NICE) return false;
66
+ return setpriority(PRIO_PROCESS, static_cast<id_t>(gettid()), static_cast<int>(appThreadPriority)) == 0;
67
+ }
68
+
69
+ #else
70
+
71
+ double priorityOfThisThread() { return NAN; }
72
+
73
+ bool adoptAppThreadPriority(double) { return false; }
74
+
75
+ #endif
76
+
77
+ }
@@ -0,0 +1,8 @@
1
+ #pragma once
2
+
3
+ namespace margelo::nitro::narsil {
4
+
5
+ double priorityOfThisThread();
6
+ bool adoptAppThreadPriority(double appThreadPriority);
7
+
8
+ }
@@ -0,0 +1,61 @@
1
+ #include "WorkletRuntimeDispatcher.hpp"
2
+
3
+ #if NARSIL_LINKS_WORKLETS
4
+
5
+ #include <NitroModules/Dispatcher.hpp>
6
+ #include <worklets/WorkletRuntime/WorkletRuntime.h>
7
+
8
+ #include <functional>
9
+ #include <memory>
10
+ #include <stdexcept>
11
+ #include <utility>
12
+
13
+ namespace margelo::nitro::narsil {
14
+
15
+ namespace {
16
+
17
+ class WorkletRuntimeDispatcher final : public Dispatcher {
18
+ public:
19
+ explicit WorkletRuntimeDispatcher(std::weak_ptr<worklets::WorkletRuntime> runtime) : _runtime(std::move(runtime)) {}
20
+
21
+ void runAsync(std::function<void()>&& function) override {
22
+ std::shared_ptr<worklets::WorkletRuntime> runtime = _runtime.lock();
23
+ if (runtime == nullptr) return;
24
+ runtime->schedule([function = std::move(function)](facebook::jsi::Runtime&) { function(); });
25
+ }
26
+
27
+ void runSync(std::function<void()>&& function) override {
28
+ std::shared_ptr<worklets::WorkletRuntime> runtime = _runtime.lock();
29
+ if (runtime == nullptr) return;
30
+ runtime->runSync([&function](facebook::jsi::Runtime&) { function(); });
31
+ }
32
+
33
+ private:
34
+ std::weak_ptr<worklets::WorkletRuntime> _runtime;
35
+ };
36
+
37
+ }
38
+
39
+ bool installWorkletRuntimeDispatcher(facebook::jsi::Runtime& runtime) {
40
+ std::weak_ptr<worklets::WorkletRuntime> workletRuntime;
41
+ try {
42
+ workletRuntime = worklets::WorkletRuntime::getWeakRuntimeFromJSIRuntime(runtime);
43
+ } catch (const std::runtime_error&) {
44
+ return false;
45
+ }
46
+ if (workletRuntime.expired()) return false;
47
+ Dispatcher::installRuntimeGlobalDispatcher(runtime, std::make_shared<WorkletRuntimeDispatcher>(workletRuntime));
48
+ return true;
49
+ }
50
+
51
+ }
52
+
53
+ #else
54
+
55
+ namespace margelo::nitro::narsil {
56
+
57
+ bool installWorkletRuntimeDispatcher(facebook::jsi::Runtime&) { return false; }
58
+
59
+ }
60
+
61
+ #endif
@@ -0,0 +1,9 @@
1
+ #pragma once
2
+
3
+ #include <jsi/jsi.h>
4
+
5
+ namespace margelo::nitro::narsil {
6
+
7
+ bool installWorkletRuntimeDispatcher(facebook::jsi::Runtime& runtime);
8
+
9
+ }
@@ -0,0 +1,208 @@
1
+ #ifndef NARSIL_CORE_H
2
+ #define NARSIL_CORE_H
3
+
4
+ #include <stddef.h>
5
+ #include <stdint.h>
6
+
7
+ #ifdef __cplusplus
8
+ extern "C" {
9
+ #endif
10
+
11
+ #define NARSIL_CORE_ABI_VERSION 6
12
+
13
+ #define NARSIL_GRAPH_HEADER_WORDS 160
14
+ #define NARSIL_STORE_HEADER_WORDS 16
15
+ #define NARSIL_HELD_WORDS_PER_THREAD 32
16
+ #define NARSIL_OSQ_TRAILER_BYTES 16
17
+ #define NARSIL_MAX_PLACEMENT_LAYERS 64
18
+ #define NARSIL_LIST_WORDS_OVER_NEIGHBOURS 2
19
+
20
+ #define NARSIL_GRAPH_WORD_ENTRY_POINT 0
21
+ #define NARSIL_GRAPH_WORD_TOP_LAYER 1
22
+ #define NARSIL_GRAPH_WORD_M 2
23
+ #define NARSIL_GRAPH_WORD_MMAX0 3
24
+ #define NARSIL_GRAPH_WORD_EF_CONSTRUCTION 4
25
+ #define NARSIL_GRAPH_WORD_NODE_COUNT 32
26
+ #define NARSIL_GRAPH_WORD_TOMBSTONE_COUNT 33
27
+ #define NARSIL_GRAPH_WORD_UNLINKED_NODES 34
28
+ #define NARSIL_GRAPH_WORD_UPPER_USED 64
29
+ #define NARSIL_GRAPH_WORD_SLOTS 65
30
+ #define NARSIL_GRAPH_WORD_UPPER_CAPACITY 66
31
+ #define NARSIL_GRAPH_WORD_SLOT_CAPACITY 67
32
+ #define NARSIL_GRAPH_WORD_LOCK 96
33
+ #define NARSIL_GRAPH_WORD_WRITERS_WAITING 97
34
+ #define NARSIL_GRAPH_WORD_ENTRY_LOCK 128
35
+
36
+ #define NARSIL_STORE_WORD_SLOTS 0
37
+ #define NARSIL_STORE_WORD_LIVE_COUNT 1
38
+ #define NARSIL_STORE_WORD_CALIBRATED 3
39
+ #define NARSIL_STORE_WORD_CODE_COUNT 4
40
+ #define NARSIL_STORE_WORD_CALIBRATION_GENERATION 7
41
+
42
+ #define NARSIL_HELD_WORD_NODE 0
43
+ #define NARSIL_HELD_WORD_NODE_VALUE 1
44
+ #define NARSIL_HELD_WORD_GRAPH 2
45
+ #define NARSIL_HELD_WORD_ENTRY 5
46
+
47
+ #define NARSIL_VECTOR_IN_A_BLOCK (-1)
48
+ #define NARSIL_GRAPH_HELD_ALONE (-1)
49
+ #define NARSIL_WAKE_ENTRY_LOCK (-1)
50
+
51
+ typedef enum {
52
+ NARSIL_METRIC_COSINE = 0,
53
+ NARSIL_METRIC_DOT_PRODUCT = 1,
54
+ NARSIL_METRIC_EUCLIDEAN = 2,
55
+ } narsil_metric;
56
+
57
+ typedef enum {
58
+ NARSIL_OK = 0,
59
+ NARSIL_INVALID_ARGUMENT = 1,
60
+ NARSIL_OUT_OF_MEMORY = 2,
61
+ NARSIL_NEEDS_ROOM = 3,
62
+ NARSIL_GRAPH_NOT_HELD_ALONE = 4,
63
+ NARSIL_FILE_UNREADABLE = 5,
64
+ } narsil_status;
65
+
66
+ typedef void (*narsil_wake)(void *context, int32_t ordinal);
67
+
68
+ typedef struct {
69
+ int32_t *header;
70
+ uint8_t *node_levels;
71
+ int32_t *level0;
72
+ int32_t *upper_base;
73
+ int32_t *upper;
74
+ int32_t *locks;
75
+ uint8_t *tombstones;
76
+ int32_t *incoming;
77
+ uint8_t *nearest_link_due;
78
+ int32_t *held_locks;
79
+ uint32_t thread_slots;
80
+ narsil_wake wake;
81
+ void *wake_context;
82
+ } narsil_graph;
83
+
84
+ typedef struct {
85
+ const uint8_t *bytes;
86
+ uint64_t length;
87
+ void *mapping;
88
+ } narsil_vector_file;
89
+
90
+ typedef struct {
91
+ int32_t *header;
92
+ uint32_t dimension;
93
+ uint32_t bits;
94
+ uint8_t *const *code_blocks;
95
+ uint32_t code_block_count;
96
+ uint32_t records_per_block;
97
+ uint8_t *code_present;
98
+ float *centroid;
99
+ const float *const *vector_blocks;
100
+ uint32_t vector_block_count;
101
+ uint32_t vectors_per_block;
102
+ uint32_t vector_stride_floats;
103
+ const double *magnitudes;
104
+ const uint8_t *present;
105
+ const int32_t *vector_file;
106
+ const uint32_t *vector_offset;
107
+ const narsil_vector_file *files;
108
+ uint32_t file_count;
109
+ } narsil_store;
110
+
111
+ typedef struct narsil_workspace narsil_workspace;
112
+
113
+ typedef struct {
114
+ int32_t *ordinals;
115
+ double *distances;
116
+ uint32_t capacity;
117
+ uint32_t count;
118
+ } narsil_candidates;
119
+
120
+ typedef struct {
121
+ const uint8_t *bits;
122
+ uint32_t bytes;
123
+ } narsil_ordinal_set;
124
+
125
+ typedef struct {
126
+ const float *query;
127
+ uint32_t metric;
128
+ uint32_t candidate_count;
129
+ uint32_t thread_slot;
130
+ narsil_ordinal_set accepted;
131
+ } narsil_search_request;
132
+
133
+ typedef struct {
134
+ const float *query;
135
+ uint32_t metric;
136
+ const int32_t *ordinals;
137
+ uint32_t count;
138
+ } narsil_score_request;
139
+
140
+ typedef struct {
141
+ uint32_t metric;
142
+ int32_t ordinal;
143
+ int32_t top_layer;
144
+ uint32_t thread_slot;
145
+ uint32_t graph_held_alone;
146
+ } narsil_place_request;
147
+
148
+ typedef struct {
149
+ int32_t ordinal;
150
+ uint32_t thread_slot;
151
+ } narsil_remove_request;
152
+
153
+ typedef struct {
154
+ uint32_t metric;
155
+ uint32_t thread_slot;
156
+ } narsil_compact_request;
157
+
158
+ typedef struct {
159
+ uint32_t metric;
160
+ uint32_t thread_slot;
161
+ const int32_t *nearby;
162
+ uint32_t nearby_count;
163
+ } narsil_connect_request;
164
+
165
+ typedef struct {
166
+ uint32_t metric;
167
+ const int32_t *ordinals;
168
+ uint32_t count;
169
+ } narsil_ordinals_request;
170
+
171
+ int32_t narsil_core_abi_version(void);
172
+ uint32_t narsil_record_bytes(uint32_t dimension, uint32_t bits);
173
+
174
+ narsil_status narsil_workspace_create(narsil_workspace **workspace);
175
+ void narsil_workspace_destroy(narsil_workspace *workspace);
176
+ size_t narsil_workspace_bytes(const narsil_workspace *workspace);
177
+
178
+ narsil_status narsil_vector_file_map(const char *path, narsil_vector_file *file);
179
+ void narsil_vector_file_unmap(narsil_vector_file *file);
180
+
181
+ narsil_status narsil_search(narsil_workspace *workspace, const narsil_graph *graph, const narsil_store *store,
182
+ const narsil_search_request *request, narsil_candidates *nearest);
183
+
184
+ narsil_status narsil_score(narsil_workspace *workspace, const narsil_store *store, const narsil_score_request *request,
185
+ double *distances);
186
+
187
+ narsil_status narsil_place(narsil_workspace *workspace, const narsil_graph *graph, const narsil_store *store,
188
+ const narsil_place_request *request);
189
+
190
+ narsil_status narsil_remove(const narsil_graph *graph, const narsil_remove_request *request);
191
+
192
+ narsil_status narsil_compact(narsil_workspace *workspace, const narsil_graph *graph, const narsil_store *store,
193
+ const narsil_compact_request *request);
194
+
195
+ narsil_status narsil_connect_unreached(narsil_workspace *workspace, const narsil_graph *graph,
196
+ const narsil_store *store, const narsil_connect_request *request);
197
+
198
+ narsil_status narsil_quantise(narsil_workspace *workspace, const narsil_store *store,
199
+ const narsil_ordinals_request *request);
200
+
201
+ narsil_status narsil_measure(narsil_workspace *workspace, const narsil_store *store,
202
+ const narsil_ordinals_request *request, double *totals);
203
+
204
+ #ifdef __cplusplus
205
+ }
206
+ #endif
207
+
208
+ #endif
@@ -0,0 +1,213 @@
1
+ #include "internal.h"
2
+
3
+ #include "../include/narsil_core.h"
4
+
5
+ #include <stddef.h>
6
+ #include <stdint.h>
7
+ #include <stdlib.h>
8
+
9
+ typedef struct {
10
+ uint8_t *reached;
11
+ int32_t *first_reached_through;
12
+ int32_t *queue;
13
+ uint32_t head;
14
+ uint32_t tail;
15
+ } layer_walk;
16
+
17
+ typedef struct {
18
+ int32_t from;
19
+ int32_t to;
20
+ } base_link;
21
+
22
+ typedef struct {
23
+ int32_t *list;
24
+ uint32_t count;
25
+ uint8_t *holds;
26
+ } nearby_nodes;
27
+
28
+ typedef int (*host_test)(const walk_context *context, const layer_walk *walk, int32_t host);
29
+
30
+ static int32_t *base_list(const walk_context *context, int32_t ordinal, uint32_t *count) {
31
+ uint32_t capacity = 0;
32
+ node_layer base = {ordinal, 0};
33
+ int32_t *list = list_at(context, base, &capacity);
34
+ *count = list == NULL ? 0 : stored_count(list, capacity);
35
+ return list;
36
+ }
37
+
38
+ static void reach(layer_walk *walk, base_link arrival) {
39
+ walk->reached[arrival.to] = 1;
40
+ walk->first_reached_through[arrival.to] = arrival.from;
41
+ walk->queue[walk->tail++] = arrival.to;
42
+ }
43
+
44
+ static void walk_from(const walk_context *context, layer_walk *walk, base_link arrival) {
45
+ if (walk->reached[arrival.to] == 1) { return; }
46
+ reach(walk, arrival);
47
+ while (walk->head < walk->tail) {
48
+ int32_t ordinal = walk->queue[walk->head++];
49
+ uint32_t count = 0;
50
+ const int32_t *list = base_list(context, ordinal, &count);
51
+ for (uint32_t i = 1; i <= count; i++) {
52
+ int32_t next = list[i];
53
+ if (node_top_layer(context, next) < 0 || walk->reached[next] == 1) { continue; }
54
+ base_link onward = {ordinal, next};
55
+ reach(walk, onward);
56
+ }
57
+ }
58
+ }
59
+
60
+ static int has_room(const walk_context *context, const layer_walk *walk, int32_t host) {
61
+ (void)walk;
62
+ uint32_t count = 0;
63
+ const int32_t *list = base_list(context, host, &count);
64
+ return list != NULL && count < list_limit(context->graph, 0);
65
+ }
66
+
67
+ static int reached_through_another_node(const walk_context *context, const layer_walk *walk, base_link link) {
68
+ if (link.to < 0 || (uint32_t)link.to >= context->slots || walk->reached[link.to] != 1) { return 0; }
69
+ int32_t through = walk->first_reached_through[link.to];
70
+ return through != NO_NODE && through != link.from;
71
+ }
72
+
73
+ static int holds_link_outside_the_walk(const walk_context *context, const layer_walk *walk, int32_t host) {
74
+ uint32_t count = 0;
75
+ const int32_t *list = base_list(context, host, &count);
76
+ for (uint32_t i = 1; i <= count; i++) {
77
+ base_link link = {host, list[i]};
78
+ if (reached_through_another_node(context, walk, link)) { return 1; }
79
+ }
80
+ return 0;
81
+ }
82
+
83
+ typedef struct {
84
+ int32_t ordinal;
85
+ double distance;
86
+ } host_choice;
87
+
88
+ static void consider(const walk_context *context, const layer_walk *walk, int32_t orphan, int32_t host, host_test test,
89
+ host_choice *best) {
90
+ if (host == orphan || walk->reached[host] != 1 || !is_live(context, host) || !test(context, walk, host)) { return; }
91
+ double away = pair_distance(context, orphan, host);
92
+ if (best->ordinal == NO_NODE || away < best->distance) {
93
+ best->ordinal = host;
94
+ best->distance = away;
95
+ }
96
+ }
97
+
98
+ static int32_t nearest_host(const walk_context *context, const layer_walk *walk, int32_t orphan,
99
+ const nearby_nodes *nearby, host_test test) {
100
+ host_choice best = {NO_NODE, 0};
101
+ for (uint32_t i = 0; i < nearby->count; i++) { consider(context, walk, orphan, nearby->list[i], test, &best); }
102
+ uint32_t count = 0;
103
+ const int32_t *list = base_list(context, orphan, &count);
104
+ for (uint32_t i = 1; i <= count; i++) {
105
+ int32_t host = list[i];
106
+ if (host < 0 || (uint32_t)host >= context->slots || nearby->holds[host] == 1) { continue; }
107
+ consider(context, walk, orphan, host, test, &best);
108
+ }
109
+ if (best.ordinal != NO_NODE) { return best.ordinal; }
110
+ for (uint32_t host = 0; host < context->slots; host++) {
111
+ consider(context, walk, orphan, (int32_t)host, test, &best);
112
+ }
113
+ return best.ordinal;
114
+ }
115
+
116
+ static void take_out_farthest_link_outside_the_walk(const walk_context *context, const layer_walk *walk, int32_t host) {
117
+ node_lock(context, host);
118
+ uint32_t count = 0;
119
+ int32_t *list = base_list(context, host, &count);
120
+ uint32_t farthest = 0;
121
+ double farthest_distance = 0;
122
+ for (uint32_t i = 1; i <= count; i++) {
123
+ base_link link = {host, list[i]};
124
+ if (!reached_through_another_node(context, walk, link)) { continue; }
125
+ double away = pair_distance(context, host, list[i]);
126
+ if (farthest == 0 || away > farthest_distance) {
127
+ farthest = i;
128
+ farthest_distance = away;
129
+ }
130
+ }
131
+ if (farthest != 0) {
132
+ int32_t taken = list[farthest];
133
+ for (uint32_t i = farthest; i < count; i++) { list[i] = list[i + 1]; }
134
+ list[0] = (int32_t)(count - 1);
135
+ note_unlinked(context, taken);
136
+ }
137
+ node_unlock(context, host);
138
+ }
139
+
140
+ static void append_to(const walk_context *context, node_layer host, int32_t orphan) {
141
+ node_lock(context, host.ordinal);
142
+ list_add(context, host, orphan);
143
+ node_unlock(context, host.ordinal);
144
+ }
145
+
146
+ static void gather_nearby(const walk_context *context, const narsil_connect_request *request, nearby_nodes *nearby) {
147
+ for (uint32_t i = 0; i < request->nearby_count; i++) {
148
+ int32_t ordinal = request->nearby[i];
149
+ if (ordinal < 0 || (uint32_t)ordinal >= context->slots || nearby->holds[ordinal] == 1) { continue; }
150
+ nearby->holds[ordinal] = 1;
151
+ nearby->list[nearby->count++] = ordinal;
152
+ }
153
+ }
154
+
155
+ static void connect_every_unreached(const walk_context *context, layer_walk *walk, const nearby_nodes *nearby) {
156
+ for (uint32_t ordinal = 0; ordinal < context->slots; ordinal++) {
157
+ int32_t orphan = (int32_t)ordinal;
158
+ if (walk->reached[orphan] == 1 || !is_live(context, orphan)) { continue; }
159
+ int32_t host = nearest_host(context, walk, orphan, nearby, has_room);
160
+ if (host == NO_NODE) {
161
+ host = nearest_host(context, walk, orphan, nearby, holds_link_outside_the_walk);
162
+ if (host == NO_NODE) { continue; }
163
+ take_out_farthest_link_outside_the_walk(context, walk, host);
164
+ }
165
+ node_layer host_base = {host, 0};
166
+ append_to(context, host_base, orphan);
167
+ base_link arrival = {host, orphan};
168
+ walk_from(context, walk, arrival);
169
+ }
170
+ }
171
+
172
+ static narsil_status begin_connect(walk_context *context, narsil_workspace *workspace, const narsil_graph *graph,
173
+ const narsil_store *store, const narsil_connect_request *request) {
174
+ if (request == NULL || (request->nearby == NULL && request->nearby_count > 0)) { return NARSIL_INVALID_ARGUMENT; }
175
+ narsil_status status = begin_walk(context, workspace, graph, store, request->metric);
176
+ if (status != NARSIL_OK) { return status; }
177
+ if (request->thread_slot >= graph->thread_slots) { return NARSIL_INVALID_ARGUMENT; }
178
+ if (load_word(graph->header, NARSIL_GRAPH_WORD_LOCK) != NARSIL_GRAPH_HELD_ALONE) {
179
+ return NARSIL_GRAPH_NOT_HELD_ALONE;
180
+ }
181
+ context->thread_slot = request->thread_slot;
182
+ return read_graph_extent(context);
183
+ }
184
+
185
+ narsil_status narsil_connect_unreached(narsil_workspace *workspace, const narsil_graph *graph,
186
+ const narsil_store *store, const narsil_connect_request *request) {
187
+ walk_context context;
188
+ narsil_status status = begin_connect(&context, workspace, graph, store, request);
189
+ if (status != NARSIL_OK) { return status; }
190
+ status = link_from_nearest(&context);
191
+ if (status != NARSIL_OK) { return status; }
192
+ int32_t entry = load_word(graph->header, NARSIL_GRAPH_WORD_ENTRY_POINT);
193
+ if (node_top_layer(&context, entry) < 0) { return NARSIL_OK; }
194
+ size_t slots = context.slots;
195
+ layer_walk walk = {calloc(slots, sizeof *walk.reached), calloc(slots, sizeof *walk.first_reached_through),
196
+ calloc(slots, sizeof *walk.queue), 0, 0};
197
+ nearby_nodes nearby = {calloc((size_t)request->nearby_count + 1, sizeof *nearby.list), 0, calloc(slots, 1)};
198
+ if (walk.reached != NULL && walk.first_reached_through != NULL && walk.queue != NULL && nearby.list != NULL &&
199
+ nearby.holds != NULL) {
200
+ base_link arrival = {NO_NODE, entry};
201
+ walk_from(&context, &walk, arrival);
202
+ gather_nearby(&context, request, &nearby);
203
+ connect_every_unreached(&context, &walk, &nearby);
204
+ } else {
205
+ status = NARSIL_OUT_OF_MEMORY;
206
+ }
207
+ free(walk.reached);
208
+ free(walk.first_reached_through);
209
+ free(walk.queue);
210
+ free(nearby.list);
211
+ free(nearby.holds);
212
+ return status;
213
+ }
@@ -0,0 +1,103 @@
1
+ #include "internal.h"
2
+
3
+ #include "../include/narsil_core.h"
4
+
5
+ #include <math.h>
6
+ #include <stddef.h>
7
+ #include <stdint.h>
8
+ #include <string.h>
9
+
10
+ typedef struct {
11
+ double lower;
12
+ double upper;
13
+ double correction;
14
+ double sum;
15
+ uint32_t bits;
16
+ } code_summary;
17
+
18
+ static uint32_t level_products(const walk_context *context, const uint8_t *record) {
19
+ const narsil_store *store = context->store;
20
+ const prepared_code *query = &context->code;
21
+ if (store->bits == BITS_PER_BYTE) { return kernel_products_8x8(record, query->levels, store->dimension); }
22
+ if (store->bits == 4) {
23
+ return kernel_products_4x4(record, query->low_levels, query->high_levels, context->code_bytes);
24
+ }
25
+ packed_code document = {record, context->code_bytes, store->bits};
26
+ return kernel_products_bits(document, query->planes);
27
+ }
28
+
29
+ static float trailer_float(const uint8_t *trailer, size_t offset) {
30
+ float value = 0;
31
+ memcpy(&value, trailer + offset, sizeof value);
32
+ return value;
33
+ }
34
+
35
+ static code_summary summary_of_record(const walk_context *context, const uint8_t *record) {
36
+ const uint8_t *trailer = record + context->code_bytes;
37
+ uint32_t sum_of_levels = 0;
38
+ memcpy(&sum_of_levels, trailer + TRAILER_SUM, sizeof sum_of_levels);
39
+ code_summary summary = {(double)trailer_float(trailer, TRAILER_LOWER), (double)trailer_float(trailer, TRAILER_UPPER),
40
+ (double)trailer_float(trailer, TRAILER_CORRECTION), (double)sum_of_levels,
41
+ context->store->bits};
42
+ return summary;
43
+ }
44
+
45
+ static double distance_from_products(const walk_context *context, uint32_t products, code_summary document,
46
+ code_summary query) {
47
+ double document_step = (document.upper - document.lower) / (double)((1U << document.bits) - 1);
48
+ double query_step = (query.upper - query.lower) / (double)((1U << query.bits) - 1);
49
+ double centred = (document.lower * query.lower * context->store->dimension) +
50
+ (query.lower * document_step * document.sum) + (document.lower * query_step * query.sum) +
51
+ (document_step * query_step * (double)products);
52
+ if (context->metric == NARSIL_METRIC_EUCLIDEAN) {
53
+ double squared = document.correction + query.correction - (2 * centred);
54
+ return sqrt(squared > 0 ? squared : 0);
55
+ }
56
+ double similarity = centred + document.correction + query.correction - context->centroid_dot;
57
+ if (context->metric == NARSIL_METRIC_DOT_PRODUCT) { return -similarity; }
58
+ if (similarity < -1) { similarity = -1; }
59
+ if (similarity > 1) { similarity = 1; }
60
+ return 1 - similarity;
61
+ }
62
+
63
+ double estimate_distance(const walk_context *context, const uint8_t *record) {
64
+ const prepared_code *code = &context->code;
65
+ code_summary query = {code->range.lower, code->range.upper, code->correction, code->sum, code->bits};
66
+ return distance_from_products(context, level_products(context, record), summary_of_record(context, record), query);
67
+ }
68
+
69
+ double estimate_pair_distance(const walk_context *context, const uint8_t *first, const uint8_t *second) {
70
+ packed_code document = {first, context->code_bytes, context->store->bits};
71
+ uint32_t products = kernel_pair_products(document, second, context->store->dimension);
72
+ return distance_from_products(context, products, summary_of_record(context, first),
73
+ summary_of_record(context, second));
74
+ }
75
+
76
+ double walk_distance(const walk_context *context, int32_t ordinal) {
77
+ if (!context->uses_codes) { return float_distance(context, context->query, ordinal); }
78
+ if (ordinal < 0 || (uint32_t)ordinal >= context->store_slots) { return HUGE_VAL; }
79
+ if (context->store->code_present[ordinal] != 1) { return HUGE_VAL; }
80
+ const uint8_t *record = stored_record(context->store, ordinal);
81
+ return record == NULL ? HUGE_VAL : estimate_distance(context, record);
82
+ }
83
+
84
+ narsil_status narsil_score(narsil_workspace *workspace, const narsil_store *store, const narsil_score_request *request,
85
+ double *distances) {
86
+ if (workspace == NULL || store == NULL || request == NULL || distances == NULL) { return NARSIL_INVALID_ARGUMENT; }
87
+ if (request->query == NULL || (request->count > 0 && request->ordinals == NULL)) { return NARSIL_INVALID_ARGUMENT; }
88
+ if (!store_is_complete(store) || !is_metric(request->metric) || store->dimension == 0) {
89
+ return NARSIL_INVALID_ARGUMENT;
90
+ }
91
+ narsil_status status = workspace_reserve_dimension(workspace, store->dimension);
92
+ if (status != NARSIL_OK) { return status; }
93
+ walk_context context;
94
+ memset(&context, 0, sizeof context);
95
+ context.store = store;
96
+ context.workspace = workspace;
97
+ context.metric = (narsil_metric)request->metric;
98
+ float_query measured = {request->query, (double)kernel_magnitude(request->query, store->dimension)};
99
+ for (uint32_t i = 0; i < request->count; i++) {
100
+ distances[i] = float_distance(&context, measured, request->ordinals[i]);
101
+ }
102
+ return NARSIL_OK;
103
+ }