@soulcraft/brainy 3.8.3 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brainy.d.ts +27 -0
- package/dist/brainy.js +231 -10
- package/dist/coreTypes.d.ts +10 -0
- package/dist/hnsw/hnswIndex.d.ts +2 -0
- package/dist/hnsw/hnswIndex.js +10 -0
- package/dist/neural/improvedNeuralAPI.d.ts +14 -1
- package/dist/neural/improvedNeuralAPI.js +59 -20
- package/dist/neural/naturalLanguageProcessorStatic.d.ts +1 -0
- package/dist/neural/naturalLanguageProcessorStatic.js +3 -2
- package/dist/storage/adapters/baseStorageAdapter.d.ts +59 -0
- package/dist/storage/adapters/baseStorageAdapter.js +137 -0
- package/dist/storage/adapters/fileSystemStorage.d.ts +41 -0
- package/dist/storage/adapters/fileSystemStorage.js +227 -19
- package/dist/storage/adapters/memoryStorage.d.ts +8 -0
- package/dist/storage/adapters/memoryStorage.js +48 -1
- package/dist/storage/adapters/opfsStorage.d.ts +12 -0
- package/dist/storage/adapters/opfsStorage.js +68 -0
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +34 -0
- package/dist/storage/adapters/s3CompatibleStorage.js +129 -3
- package/dist/storage/baseStorage.js +4 -3
- package/dist/storage/readOnlyOptimizations.d.ts +0 -9
- package/dist/storage/readOnlyOptimizations.js +6 -28
- package/dist/types/brainy.types.d.ts +15 -0
- package/dist/utils/metadataIndex.d.ts +5 -0
- package/dist/utils/metadataIndex.js +24 -0
- package/dist/utils/mutex.d.ts +53 -0
- package/dist/utils/mutex.js +221 -0
- package/dist/utils/paramValidation.js +20 -4
- package/package.json +1 -1
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Universal Mutex Implementation for Thread-Safe Operations
|
|
3
|
+
* Provides consistent locking across all storage adapters
|
|
4
|
+
* Critical for preventing race conditions in count operations
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* In-memory mutex for single-process scenarios
|
|
8
|
+
* Used by MemoryStorage and as fallback for other adapters
|
|
9
|
+
*/
|
|
10
|
+
export class InMemoryMutex {
|
|
11
|
+
constructor() {
|
|
12
|
+
this.locks = new Map();
|
|
13
|
+
}
|
|
14
|
+
async acquire(key, timeout = 30000) {
|
|
15
|
+
if (!this.locks.has(key)) {
|
|
16
|
+
this.locks.set(key, { queue: [], locked: false });
|
|
17
|
+
}
|
|
18
|
+
const lock = this.locks.get(key);
|
|
19
|
+
if (!lock.locked) {
|
|
20
|
+
lock.locked = true;
|
|
21
|
+
return () => this.release(key);
|
|
22
|
+
}
|
|
23
|
+
// Wait in queue
|
|
24
|
+
return new Promise((resolve, reject) => {
|
|
25
|
+
const timer = setTimeout(() => {
|
|
26
|
+
const index = lock.queue.indexOf(resolver);
|
|
27
|
+
if (index !== -1) {
|
|
28
|
+
lock.queue.splice(index, 1);
|
|
29
|
+
}
|
|
30
|
+
reject(new Error(`Mutex timeout for key: ${key}`));
|
|
31
|
+
}, timeout);
|
|
32
|
+
const resolver = () => {
|
|
33
|
+
clearTimeout(timer);
|
|
34
|
+
lock.locked = true;
|
|
35
|
+
resolve(() => this.release(key));
|
|
36
|
+
};
|
|
37
|
+
lock.queue.push(resolver);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
release(key) {
|
|
41
|
+
const lock = this.locks.get(key);
|
|
42
|
+
if (!lock)
|
|
43
|
+
return;
|
|
44
|
+
if (lock.queue.length > 0) {
|
|
45
|
+
const next = lock.queue.shift();
|
|
46
|
+
next();
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
lock.locked = false;
|
|
50
|
+
// Clean up if no waiters
|
|
51
|
+
if (lock.queue.length === 0) {
|
|
52
|
+
this.locks.delete(key);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
async runExclusive(key, fn, timeout) {
|
|
57
|
+
const release = await this.acquire(key, timeout);
|
|
58
|
+
try {
|
|
59
|
+
return await fn();
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
release();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
isLocked(key) {
|
|
66
|
+
return this.locks.get(key)?.locked || false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* File-based mutex for multi-process scenarios (Node.js)
|
|
71
|
+
* Uses atomic file operations to prevent TOCTOU races
|
|
72
|
+
*/
|
|
73
|
+
export class FileMutex {
|
|
74
|
+
constructor(lockDir) {
|
|
75
|
+
this.processLocks = new Map();
|
|
76
|
+
this.lockTimers = new Map();
|
|
77
|
+
this.lockDir = lockDir;
|
|
78
|
+
// Lazy load Node.js modules
|
|
79
|
+
if (typeof window === 'undefined') {
|
|
80
|
+
this.fs = require('fs');
|
|
81
|
+
this.path = require('path');
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async acquire(key, timeout = 30000) {
|
|
85
|
+
if (!this.fs || !this.path) {
|
|
86
|
+
throw new Error('FileMutex is only available in Node.js environments');
|
|
87
|
+
}
|
|
88
|
+
const lockFile = this.path.join(this.lockDir, `${key}.lock`);
|
|
89
|
+
const lockId = `${Date.now()}_${Math.random()}_${process.pid}`;
|
|
90
|
+
const startTime = Date.now();
|
|
91
|
+
// Ensure lock directory exists
|
|
92
|
+
await this.fs.promises.mkdir(this.lockDir, { recursive: true });
|
|
93
|
+
while (Date.now() - startTime < timeout) {
|
|
94
|
+
try {
|
|
95
|
+
// Atomic lock creation using 'wx' flag
|
|
96
|
+
await this.fs.promises.writeFile(lockFile, JSON.stringify({
|
|
97
|
+
lockId,
|
|
98
|
+
pid: process.pid,
|
|
99
|
+
timestamp: Date.now(),
|
|
100
|
+
expiresAt: Date.now() + timeout
|
|
101
|
+
}), { flag: 'wx' } // Write exclusive - fails if exists
|
|
102
|
+
);
|
|
103
|
+
// Successfully acquired lock
|
|
104
|
+
const release = () => this.release(key, lockFile, lockId);
|
|
105
|
+
this.processLocks.set(key, release);
|
|
106
|
+
// Auto-release on timeout
|
|
107
|
+
const timer = setTimeout(() => {
|
|
108
|
+
release();
|
|
109
|
+
}, timeout);
|
|
110
|
+
this.lockTimers.set(key, timer);
|
|
111
|
+
return release;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
if (error.code === 'EEXIST') {
|
|
115
|
+
// Lock exists - check if expired
|
|
116
|
+
try {
|
|
117
|
+
const data = await this.fs.promises.readFile(lockFile, 'utf-8');
|
|
118
|
+
const lock = JSON.parse(data);
|
|
119
|
+
if (lock.expiresAt < Date.now()) {
|
|
120
|
+
// Expired - try to remove
|
|
121
|
+
try {
|
|
122
|
+
await this.fs.promises.unlink(lockFile);
|
|
123
|
+
continue; // Retry acquisition
|
|
124
|
+
}
|
|
125
|
+
catch (unlinkError) {
|
|
126
|
+
if (unlinkError.code !== 'ENOENT') {
|
|
127
|
+
// Someone else removed it, continue
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// Can't read lock file, assume it's valid
|
|
135
|
+
}
|
|
136
|
+
// Wait before retry
|
|
137
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
throw error;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
throw new Error(`Failed to acquire mutex for key: ${key} after ${timeout}ms`);
|
|
145
|
+
}
|
|
146
|
+
async release(key, lockFile, lockId) {
|
|
147
|
+
// Clear timer
|
|
148
|
+
const timer = this.lockTimers.get(key);
|
|
149
|
+
if (timer) {
|
|
150
|
+
clearTimeout(timer);
|
|
151
|
+
this.lockTimers.delete(key);
|
|
152
|
+
}
|
|
153
|
+
// Remove from process locks
|
|
154
|
+
this.processLocks.delete(key);
|
|
155
|
+
try {
|
|
156
|
+
// Verify we own the lock before releasing
|
|
157
|
+
const data = await this.fs.promises.readFile(lockFile, 'utf-8');
|
|
158
|
+
const lock = JSON.parse(data);
|
|
159
|
+
if (lock.lockId === lockId) {
|
|
160
|
+
await this.fs.promises.unlink(lockFile);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
// Lock already released or doesn't exist
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
async runExclusive(key, fn, timeout) {
|
|
168
|
+
const release = await this.acquire(key, timeout);
|
|
169
|
+
try {
|
|
170
|
+
return await fn();
|
|
171
|
+
}
|
|
172
|
+
finally {
|
|
173
|
+
release();
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
isLocked(key) {
|
|
177
|
+
return this.processLocks.has(key);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Clean up all locks held by this process
|
|
181
|
+
*/
|
|
182
|
+
async cleanup() {
|
|
183
|
+
// Clear all timers
|
|
184
|
+
for (const timer of this.lockTimers.values()) {
|
|
185
|
+
clearTimeout(timer);
|
|
186
|
+
}
|
|
187
|
+
this.lockTimers.clear();
|
|
188
|
+
// Release all locks
|
|
189
|
+
const releases = Array.from(this.processLocks.values());
|
|
190
|
+
await Promise.all(releases.map(release => release()));
|
|
191
|
+
this.processLocks.clear();
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Factory to create appropriate mutex for the environment
|
|
196
|
+
*/
|
|
197
|
+
export function createMutex(options) {
|
|
198
|
+
const type = options?.type || (typeof window === 'undefined' ? 'file' : 'memory');
|
|
199
|
+
if (type === 'file' && typeof window === 'undefined') {
|
|
200
|
+
const lockDir = options?.lockDir || '.brainy/locks';
|
|
201
|
+
return new FileMutex(lockDir);
|
|
202
|
+
}
|
|
203
|
+
return new InMemoryMutex();
|
|
204
|
+
}
|
|
205
|
+
// Global mutex instance for count operations
|
|
206
|
+
let globalMutex = null;
|
|
207
|
+
export function getGlobalMutex() {
|
|
208
|
+
if (!globalMutex) {
|
|
209
|
+
globalMutex = createMutex();
|
|
210
|
+
}
|
|
211
|
+
return globalMutex;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Cleanup function for graceful shutdown
|
|
215
|
+
*/
|
|
216
|
+
export async function cleanupMutexes() {
|
|
217
|
+
if (globalMutex && 'cleanup' in globalMutex) {
|
|
218
|
+
await globalMutex.cleanup();
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=mutex.js.map
|
|
@@ -130,11 +130,24 @@ export function validateFindParams(params) {
|
|
|
130
130
|
export function validateAddParams(params) {
|
|
131
131
|
// Universal truth: must have data or vector
|
|
132
132
|
if (!params.data && !params.vector) {
|
|
133
|
-
throw new Error(
|
|
133
|
+
throw new Error(`Invalid add() parameters: Missing required field 'data'\n` +
|
|
134
|
+
`\nReceived: ${JSON.stringify({
|
|
135
|
+
type: params.type,
|
|
136
|
+
hasMetadata: !!params.metadata,
|
|
137
|
+
hasId: !!params.id
|
|
138
|
+
}, null, 2)}\n` +
|
|
139
|
+
`\nExpected one of:\n` +
|
|
140
|
+
` { data: 'text to store', type?: 'note', metadata?: {...} }\n` +
|
|
141
|
+
` { vector: [0.1, 0.2, ...], type?: 'embedding', metadata?: {...} }\n` +
|
|
142
|
+
`\nExamples:\n` +
|
|
143
|
+
` await brain.add({ data: 'Machine learning is AI', type: 'concept' })\n` +
|
|
144
|
+
` await brain.add({ data: { title: 'Doc', content: '...' }, type: 'document' })`);
|
|
134
145
|
}
|
|
135
146
|
// Validate noun type
|
|
136
147
|
if (!Object.values(NounType).includes(params.type)) {
|
|
137
|
-
throw new Error(`
|
|
148
|
+
throw new Error(`Invalid NounType: '${params.type}'\n` +
|
|
149
|
+
`\nValid types: ${Object.values(NounType).join(', ')}\n` +
|
|
150
|
+
`\nExample: await brain.add({ data: 'text', type: NounType.Note })`);
|
|
138
151
|
}
|
|
139
152
|
// Validate vector dimensions if provided
|
|
140
153
|
if (params.vector) {
|
|
@@ -182,8 +195,11 @@ export function validateRelateParams(params) {
|
|
|
182
195
|
if (params.from === params.to) {
|
|
183
196
|
throw new Error('cannot create self-referential relationship');
|
|
184
197
|
}
|
|
185
|
-
// Validate verb type
|
|
186
|
-
if (
|
|
198
|
+
// Validate verb type - default to RelatedTo if not specified
|
|
199
|
+
if (params.type === undefined) {
|
|
200
|
+
params.type = VerbType.RelatedTo;
|
|
201
|
+
}
|
|
202
|
+
else if (!Object.values(VerbType).includes(params.type)) {
|
|
187
203
|
throw new Error(`invalid VerbType: ${params.type}`);
|
|
188
204
|
}
|
|
189
205
|
// Universal truth: weight must be 0-1
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.9.0",
|
|
4
4
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|