@levalicious/server-memory 0.0.10 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -27
- package/dist/index.js +4 -0
- package/dist/scripts/migrate-jsonl.js +169 -0
- package/dist/scripts/verify-migration.js +39 -0
- package/dist/server.js +763 -536
- package/dist/src/graphfile.js +560 -0
- package/dist/src/memoryfile.js +121 -0
- package/dist/src/pagerank.js +78 -0
- package/dist/src/stringtable.js +373 -0
- package/dist/tests/concurrency.test.js +189 -0
- package/dist/tests/memory-server.test.js +225 -53
- package/package.json +6 -4
|
@@ -0,0 +1,560 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Graph record types — binary layouts for entities, adjacency blocks, and node log.
|
|
3
|
+
*
|
|
4
|
+
* All records live in a MemoryFile (graph.mem). Variable-length strings are
|
|
5
|
+
* stored in a separate StringTable (strings.mem) and referenced by u32 ID.
|
|
6
|
+
*
|
|
7
|
+
* Graph file layout:
|
|
8
|
+
* [memfile header: 32 bytes]
|
|
9
|
+
* [graph header block: first allocation]
|
|
10
|
+
* u64 node_log_offset
|
|
11
|
+
* u64 structural_total total structural walk visits (global counter)
|
|
12
|
+
* u64 walker_total total walker visits (global counter)
|
|
13
|
+
* [entity records, adj blocks, node log ...]
|
|
14
|
+
*
|
|
15
|
+
* EntityRecord: 64 bytes fixed
|
|
16
|
+
* u32 name_id string table ID
|
|
17
|
+
* u32 type_id string table ID
|
|
18
|
+
* u64 adj_offset offset to AdjBlock (0 = no edges)
|
|
19
|
+
* u64 mtime general modification timestamp (ms)
|
|
20
|
+
* u64 obsMtime observation modification timestamp (ms)
|
|
21
|
+
* u8 obs_count 0, 1, or 2
|
|
22
|
+
* u8 _pad[3]
|
|
23
|
+
* u32 obs0_id string table ID (0 = empty)
|
|
24
|
+
* u32 obs1_id string table ID (0 = empty)
|
|
25
|
+
* u64 structural_visits structural PageRank visit count
|
|
26
|
+
* u64 walker_visits walker PageRank visit count
|
|
27
|
+
*
|
|
28
|
+
* AdjBlock:
|
|
29
|
+
* u32 count
|
|
30
|
+
* u32 capacity
|
|
31
|
+
* AdjEntry[capacity]:
|
|
32
|
+
* u64 target_and_dir 62 bits target offset | 2 bits direction
|
|
33
|
+
* u32 relType_id string table ID
|
|
34
|
+
* u32 _pad
|
|
35
|
+
* u64 mtime
|
|
36
|
+
*
|
|
37
|
+
* AdjEntry direction bits (in the low 2 bits of target_and_dir):
|
|
38
|
+
* 0b00 = FORWARD (this entity is 'from')
|
|
39
|
+
* 0b01 = BACKWARD (this entity is 'to', edge points at us)
|
|
40
|
+
* 0b10 = BIDIR (reserved)
|
|
41
|
+
*
|
|
42
|
+
* NodeLog:
|
|
43
|
+
* u32 count
|
|
44
|
+
* u32 capacity
|
|
45
|
+
* u64 offsets[capacity]
|
|
46
|
+
*/
|
|
47
|
+
import { MemoryFile } from './memoryfile.js';
|
|
48
|
+
// --- Constants ---
|
|
49
|
+
export const ENTITY_RECORD_SIZE = 64;
|
|
50
|
+
export const ADJ_ENTRY_SIZE = 24; // 8 + 4 + 4 + 8, naturally aligned
|
|
51
|
+
const ADJ_HEADER_SIZE = 8; // count:u32 + capacity:u32
|
|
52
|
+
const NODE_LOG_HEADER_SIZE = 8; // count:u32 + capacity:u32
|
|
53
|
+
const GRAPH_HEADER_SIZE = 24; // node_log_offset:u64 + structural_total:u64 + walker_total:u64
|
|
54
|
+
const INITIAL_ADJ_CAPACITY = 4;
|
|
55
|
+
const INITIAL_LOG_CAPACITY = 256;
|
|
56
|
+
// Direction flags
|
|
57
|
+
export const DIR_FORWARD = 0n;
|
|
58
|
+
export const DIR_BACKWARD = 1n;
|
|
59
|
+
export const DIR_BIDIR = 2n;
|
|
60
|
+
const DIR_MASK = 3n;
|
|
61
|
+
const OFFSET_SHIFT = 2n;
|
|
62
|
+
// Entity record field offsets
|
|
63
|
+
const E_NAME_ID = 0;
|
|
64
|
+
const E_TYPE_ID = 4;
|
|
65
|
+
const E_ADJ_OFFSET = 8;
|
|
66
|
+
const E_MTIME = 16;
|
|
67
|
+
const E_OBS_MTIME = 24;
|
|
68
|
+
const E_OBS_COUNT = 32;
|
|
69
|
+
// 3 bytes pad at 33
|
|
70
|
+
const E_OBS0_ID = 36;
|
|
71
|
+
const E_OBS1_ID = 40;
|
|
72
|
+
// 4 bytes pad at 44
|
|
73
|
+
const E_STRUCTURAL_VISITS = 48; // u64: 48..55, 8-aligned
|
|
74
|
+
const E_WALKER_VISITS = 56; // u64: 56..63, 8-aligned
|
|
75
|
+
// total = 64
|
|
76
|
+
// AdjEntry field offsets (within each entry)
|
|
77
|
+
const AE_TARGET_DIR = 0;
|
|
78
|
+
const AE_RELTYPE_ID = 8;
|
|
79
|
+
// 4 bytes pad at 12
|
|
80
|
+
const AE_MTIME = 16;
|
|
81
|
+
// Graph header field offsets
|
|
82
|
+
const GH_NODE_LOG_OFFSET = 0;
|
|
83
|
+
const GH_STRUCTURAL_TOTAL = 8;
|
|
84
|
+
const GH_WALKER_TOTAL = 16;
|
|
85
|
+
// --- Encoding helpers ---
|
|
86
|
+
export function packTargetDir(targetOffset, direction) {
|
|
87
|
+
return (targetOffset << OFFSET_SHIFT) | (direction & DIR_MASK);
|
|
88
|
+
}
|
|
89
|
+
export function unpackTarget(packed) {
|
|
90
|
+
return packed >> OFFSET_SHIFT;
|
|
91
|
+
}
|
|
92
|
+
export function unpackDir(packed) {
|
|
93
|
+
return packed & DIR_MASK;
|
|
94
|
+
}
|
|
95
|
+
// --- Read/write functions ---
|
|
96
|
+
export function readEntityRecord(mf, offset) {
|
|
97
|
+
const buf = mf.read(offset, BigInt(ENTITY_RECORD_SIZE));
|
|
98
|
+
return {
|
|
99
|
+
offset,
|
|
100
|
+
nameId: buf.readUInt32LE(E_NAME_ID),
|
|
101
|
+
typeId: buf.readUInt32LE(E_TYPE_ID),
|
|
102
|
+
adjOffset: buf.readBigUInt64LE(E_ADJ_OFFSET),
|
|
103
|
+
mtime: buf.readBigUInt64LE(E_MTIME),
|
|
104
|
+
obsMtime: buf.readBigUInt64LE(E_OBS_MTIME),
|
|
105
|
+
obsCount: buf.readUInt8(E_OBS_COUNT),
|
|
106
|
+
obs0Id: buf.readUInt32LE(E_OBS0_ID),
|
|
107
|
+
obs1Id: buf.readUInt32LE(E_OBS1_ID),
|
|
108
|
+
structuralVisits: buf.readBigUInt64LE(E_STRUCTURAL_VISITS),
|
|
109
|
+
walkerVisits: buf.readBigUInt64LE(E_WALKER_VISITS),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
export function writeEntityRecord(mf, rec) {
|
|
113
|
+
const buf = Buffer.alloc(ENTITY_RECORD_SIZE);
|
|
114
|
+
buf.writeUInt32LE(rec.nameId, E_NAME_ID);
|
|
115
|
+
buf.writeUInt32LE(rec.typeId, E_TYPE_ID);
|
|
116
|
+
buf.writeBigUInt64LE(rec.adjOffset, E_ADJ_OFFSET);
|
|
117
|
+
buf.writeBigUInt64LE(rec.mtime, E_MTIME);
|
|
118
|
+
buf.writeBigUInt64LE(rec.obsMtime, E_OBS_MTIME);
|
|
119
|
+
buf.writeUInt8(rec.obsCount, E_OBS_COUNT);
|
|
120
|
+
buf.writeUInt32LE(rec.obs0Id, E_OBS0_ID);
|
|
121
|
+
buf.writeUInt32LE(rec.obs1Id, E_OBS1_ID);
|
|
122
|
+
buf.writeBigUInt64LE(rec.structuralVisits, E_STRUCTURAL_VISITS);
|
|
123
|
+
buf.writeBigUInt64LE(rec.walkerVisits, E_WALKER_VISITS);
|
|
124
|
+
mf.write(rec.offset, buf);
|
|
125
|
+
}
|
|
126
|
+
export function readAdjBlock(mf, adjOffset) {
|
|
127
|
+
const header = mf.read(adjOffset, BigInt(ADJ_HEADER_SIZE));
|
|
128
|
+
const count = header.readUInt32LE(0);
|
|
129
|
+
const capacity = header.readUInt32LE(4);
|
|
130
|
+
const entries = [];
|
|
131
|
+
if (count > 0) {
|
|
132
|
+
const dataSize = count * ADJ_ENTRY_SIZE;
|
|
133
|
+
const data = mf.read(adjOffset + BigInt(ADJ_HEADER_SIZE), BigInt(dataSize));
|
|
134
|
+
for (let i = 0; i < count; i++) {
|
|
135
|
+
const base = i * ADJ_ENTRY_SIZE;
|
|
136
|
+
const packed = data.readBigUInt64LE(base + AE_TARGET_DIR);
|
|
137
|
+
entries.push({
|
|
138
|
+
targetOffset: unpackTarget(packed),
|
|
139
|
+
direction: unpackDir(packed),
|
|
140
|
+
relTypeId: data.readUInt32LE(base + AE_RELTYPE_ID),
|
|
141
|
+
mtime: data.readBigUInt64LE(base + AE_MTIME),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return { count, capacity, entries };
|
|
146
|
+
}
|
|
147
|
+
function writeAdjEntry(buf, offset, entry) {
|
|
148
|
+
buf.writeBigUInt64LE(packTargetDir(entry.targetOffset, entry.direction), offset + AE_TARGET_DIR);
|
|
149
|
+
buf.writeUInt32LE(entry.relTypeId, offset + AE_RELTYPE_ID);
|
|
150
|
+
buf.writeUInt32LE(0, offset + 12); // pad
|
|
151
|
+
buf.writeBigUInt64LE(entry.mtime, offset + AE_MTIME);
|
|
152
|
+
}
|
|
153
|
+
// --- NodeLog ---
|
|
154
|
+
export function readNodeLog(mf, logOffset) {
|
|
155
|
+
const header = mf.read(logOffset, BigInt(NODE_LOG_HEADER_SIZE));
|
|
156
|
+
const count = header.readUInt32LE(0);
|
|
157
|
+
const capacity = header.readUInt32LE(4);
|
|
158
|
+
const offsets = [];
|
|
159
|
+
if (count > 0) {
|
|
160
|
+
const data = mf.read(logOffset + BigInt(NODE_LOG_HEADER_SIZE), BigInt(count * 8));
|
|
161
|
+
for (let i = 0; i < count; i++) {
|
|
162
|
+
offsets.push(data.readBigUInt64LE(i * 8));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return { count, capacity, offsets };
|
|
166
|
+
}
|
|
167
|
+
// --- GraphFile: high-level operations ---
|
|
168
|
+
export class GraphFile {
|
|
169
|
+
mf;
|
|
170
|
+
st;
|
|
171
|
+
graphHeaderOffset;
|
|
172
|
+
constructor(graphPath, stringTable, initialSize = 65536) {
|
|
173
|
+
this.mf = new MemoryFile(graphPath, initialSize);
|
|
174
|
+
this.st = stringTable;
|
|
175
|
+
const stats = this.mf.stats();
|
|
176
|
+
if (stats.allocated <= 32n) {
|
|
177
|
+
this.graphHeaderOffset = this.initGraphHeader();
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
// First allocation is graph header, at offset 40 (32 memfile header + 8 alloc_t header)
|
|
181
|
+
this.graphHeaderOffset = 40n;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
initGraphHeader() {
|
|
185
|
+
// Allocate graph header block
|
|
186
|
+
const hdrOffset = this.mf.alloc(BigInt(GRAPH_HEADER_SIZE));
|
|
187
|
+
if (hdrOffset === 0n)
|
|
188
|
+
throw new Error('GraphFile: failed to allocate header');
|
|
189
|
+
// Allocate initial node log
|
|
190
|
+
const logSize = NODE_LOG_HEADER_SIZE + INITIAL_LOG_CAPACITY * 8;
|
|
191
|
+
const logOffset = this.mf.alloc(BigInt(logSize));
|
|
192
|
+
if (logOffset === 0n)
|
|
193
|
+
throw new Error('GraphFile: failed to allocate node log');
|
|
194
|
+
// Write node log header: count=0, capacity
|
|
195
|
+
const logHeader = Buffer.alloc(NODE_LOG_HEADER_SIZE);
|
|
196
|
+
logHeader.writeUInt32LE(0, 0);
|
|
197
|
+
logHeader.writeUInt32LE(INITIAL_LOG_CAPACITY, 4);
|
|
198
|
+
this.mf.write(logOffset, logHeader);
|
|
199
|
+
// Write graph header: node_log_offset + global PageRank counters
|
|
200
|
+
const hdr = Buffer.alloc(GRAPH_HEADER_SIZE);
|
|
201
|
+
hdr.writeBigUInt64LE(logOffset, GH_NODE_LOG_OFFSET);
|
|
202
|
+
hdr.writeBigUInt64LE(0n, GH_STRUCTURAL_TOTAL);
|
|
203
|
+
hdr.writeBigUInt64LE(0n, GH_WALKER_TOTAL);
|
|
204
|
+
this.mf.write(hdrOffset, hdr);
|
|
205
|
+
return hdrOffset;
|
|
206
|
+
}
|
|
207
|
+
// --- Header access ---
|
|
208
|
+
getNodeLogOffset() {
|
|
209
|
+
const buf = this.mf.read(this.graphHeaderOffset, BigInt(GRAPH_HEADER_SIZE));
|
|
210
|
+
return buf.readBigUInt64LE(GH_NODE_LOG_OFFSET);
|
|
211
|
+
}
|
|
212
|
+
setNodeLogOffset(offset) {
|
|
213
|
+
const buf = Buffer.alloc(8);
|
|
214
|
+
buf.writeBigUInt64LE(offset, 0);
|
|
215
|
+
this.mf.write(this.graphHeaderOffset + BigInt(GH_NODE_LOG_OFFSET), buf);
|
|
216
|
+
}
|
|
217
|
+
// --- Entity CRUD ---
|
|
218
|
+
createEntity(name, entityType, mtime, obsMtime) {
|
|
219
|
+
const nameId = Number(this.st.intern(name));
|
|
220
|
+
const typeId = Number(this.st.intern(entityType));
|
|
221
|
+
const offset = this.mf.alloc(BigInt(ENTITY_RECORD_SIZE));
|
|
222
|
+
if (offset === 0n)
|
|
223
|
+
throw new Error('GraphFile: entity alloc failed');
|
|
224
|
+
const rec = {
|
|
225
|
+
offset,
|
|
226
|
+
nameId,
|
|
227
|
+
typeId,
|
|
228
|
+
adjOffset: 0n,
|
|
229
|
+
mtime,
|
|
230
|
+
obsMtime: obsMtime ?? mtime,
|
|
231
|
+
obsCount: 0,
|
|
232
|
+
obs0Id: 0,
|
|
233
|
+
obs1Id: 0,
|
|
234
|
+
structuralVisits: 0n,
|
|
235
|
+
walkerVisits: 0n,
|
|
236
|
+
};
|
|
237
|
+
writeEntityRecord(this.mf, rec);
|
|
238
|
+
this.nodeLogAppend(offset);
|
|
239
|
+
return rec;
|
|
240
|
+
}
|
|
241
|
+
readEntity(offset) {
|
|
242
|
+
return readEntityRecord(this.mf, offset);
|
|
243
|
+
}
|
|
244
|
+
updateEntity(rec) {
|
|
245
|
+
writeEntityRecord(this.mf, rec);
|
|
246
|
+
}
|
|
247
|
+
deleteEntity(offset) {
|
|
248
|
+
const rec = readEntityRecord(this.mf, offset);
|
|
249
|
+
// Release string table refs
|
|
250
|
+
this.st.release(BigInt(rec.nameId));
|
|
251
|
+
this.st.release(BigInt(rec.typeId));
|
|
252
|
+
if (rec.obs0Id !== 0)
|
|
253
|
+
this.st.release(BigInt(rec.obs0Id));
|
|
254
|
+
if (rec.obs1Id !== 0)
|
|
255
|
+
this.st.release(BigInt(rec.obs1Id));
|
|
256
|
+
// Free adj block if present
|
|
257
|
+
if (rec.adjOffset !== 0n) {
|
|
258
|
+
this.mf.free(rec.adjOffset);
|
|
259
|
+
}
|
|
260
|
+
// Remove from node log
|
|
261
|
+
this.nodeLogRemove(offset);
|
|
262
|
+
// Free entity record
|
|
263
|
+
this.mf.free(offset);
|
|
264
|
+
}
|
|
265
|
+
// --- Observation management ---
|
|
266
|
+
addObservation(entityOffset, observation, mtime) {
|
|
267
|
+
const rec = readEntityRecord(this.mf, entityOffset);
|
|
268
|
+
if (rec.obsCount >= 2)
|
|
269
|
+
throw new Error('Entity already has max observations');
|
|
270
|
+
const obsId = Number(this.st.intern(observation));
|
|
271
|
+
if (rec.obsCount === 0) {
|
|
272
|
+
rec.obs0Id = obsId;
|
|
273
|
+
}
|
|
274
|
+
else {
|
|
275
|
+
rec.obs1Id = obsId;
|
|
276
|
+
}
|
|
277
|
+
rec.obsCount++;
|
|
278
|
+
rec.obsMtime = mtime;
|
|
279
|
+
rec.mtime = mtime;
|
|
280
|
+
writeEntityRecord(this.mf, rec);
|
|
281
|
+
}
|
|
282
|
+
removeObservation(entityOffset, observation, mtime) {
|
|
283
|
+
const rec = readEntityRecord(this.mf, entityOffset);
|
|
284
|
+
// Find the observation by matching the string
|
|
285
|
+
const obs0 = rec.obs0Id !== 0 ? this.st.get(BigInt(rec.obs0Id)) : null;
|
|
286
|
+
const obs1 = rec.obs1Id !== 0 ? this.st.get(BigInt(rec.obs1Id)) : null;
|
|
287
|
+
if (obs0 === observation) {
|
|
288
|
+
this.st.release(BigInt(rec.obs0Id));
|
|
289
|
+
// Shift obs1 into slot 0 if present
|
|
290
|
+
rec.obs0Id = rec.obs1Id;
|
|
291
|
+
rec.obs1Id = 0;
|
|
292
|
+
rec.obsCount--;
|
|
293
|
+
rec.obsMtime = mtime;
|
|
294
|
+
rec.mtime = mtime;
|
|
295
|
+
writeEntityRecord(this.mf, rec);
|
|
296
|
+
return true;
|
|
297
|
+
}
|
|
298
|
+
else if (obs1 === observation) {
|
|
299
|
+
this.st.release(BigInt(rec.obs1Id));
|
|
300
|
+
rec.obs1Id = 0;
|
|
301
|
+
rec.obsCount--;
|
|
302
|
+
rec.obsMtime = mtime;
|
|
303
|
+
rec.mtime = mtime;
|
|
304
|
+
writeEntityRecord(this.mf, rec);
|
|
305
|
+
return true;
|
|
306
|
+
}
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
// --- Adjacency management ---
|
|
310
|
+
addEdge(entityOffset, entry) {
|
|
311
|
+
const rec = readEntityRecord(this.mf, entityOffset);
|
|
312
|
+
if (rec.adjOffset === 0n) {
|
|
313
|
+
// No adj block yet — allocate one
|
|
314
|
+
const adjSize = ADJ_HEADER_SIZE + INITIAL_ADJ_CAPACITY * ADJ_ENTRY_SIZE;
|
|
315
|
+
const adjOffset = this.mf.alloc(BigInt(adjSize));
|
|
316
|
+
if (adjOffset === 0n)
|
|
317
|
+
throw new Error('GraphFile: adj alloc failed');
|
|
318
|
+
// Write header: count=1, capacity
|
|
319
|
+
const header = Buffer.alloc(ADJ_HEADER_SIZE);
|
|
320
|
+
header.writeUInt32LE(1, 0);
|
|
321
|
+
header.writeUInt32LE(INITIAL_ADJ_CAPACITY, 4);
|
|
322
|
+
this.mf.write(adjOffset, header);
|
|
323
|
+
// Write entry
|
|
324
|
+
const entryBuf = Buffer.alloc(ADJ_ENTRY_SIZE);
|
|
325
|
+
writeAdjEntry(entryBuf, 0, entry);
|
|
326
|
+
this.mf.write(adjOffset + BigInt(ADJ_HEADER_SIZE), entryBuf);
|
|
327
|
+
// Update entity record
|
|
328
|
+
rec.adjOffset = adjOffset;
|
|
329
|
+
writeEntityRecord(this.mf, rec);
|
|
330
|
+
}
|
|
331
|
+
else {
|
|
332
|
+
const adj = readAdjBlock(this.mf, rec.adjOffset);
|
|
333
|
+
if (adj.count < adj.capacity) {
|
|
334
|
+
// Append in place
|
|
335
|
+
const entryBuf = Buffer.alloc(ADJ_ENTRY_SIZE);
|
|
336
|
+
writeAdjEntry(entryBuf, 0, entry);
|
|
337
|
+
const entryPos = rec.adjOffset + BigInt(ADJ_HEADER_SIZE + adj.count * ADJ_ENTRY_SIZE);
|
|
338
|
+
this.mf.write(entryPos, entryBuf);
|
|
339
|
+
// Bump count
|
|
340
|
+
const countBuf = Buffer.alloc(4);
|
|
341
|
+
countBuf.writeUInt32LE(adj.count + 1, 0);
|
|
342
|
+
this.mf.write(rec.adjOffset, countBuf);
|
|
343
|
+
}
|
|
344
|
+
else {
|
|
345
|
+
// Need to grow — allocate new block with double capacity
|
|
346
|
+
const newCapacity = adj.capacity * 2;
|
|
347
|
+
const newSize = ADJ_HEADER_SIZE + newCapacity * ADJ_ENTRY_SIZE;
|
|
348
|
+
const newOffset = this.mf.alloc(BigInt(newSize));
|
|
349
|
+
if (newOffset === 0n)
|
|
350
|
+
throw new Error('GraphFile: adj grow failed');
|
|
351
|
+
// Write new header
|
|
352
|
+
const header = Buffer.alloc(ADJ_HEADER_SIZE);
|
|
353
|
+
header.writeUInt32LE(adj.count + 1, 0);
|
|
354
|
+
header.writeUInt32LE(newCapacity, 4);
|
|
355
|
+
this.mf.write(newOffset, header);
|
|
356
|
+
// Copy existing entries
|
|
357
|
+
if (adj.count > 0) {
|
|
358
|
+
const existing = this.mf.read(rec.adjOffset + BigInt(ADJ_HEADER_SIZE), BigInt(adj.count * ADJ_ENTRY_SIZE));
|
|
359
|
+
this.mf.write(newOffset + BigInt(ADJ_HEADER_SIZE), existing);
|
|
360
|
+
}
|
|
361
|
+
// Append new entry
|
|
362
|
+
const entryBuf = Buffer.alloc(ADJ_ENTRY_SIZE);
|
|
363
|
+
writeAdjEntry(entryBuf, 0, entry);
|
|
364
|
+
this.mf.write(newOffset + BigInt(ADJ_HEADER_SIZE + adj.count * ADJ_ENTRY_SIZE), entryBuf);
|
|
365
|
+
// Free old block, update entity
|
|
366
|
+
this.mf.free(rec.adjOffset);
|
|
367
|
+
rec.adjOffset = newOffset;
|
|
368
|
+
writeEntityRecord(this.mf, rec);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
removeEdge(entityOffset, targetOffset, relTypeId, direction) {
|
|
373
|
+
const rec = readEntityRecord(this.mf, entityOffset);
|
|
374
|
+
if (rec.adjOffset === 0n)
|
|
375
|
+
return false;
|
|
376
|
+
const adj = readAdjBlock(this.mf, rec.adjOffset);
|
|
377
|
+
const packed = packTargetDir(targetOffset, direction);
|
|
378
|
+
for (let i = 0; i < adj.count; i++) {
|
|
379
|
+
const e = adj.entries[i];
|
|
380
|
+
const ePacked = packTargetDir(e.targetOffset, e.direction);
|
|
381
|
+
if (ePacked === packed && e.relTypeId === relTypeId) {
|
|
382
|
+
// Found — swap with last entry and decrement count
|
|
383
|
+
if (i < adj.count - 1) {
|
|
384
|
+
// Read last entry and write over this slot
|
|
385
|
+
const lastPos = rec.adjOffset + BigInt(ADJ_HEADER_SIZE + (adj.count - 1) * ADJ_ENTRY_SIZE);
|
|
386
|
+
const lastBuf = this.mf.read(lastPos, BigInt(ADJ_ENTRY_SIZE));
|
|
387
|
+
const slotPos = rec.adjOffset + BigInt(ADJ_HEADER_SIZE + i * ADJ_ENTRY_SIZE);
|
|
388
|
+
this.mf.write(slotPos, lastBuf);
|
|
389
|
+
}
|
|
390
|
+
// Decrement count
|
|
391
|
+
const countBuf = Buffer.alloc(4);
|
|
392
|
+
countBuf.writeUInt32LE(adj.count - 1, 0);
|
|
393
|
+
this.mf.write(rec.adjOffset, countBuf);
|
|
394
|
+
return true;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
return false;
|
|
398
|
+
}
|
|
399
|
+
getEdges(entityOffset) {
|
|
400
|
+
const rec = readEntityRecord(this.mf, entityOffset);
|
|
401
|
+
if (rec.adjOffset === 0n)
|
|
402
|
+
return [];
|
|
403
|
+
return readAdjBlock(this.mf, rec.adjOffset).entries;
|
|
404
|
+
}
|
|
405
|
+
// --- Node log ---
|
|
406
|
+
nodeLogAppend(entityOffset) {
|
|
407
|
+
const logOffset = this.getNodeLogOffset();
|
|
408
|
+
const header = this.mf.read(logOffset, BigInt(NODE_LOG_HEADER_SIZE));
|
|
409
|
+
const count = header.readUInt32LE(0);
|
|
410
|
+
const capacity = header.readUInt32LE(4);
|
|
411
|
+
if (count < capacity) {
|
|
412
|
+
// Append in place
|
|
413
|
+
const pos = logOffset + BigInt(NODE_LOG_HEADER_SIZE + count * 8);
|
|
414
|
+
const buf = Buffer.alloc(8);
|
|
415
|
+
buf.writeBigUInt64LE(entityOffset, 0);
|
|
416
|
+
this.mf.write(pos, buf);
|
|
417
|
+
const countBuf = Buffer.alloc(4);
|
|
418
|
+
countBuf.writeUInt32LE(count + 1, 0);
|
|
419
|
+
this.mf.write(logOffset, countBuf);
|
|
420
|
+
}
|
|
421
|
+
else {
|
|
422
|
+
// Grow: allocate new log with double capacity
|
|
423
|
+
const newCapacity = capacity * 2;
|
|
424
|
+
const newSize = NODE_LOG_HEADER_SIZE + newCapacity * 8;
|
|
425
|
+
const newLogOffset = this.mf.alloc(BigInt(newSize));
|
|
426
|
+
if (newLogOffset === 0n)
|
|
427
|
+
throw new Error('GraphFile: node log grow failed');
|
|
428
|
+
// Write new header
|
|
429
|
+
const newHeader = Buffer.alloc(NODE_LOG_HEADER_SIZE);
|
|
430
|
+
newHeader.writeUInt32LE(count + 1, 0);
|
|
431
|
+
newHeader.writeUInt32LE(newCapacity, 4);
|
|
432
|
+
this.mf.write(newLogOffset, newHeader);
|
|
433
|
+
// Copy existing entries
|
|
434
|
+
if (count > 0) {
|
|
435
|
+
const existing = this.mf.read(logOffset + BigInt(NODE_LOG_HEADER_SIZE), BigInt(count * 8));
|
|
436
|
+
this.mf.write(newLogOffset + BigInt(NODE_LOG_HEADER_SIZE), existing);
|
|
437
|
+
}
|
|
438
|
+
// Append new entry
|
|
439
|
+
const entryBuf = Buffer.alloc(8);
|
|
440
|
+
entryBuf.writeBigUInt64LE(entityOffset, 0);
|
|
441
|
+
this.mf.write(newLogOffset + BigInt(NODE_LOG_HEADER_SIZE + count * 8), entryBuf);
|
|
442
|
+
// Free old, update header
|
|
443
|
+
this.mf.free(logOffset);
|
|
444
|
+
this.setNodeLogOffset(newLogOffset);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
nodeLogRemove(entityOffset) {
|
|
448
|
+
const logOffset = this.getNodeLogOffset();
|
|
449
|
+
const log = readNodeLog(this.mf, logOffset);
|
|
450
|
+
const idx = log.offsets.indexOf(entityOffset);
|
|
451
|
+
if (idx === -1)
|
|
452
|
+
return;
|
|
453
|
+
const lastIdx = log.count - 1;
|
|
454
|
+
if (idx < lastIdx) {
|
|
455
|
+
// Swap with last
|
|
456
|
+
const lastPos = logOffset + BigInt(NODE_LOG_HEADER_SIZE + lastIdx * 8);
|
|
457
|
+
const lastBuf = this.mf.read(lastPos, 8n);
|
|
458
|
+
const slotPos = logOffset + BigInt(NODE_LOG_HEADER_SIZE + idx * 8);
|
|
459
|
+
this.mf.write(slotPos, lastBuf);
|
|
460
|
+
}
|
|
461
|
+
// Decrement count
|
|
462
|
+
const countBuf = Buffer.alloc(4);
|
|
463
|
+
countBuf.writeUInt32LE(log.count - 1, 0);
|
|
464
|
+
this.mf.write(logOffset, countBuf);
|
|
465
|
+
}
|
|
466
|
+
// --- Scan all nodes ---
|
|
467
|
+
getAllEntityOffsets() {
|
|
468
|
+
const logOffset = this.getNodeLogOffset();
|
|
469
|
+
return readNodeLog(this.mf, logOffset).offsets;
|
|
470
|
+
}
|
|
471
|
+
getEntityCount() {
|
|
472
|
+
const logOffset = this.getNodeLogOffset();
|
|
473
|
+
const header = this.mf.read(logOffset, BigInt(NODE_LOG_HEADER_SIZE));
|
|
474
|
+
return header.readUInt32LE(0);
|
|
475
|
+
}
|
|
476
|
+
// --- PageRank visit counts ---
|
|
477
|
+
/** Read global structural visit total */
|
|
478
|
+
getStructuralTotal() {
|
|
479
|
+
const buf = this.mf.read(this.graphHeaderOffset + BigInt(GH_STRUCTURAL_TOTAL), 8n);
|
|
480
|
+
return buf.readBigUInt64LE(0);
|
|
481
|
+
}
|
|
482
|
+
/** Read global walker visit total */
|
|
483
|
+
getWalkerTotal() {
|
|
484
|
+
const buf = this.mf.read(this.graphHeaderOffset + BigInt(GH_WALKER_TOTAL), 8n);
|
|
485
|
+
return buf.readBigUInt64LE(0);
|
|
486
|
+
}
|
|
487
|
+
/** Increment structural visit count for one entity and bump the global counter. */
|
|
488
|
+
incrementStructuralVisit(entityOffset) {
|
|
489
|
+
// Read current entity visit count
|
|
490
|
+
const vbuf = this.mf.read(entityOffset + BigInt(E_STRUCTURAL_VISITS), 8n);
|
|
491
|
+
const current = vbuf.readBigUInt64LE(0);
|
|
492
|
+
const wbuf = Buffer.alloc(8);
|
|
493
|
+
wbuf.writeBigUInt64LE(current + 1n, 0);
|
|
494
|
+
this.mf.write(entityOffset + BigInt(E_STRUCTURAL_VISITS), wbuf);
|
|
495
|
+
// Bump global counter
|
|
496
|
+
const gbuf = this.mf.read(this.graphHeaderOffset + BigInt(GH_STRUCTURAL_TOTAL), 8n);
|
|
497
|
+
const total = gbuf.readBigUInt64LE(0);
|
|
498
|
+
const gwbuf = Buffer.alloc(8);
|
|
499
|
+
gwbuf.writeBigUInt64LE(total + 1n, 0);
|
|
500
|
+
this.mf.write(this.graphHeaderOffset + BigInt(GH_STRUCTURAL_TOTAL), gwbuf);
|
|
501
|
+
}
|
|
502
|
+
/** Increment walker visit count for one entity and bump the global counter. */
|
|
503
|
+
incrementWalkerVisit(entityOffset) {
|
|
504
|
+
const vbuf = this.mf.read(entityOffset + BigInt(E_WALKER_VISITS), 8n);
|
|
505
|
+
const current = vbuf.readBigUInt64LE(0);
|
|
506
|
+
const wbuf = Buffer.alloc(8);
|
|
507
|
+
wbuf.writeBigUInt64LE(current + 1n, 0);
|
|
508
|
+
this.mf.write(entityOffset + BigInt(E_WALKER_VISITS), wbuf);
|
|
509
|
+
const gbuf = this.mf.read(this.graphHeaderOffset + BigInt(GH_WALKER_TOTAL), 8n);
|
|
510
|
+
const total = gbuf.readBigUInt64LE(0);
|
|
511
|
+
const gwbuf = Buffer.alloc(8);
|
|
512
|
+
gwbuf.writeBigUInt64LE(total + 1n, 0);
|
|
513
|
+
this.mf.write(this.graphHeaderOffset + BigInt(GH_WALKER_TOTAL), gwbuf);
|
|
514
|
+
}
|
|
515
|
+
/**
|
|
516
|
+
* Get the structural PageRank score for an entity.
|
|
517
|
+
* Returns structuralVisits / structuralTotal, or 0 if no visits yet.
|
|
518
|
+
*/
|
|
519
|
+
getStructuralRank(entityOffset) {
|
|
520
|
+
const total = this.getStructuralTotal();
|
|
521
|
+
if (total === 0n)
|
|
522
|
+
return 0;
|
|
523
|
+
const rec = this.readEntity(entityOffset);
|
|
524
|
+
return Number(rec.structuralVisits) / Number(total);
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Get the walker PageRank score for an entity.
|
|
528
|
+
* Returns walkerVisits / walkerTotal, or 0 if no visits yet.
|
|
529
|
+
*/
|
|
530
|
+
getWalkerRank(entityOffset) {
|
|
531
|
+
const total = this.getWalkerTotal();
|
|
532
|
+
if (total === 0n)
|
|
533
|
+
return 0;
|
|
534
|
+
const rec = this.readEntity(entityOffset);
|
|
535
|
+
return Number(rec.walkerVisits) / Number(total);
|
|
536
|
+
}
|
|
537
|
+
// --- Lifecycle & Concurrency ---
|
|
538
|
+
/** Acquire a shared (read) lock on the graph file. */
|
|
539
|
+
lockShared() {
|
|
540
|
+
this.mf.lockShared();
|
|
541
|
+
}
|
|
542
|
+
/** Acquire an exclusive (write) lock on the graph file. */
|
|
543
|
+
lockExclusive() {
|
|
544
|
+
this.mf.lockExclusive();
|
|
545
|
+
}
|
|
546
|
+
/** Release the lock on the graph file. */
|
|
547
|
+
unlock() {
|
|
548
|
+
this.mf.unlock();
|
|
549
|
+
}
|
|
550
|
+
/** Refresh the mmap if the file was grown by another process. */
|
|
551
|
+
refresh() {
|
|
552
|
+
this.mf.refresh();
|
|
553
|
+
}
|
|
554
|
+
sync() {
|
|
555
|
+
this.mf.sync();
|
|
556
|
+
}
|
|
557
|
+
close() {
|
|
558
|
+
this.mf.close();
|
|
559
|
+
}
|
|
560
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MemoryFile - TypeScript wrapper around the native mmap arena allocator.
|
|
3
|
+
*
|
|
4
|
+
* All offsets are BigInt (u64 on the C side).
|
|
5
|
+
* Buffers passed to/from the native layer are Node Buffers.
|
|
6
|
+
*/
|
|
7
|
+
import { createRequire } from 'module';
|
|
8
|
+
import { dirname, join } from 'path';
|
|
9
|
+
import { fileURLToPath } from 'url';
|
|
10
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
11
|
+
const require = createRequire(import.meta.url);
|
|
12
|
+
// The .node binary is in build/Release/ relative to project root
|
|
13
|
+
const native = require(join(__dirname, '..', 'build', 'Release', 'memoryfile.node'));
|
|
14
|
+
export class MemoryFile {
|
|
15
|
+
handle;
|
|
16
|
+
closed = false;
|
|
17
|
+
constructor(path, initialSize = 4096) {
|
|
18
|
+
this.handle = native.open(path, initialSize);
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Allocate a block of the given size.
|
|
22
|
+
* Returns the offset to the usable region (after the alloc header).
|
|
23
|
+
* Returns 0n on failure.
|
|
24
|
+
*/
|
|
25
|
+
alloc(size) {
|
|
26
|
+
this.assertOpen();
|
|
27
|
+
return native.alloc(this.handle, size);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Free a previously allocated block by its offset.
|
|
31
|
+
* The block goes onto the free list for reuse.
|
|
32
|
+
*/
|
|
33
|
+
free(offset) {
|
|
34
|
+
this.assertOpen();
|
|
35
|
+
native.free(this.handle, offset);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Merge adjacent free blocks to reduce fragmentation.
|
|
39
|
+
*/
|
|
40
|
+
coalesce() {
|
|
41
|
+
this.assertOpen();
|
|
42
|
+
native.coalesce(this.handle);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Read `length` bytes starting at `offset`.
|
|
46
|
+
* Returns a Buffer with the data.
|
|
47
|
+
*/
|
|
48
|
+
read(offset, length) {
|
|
49
|
+
this.assertOpen();
|
|
50
|
+
return native.read(this.handle, offset, length);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Write a Buffer at the given offset.
|
|
54
|
+
*/
|
|
55
|
+
write(offset, data) {
|
|
56
|
+
this.assertOpen();
|
|
57
|
+
native.write(this.handle, offset, data);
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Acquire a shared (read) lock on the file.
|
|
61
|
+
* Blocks until the lock is acquired.
|
|
62
|
+
*/
|
|
63
|
+
lockShared() {
|
|
64
|
+
this.assertOpen();
|
|
65
|
+
native.lockShared(this.handle);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Acquire an exclusive (write) lock on the file.
|
|
69
|
+
* Blocks until the lock is acquired.
|
|
70
|
+
*/
|
|
71
|
+
lockExclusive() {
|
|
72
|
+
this.assertOpen();
|
|
73
|
+
native.lockExclusive(this.handle);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Release the lock on the file.
|
|
77
|
+
*/
|
|
78
|
+
unlock() {
|
|
79
|
+
this.assertOpen();
|
|
80
|
+
native.unlock(this.handle);
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Refresh the mmap if the file was grown by another process.
|
|
84
|
+
* Call this after acquiring a lock, before reading, to ensure
|
|
85
|
+
* the mapping covers the full file.
|
|
86
|
+
*/
|
|
87
|
+
refresh() {
|
|
88
|
+
this.assertOpen();
|
|
89
|
+
native.refresh(this.handle);
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Flush all changes to disk.
|
|
93
|
+
*/
|
|
94
|
+
sync() {
|
|
95
|
+
this.assertOpen();
|
|
96
|
+
native.sync(this.handle);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Get arena statistics.
|
|
100
|
+
*/
|
|
101
|
+
stats() {
|
|
102
|
+
this.assertOpen();
|
|
103
|
+
return native.stats(this.handle);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Close the memory file. Syncs and unmaps.
|
|
107
|
+
* The instance is unusable after this.
|
|
108
|
+
*/
|
|
109
|
+
close() {
|
|
110
|
+
if (this.closed)
|
|
111
|
+
return;
|
|
112
|
+
this.closed = true;
|
|
113
|
+
native.close(this.handle);
|
|
114
|
+
this.handle = null;
|
|
115
|
+
}
|
|
116
|
+
assertOpen() {
|
|
117
|
+
if (this.closed) {
|
|
118
|
+
throw new Error('MemoryFile is closed');
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|