@zokizuan/satori-core 0.2.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/core/context.d.ts +8 -8
- package/dist/core/context.js +140 -32
- package/dist/sync/merkle.d.ts +6 -25
- package/dist/sync/merkle.js +22 -74
- package/dist/sync/synchronizer.d.ts +36 -14
- package/dist/sync/synchronizer.js +447 -159
- package/dist/vectordb/index.d.ts +1 -1
- package/dist/vectordb/index.js +4 -1
- package/dist/vectordb/milvus-restful-vectordb.d.ts +0 -5
- package/dist/vectordb/milvus-restful-vectordb.js +44 -12
- package/dist/vectordb/milvus-vectordb.js +8 -4
- package/dist/vectordb/types.d.ts +20 -0
- package/dist/vectordb/types.js +4 -1
- package/package.json +1 -1
|
@@ -37,98 +37,113 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
39
|
exports.FileSynchronizer = void 0;
|
|
40
|
-
const
|
|
40
|
+
const fsp = __importStar(require("fs/promises"));
|
|
41
|
+
const fsSync = __importStar(require("fs"));
|
|
42
|
+
const fs_1 = require("fs");
|
|
41
43
|
const path = __importStar(require("path"));
|
|
42
44
|
const crypto = __importStar(require("crypto"));
|
|
43
|
-
const merkle_1 = require("./merkle");
|
|
44
45
|
const os = __importStar(require("os"));
|
|
45
46
|
const ignore_1 = __importDefault(require("ignore"));
|
|
47
|
+
const merkle_1 = require("./merkle");
|
|
48
|
+
const SNAPSHOT_VERSION = 2;
|
|
49
|
+
const DEFAULT_HASH_CONCURRENCY = 16;
|
|
46
50
|
class FileSynchronizer {
|
|
47
51
|
constructor(rootDir, ignorePatterns = []) {
|
|
48
|
-
this.rootDir = rootDir;
|
|
49
|
-
this.snapshotPath = this.
|
|
52
|
+
this.rootDir = FileSynchronizer.canonicalizeSnapshotIdentityPath(rootDir);
|
|
53
|
+
this.snapshotPath = FileSynchronizer.getSnapshotPathForCodebase(this.rootDir);
|
|
50
54
|
this.fileHashes = new Map();
|
|
51
|
-
this.
|
|
55
|
+
this.fileStats = new Map();
|
|
56
|
+
this.merkleRoot = '';
|
|
52
57
|
this.ignorePatterns = ignorePatterns;
|
|
53
58
|
this.ignoreMatcher = (0, ignore_1.default)();
|
|
54
59
|
this.ignoreMatcher.add(this.ignorePatterns);
|
|
60
|
+
this.partialScan = false;
|
|
61
|
+
this.unscannedDirPrefixes = [];
|
|
62
|
+
this.fullHashCounter = 0;
|
|
63
|
+
}
|
|
64
|
+
static canonicalizeSnapshotIdentityPath(codebasePath) {
|
|
65
|
+
const resolved = path.resolve(codebasePath);
|
|
66
|
+
try {
|
|
67
|
+
const realPath = typeof fsSync.realpathSync.native === 'function'
|
|
68
|
+
? fsSync.realpathSync.native(resolved)
|
|
69
|
+
: fsSync.realpathSync(resolved);
|
|
70
|
+
return FileSynchronizer.trimTrailingSeparators(path.normalize(realPath));
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return FileSynchronizer.trimTrailingSeparators(path.normalize(resolved));
|
|
74
|
+
}
|
|
55
75
|
}
|
|
56
|
-
|
|
76
|
+
static snapshotPathFromCanonicalPath(canonicalPath) {
|
|
57
77
|
const homeDir = os.homedir();
|
|
58
78
|
const merkleDir = path.join(homeDir, '.satori', 'merkle');
|
|
59
|
-
const
|
|
60
|
-
const hash = crypto.createHash('md5').update(normalizedPath).digest('hex');
|
|
79
|
+
const hash = crypto.createHash('md5').update(canonicalPath).digest('hex');
|
|
61
80
|
return path.join(merkleDir, `${hash}.json`);
|
|
62
81
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
if (stat.isDirectory()) {
|
|
67
|
-
throw new Error(`Attempted to hash a directory: ${filePath}`);
|
|
68
|
-
}
|
|
69
|
-
const content = await fs.readFile(filePath, 'utf-8');
|
|
70
|
-
return crypto.createHash('sha256').update(content).digest('hex');
|
|
82
|
+
static getSnapshotPathForCodebase(codebasePath) {
|
|
83
|
+
const canonicalPath = FileSynchronizer.canonicalizeSnapshotIdentityPath(codebasePath);
|
|
84
|
+
return FileSynchronizer.snapshotPathFromCanonicalPath(canonicalPath);
|
|
71
85
|
}
|
|
72
|
-
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
86
|
+
static trimTrailingSeparators(inputPath) {
|
|
87
|
+
const parsedRoot = path.parse(inputPath).root;
|
|
88
|
+
if (inputPath === parsedRoot) {
|
|
89
|
+
return inputPath;
|
|
77
90
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
91
|
+
return inputPath.replace(/[\\/]+$/, '');
|
|
92
|
+
}
|
|
93
|
+
normalizeRelPath(candidatePath) {
|
|
94
|
+
if (typeof candidatePath !== 'string') {
|
|
95
|
+
return '';
|
|
81
96
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
97
|
+
const trimmed = candidatePath.trim();
|
|
98
|
+
if (!trimmed) {
|
|
99
|
+
return '';
|
|
100
|
+
}
|
|
101
|
+
let normalized = trimmed.replace(/\\/g, '/');
|
|
102
|
+
normalized = normalized.replace(/\/+/g, '/');
|
|
103
|
+
normalized = normalized.replace(/^(\.\/)+/, '');
|
|
104
|
+
normalized = normalized.replace(/^\/+/, '');
|
|
105
|
+
normalized = normalized.replace(/\/+$/, '');
|
|
106
|
+
if (!normalized || normalized === '.') {
|
|
107
|
+
return '';
|
|
108
|
+
}
|
|
109
|
+
const parts = normalized.split('/');
|
|
110
|
+
const cleanParts = [];
|
|
111
|
+
for (const part of parts) {
|
|
112
|
+
if (!part || part === '.') {
|
|
96
113
|
continue;
|
|
97
114
|
}
|
|
98
|
-
if (
|
|
99
|
-
|
|
100
|
-
if (!this.shouldIgnore(relativePath, true)) {
|
|
101
|
-
const subHashes = await this.generateFileHashes(fullPath);
|
|
102
|
-
const entries = Array.from(subHashes.entries());
|
|
103
|
-
for (let i = 0; i < entries.length; i++) {
|
|
104
|
-
const [p, h] = entries[i];
|
|
105
|
-
fileHashes.set(p, h);
|
|
106
|
-
}
|
|
107
|
-
}
|
|
115
|
+
if (part === '..') {
|
|
116
|
+
return '';
|
|
108
117
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
118
|
+
cleanParts.push(part);
|
|
119
|
+
}
|
|
120
|
+
if (cleanParts.length === 0) {
|
|
121
|
+
return '';
|
|
122
|
+
}
|
|
123
|
+
return cleanParts.join('/');
|
|
124
|
+
}
|
|
125
|
+
isPathWithinPrefix(candidatePath, prefix) {
|
|
126
|
+
return candidatePath === prefix || candidatePath.startsWith(`${prefix}/`);
|
|
127
|
+
}
|
|
128
|
+
normalizeAndCompressPrefixes(prefixes) {
|
|
129
|
+
const normalized = Array.from(prefixes)
|
|
130
|
+
.map((prefix) => this.normalizeRelPath(prefix))
|
|
131
|
+
.filter((prefix) => prefix.length > 0)
|
|
132
|
+
.sort();
|
|
133
|
+
const compressed = [];
|
|
134
|
+
for (const prefix of normalized) {
|
|
135
|
+
const covered = compressed.some((existingPrefix) => this.isPathWithinPrefix(prefix, existingPrefix));
|
|
136
|
+
if (!covered) {
|
|
137
|
+
compressed.push(prefix);
|
|
121
138
|
}
|
|
122
|
-
// Skip other types (symlinks, etc.)
|
|
123
139
|
}
|
|
124
|
-
return
|
|
140
|
+
return compressed;
|
|
125
141
|
}
|
|
126
142
|
shouldIgnore(relativePath, isDirectory = false) {
|
|
127
|
-
const normalizedPath =
|
|
128
|
-
if (!normalizedPath
|
|
129
|
-
return false;
|
|
143
|
+
const normalizedPath = this.normalizeRelPath(relativePath);
|
|
144
|
+
if (!normalizedPath) {
|
|
145
|
+
return false;
|
|
130
146
|
}
|
|
131
|
-
// Always ignore hidden files and directories (starting with .)
|
|
132
147
|
const pathParts = normalizedPath.split('/');
|
|
133
148
|
if (pathParts.some(part => part.startsWith('.'))) {
|
|
134
149
|
return true;
|
|
@@ -142,135 +157,408 @@ class FileSynchronizer {
|
|
|
142
157
|
}
|
|
143
158
|
return this.ignoreMatcher.ignores(normalizedPath);
|
|
144
159
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
160
|
+
parsePositiveInt(rawValue, fallback, min, max) {
|
|
161
|
+
if (!rawValue || rawValue.trim().length === 0) {
|
|
162
|
+
return fallback;
|
|
163
|
+
}
|
|
164
|
+
const parsed = Number.parseInt(rawValue, 10);
|
|
165
|
+
if (!Number.isFinite(parsed) || Number.isNaN(parsed)) {
|
|
166
|
+
return fallback;
|
|
167
|
+
}
|
|
168
|
+
if (parsed < min) {
|
|
169
|
+
return min;
|
|
170
|
+
}
|
|
171
|
+
if (parsed > max) {
|
|
172
|
+
return max;
|
|
173
|
+
}
|
|
174
|
+
return parsed;
|
|
175
|
+
}
|
|
176
|
+
getHashConcurrency() {
|
|
177
|
+
return this.parsePositiveInt(process.env.SATORI_SYNC_HASH_CONCURRENCY, DEFAULT_HASH_CONCURRENCY, 1, 64);
|
|
178
|
+
}
|
|
179
|
+
getFullHashInterval() {
|
|
180
|
+
return this.parsePositiveInt(process.env.SATORI_SYNC_FULL_HASH_EVERY_N, 0, 0, 1000000);
|
|
181
|
+
}
|
|
182
|
+
async hashFileBytes(filePath) {
|
|
183
|
+
const stat = await fsp.stat(filePath);
|
|
184
|
+
if (!stat.isFile()) {
|
|
185
|
+
throw new Error(`Attempted to hash non-file path: ${filePath}`);
|
|
186
|
+
}
|
|
187
|
+
return new Promise((resolve, reject) => {
|
|
188
|
+
const hasher = crypto.createHash('sha256');
|
|
189
|
+
const stream = (0, fs_1.createReadStream)(filePath);
|
|
190
|
+
stream.on('data', (chunk) => {
|
|
191
|
+
hasher.update(chunk);
|
|
192
|
+
});
|
|
193
|
+
stream.on('error', reject);
|
|
194
|
+
stream.on('end', () => {
|
|
195
|
+
resolve(hasher.digest('hex'));
|
|
196
|
+
});
|
|
153
197
|
});
|
|
154
|
-
const rootNodeData = "root:" + valuesString;
|
|
155
|
-
const rootNodeId = dag.addNode(rootNodeData);
|
|
156
|
-
// Add each file as a child of the root
|
|
157
|
-
for (const path of sortedPaths) {
|
|
158
|
-
const fileData = path + ":" + fileHashes.get(path);
|
|
159
|
-
dag.addNode(fileData, rootNodeId);
|
|
160
|
-
}
|
|
161
|
-
return dag;
|
|
162
198
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
await this.loadSnapshot();
|
|
166
|
-
this.merkleDAG = this.buildMerkleDAG(this.fileHashes);
|
|
167
|
-
console.log(`[Synchronizer] File synchronizer initialized. Loaded ${this.fileHashes.size} file hashes.`);
|
|
199
|
+
isSignatureEqual(a, b) {
|
|
200
|
+
return !!a && a.size === b.size && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs;
|
|
168
201
|
}
|
|
169
|
-
async
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
const
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
console.
|
|
183
|
-
return
|
|
202
|
+
async scanDirectory(directoryPath, previousHashes, previousStats, forceFullHash, result) {
|
|
203
|
+
let entries;
|
|
204
|
+
try {
|
|
205
|
+
entries = await fsp.readdir(directoryPath, { withFileTypes: true });
|
|
206
|
+
}
|
|
207
|
+
catch (error) {
|
|
208
|
+
if (directoryPath === this.rootDir) {
|
|
209
|
+
throw new Error(`[Synchronizer] Cannot read root directory ${directoryPath}: ${error.message}`);
|
|
210
|
+
}
|
|
211
|
+
const relativeDir = this.normalizeRelPath(path.relative(this.rootDir, directoryPath));
|
|
212
|
+
if (relativeDir) {
|
|
213
|
+
result.unscannedDirPrefixes.add(relativeDir);
|
|
214
|
+
}
|
|
215
|
+
console.warn(`[Synchronizer] Cannot read directory ${directoryPath}: ${error.message}`);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
219
|
+
for (const entry of entries) {
|
|
220
|
+
const absolutePath = path.join(directoryPath, entry.name);
|
|
221
|
+
const relativePath = this.normalizeRelPath(path.relative(this.rootDir, absolutePath));
|
|
222
|
+
if (!relativePath) {
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (this.shouldIgnore(relativePath, entry.isDirectory())) {
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
let stat;
|
|
229
|
+
try {
|
|
230
|
+
stat = await fsp.stat(absolutePath);
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
if (entry.isDirectory()) {
|
|
234
|
+
result.unscannedDirPrefixes.add(relativePath);
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
result.unreadableFiles.add(relativePath);
|
|
238
|
+
}
|
|
239
|
+
console.warn(`[Synchronizer] Cannot stat ${absolutePath}: ${error.message}`);
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
if (stat.isDirectory()) {
|
|
243
|
+
if (!this.shouldIgnore(relativePath, true)) {
|
|
244
|
+
await this.scanDirectory(absolutePath, previousHashes, previousStats, forceFullHash, result);
|
|
245
|
+
}
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
if (!stat.isFile()) {
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
if (this.shouldIgnore(relativePath, false)) {
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
const signature = {
|
|
255
|
+
size: stat.size,
|
|
256
|
+
mtimeMs: Number(stat.mtimeMs),
|
|
257
|
+
ctimeMs: Number(stat.ctimeMs)
|
|
258
|
+
};
|
|
259
|
+
result.scannedStats.set(relativePath, signature);
|
|
260
|
+
const previousSignature = previousStats.get(relativePath);
|
|
261
|
+
const previousHash = previousHashes.get(relativePath);
|
|
262
|
+
const canReuseHash = !forceFullHash
|
|
263
|
+
&& this.isSignatureEqual(previousSignature, signature)
|
|
264
|
+
&& typeof previousHash === 'string';
|
|
265
|
+
if (canReuseHash) {
|
|
266
|
+
result.scannedHashes.set(relativePath, previousHash);
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
result.hashCandidates.push({ relativePath, absolutePath, signature });
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
async hashCandidatesWithConcurrency(result) {
|
|
273
|
+
if (result.hashCandidates.length === 0) {
|
|
274
|
+
return 0;
|
|
275
|
+
}
|
|
276
|
+
const concurrency = this.getHashConcurrency();
|
|
277
|
+
let cursor = 0;
|
|
278
|
+
let hashedCount = 0;
|
|
279
|
+
const workers = Array.from({ length: Math.min(concurrency, result.hashCandidates.length) }).map(async () => {
|
|
280
|
+
while (true) {
|
|
281
|
+
const currentIndex = cursor;
|
|
282
|
+
cursor += 1;
|
|
283
|
+
if (currentIndex >= result.hashCandidates.length) {
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
const candidate = result.hashCandidates[currentIndex];
|
|
287
|
+
try {
|
|
288
|
+
const hash = await this.hashFileBytes(candidate.absolutePath);
|
|
289
|
+
result.scannedHashes.set(candidate.relativePath, hash);
|
|
290
|
+
hashedCount += 1;
|
|
291
|
+
}
|
|
292
|
+
catch (error) {
|
|
293
|
+
result.unreadableFiles.add(candidate.relativePath);
|
|
294
|
+
result.scannedStats.delete(candidate.relativePath);
|
|
295
|
+
console.warn(`[Synchronizer] Cannot hash file ${candidate.absolutePath}: ${error.message}`);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
await Promise.all(workers);
|
|
300
|
+
return hashedCount;
|
|
301
|
+
}
|
|
302
|
+
buildEffectiveState(previousHashes, previousStats, result) {
|
|
303
|
+
const unscannedDirPrefixes = this.normalizeAndCompressPrefixes(result.unscannedDirPrefixes);
|
|
304
|
+
const partialScan = unscannedDirPrefixes.length > 0 || result.unreadableFiles.size > 0;
|
|
305
|
+
const effectiveHashes = new Map();
|
|
306
|
+
const effectiveStats = new Map();
|
|
307
|
+
for (const [relativePath, hash] of result.scannedHashes.entries()) {
|
|
308
|
+
effectiveHashes.set(relativePath, hash);
|
|
309
|
+
}
|
|
310
|
+
for (const [relativePath, signature] of result.scannedStats.entries()) {
|
|
311
|
+
effectiveStats.set(relativePath, signature);
|
|
312
|
+
}
|
|
313
|
+
const shouldPreservePrevious = (relativePath) => {
|
|
314
|
+
if (result.unreadableFiles.has(relativePath)) {
|
|
315
|
+
return true;
|
|
316
|
+
}
|
|
317
|
+
return unscannedDirPrefixes.some((prefix) => this.isPathWithinPrefix(relativePath, prefix));
|
|
318
|
+
};
|
|
319
|
+
for (const [relativePath, previousHash] of previousHashes.entries()) {
|
|
320
|
+
if (effectiveHashes.has(relativePath)) {
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
323
|
+
if (!shouldPreservePrevious(relativePath)) {
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
if (this.shouldIgnore(relativePath, false)) {
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
effectiveHashes.set(relativePath, previousHash);
|
|
330
|
+
const previousSignature = previousStats.get(relativePath);
|
|
331
|
+
if (previousSignature) {
|
|
332
|
+
effectiveStats.set(relativePath, previousSignature);
|
|
333
|
+
}
|
|
184
334
|
}
|
|
185
|
-
|
|
186
|
-
|
|
335
|
+
for (const relativePath of Array.from(effectiveHashes.keys())) {
|
|
336
|
+
if (this.shouldIgnore(relativePath, false)) {
|
|
337
|
+
effectiveHashes.delete(relativePath);
|
|
338
|
+
effectiveStats.delete(relativePath);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
return {
|
|
342
|
+
fileHashes: effectiveHashes,
|
|
343
|
+
fileStats: effectiveStats,
|
|
344
|
+
unscannedDirPrefixes,
|
|
345
|
+
partialScan
|
|
346
|
+
};
|
|
187
347
|
}
|
|
188
348
|
compareStates(oldHashes, newHashes) {
|
|
189
349
|
const added = [];
|
|
190
350
|
const removed = [];
|
|
191
351
|
const modified = [];
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
352
|
+
for (const [filePath, hash] of newHashes.entries()) {
|
|
353
|
+
const previousHash = oldHashes.get(filePath);
|
|
354
|
+
if (typeof previousHash === 'undefined') {
|
|
355
|
+
added.push(filePath);
|
|
356
|
+
continue;
|
|
197
357
|
}
|
|
198
|
-
|
|
199
|
-
modified.push(
|
|
358
|
+
if (previousHash !== hash) {
|
|
359
|
+
modified.push(filePath);
|
|
200
360
|
}
|
|
201
361
|
}
|
|
202
|
-
const
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
if (!newHashes.has(file)) {
|
|
206
|
-
removed.push(file);
|
|
362
|
+
for (const filePath of oldHashes.keys()) {
|
|
363
|
+
if (!newHashes.has(filePath)) {
|
|
364
|
+
removed.push(filePath);
|
|
207
365
|
}
|
|
208
366
|
}
|
|
367
|
+
added.sort();
|
|
368
|
+
removed.sort();
|
|
369
|
+
modified.sort();
|
|
209
370
|
return { added, removed, modified };
|
|
210
371
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
372
|
+
arraysEqual(a, b) {
|
|
373
|
+
if (a.length !== b.length) {
|
|
374
|
+
return false;
|
|
375
|
+
}
|
|
376
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
377
|
+
if (a[i] !== b[i]) {
|
|
378
|
+
return false;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
return true;
|
|
220
382
|
}
|
|
221
383
|
async saveSnapshot() {
|
|
222
384
|
const merkleDir = path.dirname(this.snapshotPath);
|
|
223
|
-
await
|
|
224
|
-
|
|
225
|
-
const
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
385
|
+
await fsp.mkdir(merkleDir, { recursive: true });
|
|
386
|
+
const fileHashes = Array.from(this.fileHashes.entries()).sort(([a], [b]) => a.localeCompare(b));
|
|
387
|
+
const fileStats = Array.from(this.fileStats.entries()).sort(([a], [b]) => a.localeCompare(b));
|
|
388
|
+
const payload = {
|
|
389
|
+
snapshotVersion: SNAPSHOT_VERSION,
|
|
390
|
+
fileHashes,
|
|
391
|
+
fileStats,
|
|
392
|
+
merkleRoot: this.merkleRoot,
|
|
393
|
+
partialScan: this.partialScan,
|
|
394
|
+
unscannedDirPrefixes: [...this.unscannedDirPrefixes],
|
|
395
|
+
fullHashCounter: this.fullHashCounter
|
|
396
|
+
};
|
|
397
|
+
await fsp.writeFile(this.snapshotPath, JSON.stringify(payload), 'utf-8');
|
|
235
398
|
console.log(`Saved snapshot to ${this.snapshotPath}`);
|
|
236
399
|
}
|
|
237
400
|
async loadSnapshot() {
|
|
238
401
|
try {
|
|
239
|
-
const data = await
|
|
402
|
+
const data = await fsp.readFile(this.snapshotPath, 'utf-8');
|
|
240
403
|
const obj = JSON.parse(data);
|
|
241
|
-
|
|
404
|
+
const rawFileHashes = Array.isArray(obj.fileHashes) ? obj.fileHashes : [];
|
|
242
405
|
this.fileHashes = new Map();
|
|
243
|
-
for (const
|
|
244
|
-
|
|
406
|
+
for (const entry of rawFileHashes) {
|
|
407
|
+
if (!Array.isArray(entry) || entry.length !== 2) {
|
|
408
|
+
continue;
|
|
409
|
+
}
|
|
410
|
+
const normalizedPath = this.normalizeRelPath(String(entry[0] ?? ''));
|
|
411
|
+
const hash = String(entry[1] ?? '');
|
|
412
|
+
if (!normalizedPath || !hash) {
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
this.fileHashes.set(normalizedPath, hash);
|
|
245
416
|
}
|
|
246
|
-
|
|
247
|
-
|
|
417
|
+
const rawFileStats = Array.isArray(obj.fileStats) ? obj.fileStats : [];
|
|
418
|
+
this.fileStats = new Map();
|
|
419
|
+
for (const entry of rawFileStats) {
|
|
420
|
+
if (!Array.isArray(entry) || entry.length !== 2) {
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
const normalizedPath = this.normalizeRelPath(String(entry[0] ?? ''));
|
|
424
|
+
const rawSignature = entry[1];
|
|
425
|
+
if (!normalizedPath || !rawSignature) {
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
const size = Number(rawSignature.size);
|
|
429
|
+
const mtimeMs = Number(rawSignature.mtimeMs);
|
|
430
|
+
const ctimeMs = Number(rawSignature.ctimeMs ?? rawSignature.mtimeMs);
|
|
431
|
+
if (!Number.isFinite(size) || !Number.isFinite(mtimeMs) || !Number.isFinite(ctimeMs)) {
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
this.fileStats.set(normalizedPath, {
|
|
435
|
+
size,
|
|
436
|
+
mtimeMs,
|
|
437
|
+
ctimeMs
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
this.merkleRoot = typeof obj.merkleRoot === 'string' ? obj.merkleRoot : '';
|
|
441
|
+
this.partialScan = Boolean(obj.partialScan);
|
|
442
|
+
this.unscannedDirPrefixes = this.normalizeAndCompressPrefixes(new Set(Array.isArray(obj.unscannedDirPrefixes) ? obj.unscannedDirPrefixes : []));
|
|
443
|
+
this.fullHashCounter = Number.isFinite(Number(obj.fullHashCounter)) ? Number(obj.fullHashCounter) : 0;
|
|
444
|
+
const isV2 = obj.snapshotVersion === SNAPSHOT_VERSION;
|
|
445
|
+
const hasCompatibleStats = this.fileStats.size > 0 || this.fileHashes.size === 0;
|
|
446
|
+
const migrated = !isV2 || !hasCompatibleStats;
|
|
447
|
+
if (migrated) {
|
|
448
|
+
console.log(`Loaded legacy snapshot from ${this.snapshotPath}. Migration to v${SNAPSHOT_VERSION} required.`);
|
|
449
|
+
}
|
|
450
|
+
else {
|
|
451
|
+
console.log(`Loaded snapshot from ${this.snapshotPath}`);
|
|
248
452
|
}
|
|
249
|
-
|
|
453
|
+
return { migrated };
|
|
250
454
|
}
|
|
251
455
|
catch (error) {
|
|
252
456
|
if (error.code === 'ENOENT') {
|
|
253
|
-
console.log(`Snapshot file not found at ${this.snapshotPath}.
|
|
254
|
-
this.fileHashes =
|
|
255
|
-
this.
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
457
|
+
console.log(`Snapshot file not found at ${this.snapshotPath}. Creating baseline snapshot.`);
|
|
458
|
+
this.fileHashes = new Map();
|
|
459
|
+
this.fileStats = new Map();
|
|
460
|
+
this.merkleRoot = '';
|
|
461
|
+
this.partialScan = false;
|
|
462
|
+
this.unscannedDirPrefixes = [];
|
|
463
|
+
this.fullHashCounter = 0;
|
|
464
|
+
return { migrated: true };
|
|
260
465
|
}
|
|
466
|
+
throw error;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
async scanCurrentState(previousHashes, previousStats, forceFullHash) {
|
|
470
|
+
const scanResult = {
|
|
471
|
+
scannedHashes: new Map(),
|
|
472
|
+
scannedStats: new Map(),
|
|
473
|
+
hashCandidates: [],
|
|
474
|
+
unreadableFiles: new Set(),
|
|
475
|
+
unscannedDirPrefixes: new Set()
|
|
476
|
+
};
|
|
477
|
+
await this.scanDirectory(this.rootDir, previousHashes, previousStats, forceFullHash, scanResult);
|
|
478
|
+
const hashedCount = await this.hashCandidatesWithConcurrency(scanResult);
|
|
479
|
+
const effective = this.buildEffectiveState(previousHashes, previousStats, scanResult);
|
|
480
|
+
return { effective, hashedCount };
|
|
481
|
+
}
|
|
482
|
+
async initialize() {
|
|
483
|
+
console.log(`Initializing file synchronizer for ${this.rootDir}`);
|
|
484
|
+
const { migrated } = await this.loadSnapshot();
|
|
485
|
+
if (migrated) {
|
|
486
|
+
const previousHashes = new Map(this.fileHashes);
|
|
487
|
+
const previousStats = new Map(this.fileStats);
|
|
488
|
+
const { effective } = await this.scanCurrentState(previousHashes, previousStats, true);
|
|
489
|
+
this.fileHashes = effective.fileHashes;
|
|
490
|
+
this.fileStats = effective.fileStats;
|
|
491
|
+
this.partialScan = effective.partialScan;
|
|
492
|
+
this.unscannedDirPrefixes = effective.unscannedDirPrefixes;
|
|
493
|
+
this.merkleRoot = (0, merkle_1.computeMerkleRoot)(this.fileHashes);
|
|
494
|
+
await this.saveSnapshot();
|
|
495
|
+
}
|
|
496
|
+
else if (!this.merkleRoot) {
|
|
497
|
+
this.merkleRoot = (0, merkle_1.computeMerkleRoot)(this.fileHashes);
|
|
498
|
+
}
|
|
499
|
+
console.log(`[Synchronizer] File synchronizer initialized. Loaded ${this.fileHashes.size} tracked files.`);
|
|
500
|
+
}
|
|
501
|
+
async checkForChanges() {
|
|
502
|
+
console.log('[Synchronizer] Checking for file changes...');
|
|
503
|
+
const previousHashes = new Map(this.fileHashes);
|
|
504
|
+
const previousStats = new Map(this.fileStats);
|
|
505
|
+
const previousPartialScan = this.partialScan;
|
|
506
|
+
const previousUnscannedDirPrefixes = [...this.unscannedDirPrefixes];
|
|
507
|
+
const previousCounter = this.fullHashCounter;
|
|
508
|
+
const fullHashInterval = this.getFullHashInterval();
|
|
509
|
+
const nextCounter = fullHashInterval > 0 ? this.fullHashCounter + 1 : this.fullHashCounter;
|
|
510
|
+
const fullHashRun = fullHashInterval > 0 && nextCounter % fullHashInterval === 0;
|
|
511
|
+
const { effective, hashedCount } = await this.scanCurrentState(previousHashes, previousStats, fullHashRun);
|
|
512
|
+
const nextMerkleRoot = (0, merkle_1.computeMerkleRoot)(effective.fileHashes);
|
|
513
|
+
const fileChanges = this.compareStates(previousHashes, effective.fileHashes);
|
|
514
|
+
this.fileHashes = effective.fileHashes;
|
|
515
|
+
this.fileStats = effective.fileStats;
|
|
516
|
+
this.partialScan = effective.partialScan;
|
|
517
|
+
this.unscannedDirPrefixes = effective.unscannedDirPrefixes;
|
|
518
|
+
this.merkleRoot = nextMerkleRoot;
|
|
519
|
+
this.fullHashCounter = nextCounter;
|
|
520
|
+
const hasDiffs = fileChanges.added.length > 0 || fileChanges.removed.length > 0 || fileChanges.modified.length > 0;
|
|
521
|
+
const metadataChanged = previousPartialScan !== this.partialScan
|
|
522
|
+
|| !this.arraysEqual(previousUnscannedDirPrefixes, this.unscannedDirPrefixes);
|
|
523
|
+
const counterAdvanced = previousCounter !== this.fullHashCounter;
|
|
524
|
+
if (hasDiffs || hashedCount > 0 || metadataChanged || counterAdvanced) {
|
|
525
|
+
await this.saveSnapshot();
|
|
526
|
+
}
|
|
527
|
+
if (hasDiffs) {
|
|
528
|
+
console.log(`[Synchronizer] Found changes: ${fileChanges.added.length} added, ${fileChanges.removed.length} removed, ${fileChanges.modified.length} modified.`);
|
|
529
|
+
}
|
|
530
|
+
else {
|
|
531
|
+
console.log('[Synchronizer] No file content changes detected.');
|
|
261
532
|
}
|
|
533
|
+
return {
|
|
534
|
+
...fileChanges,
|
|
535
|
+
hashedCount,
|
|
536
|
+
partialScan: this.partialScan,
|
|
537
|
+
unscannedDirPrefixes: [...this.unscannedDirPrefixes],
|
|
538
|
+
fullHashRun
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
getFileHash(filePath) {
|
|
542
|
+
const normalizedPath = this.normalizeRelPath(filePath);
|
|
543
|
+
if (!normalizedPath) {
|
|
544
|
+
return undefined;
|
|
545
|
+
}
|
|
546
|
+
return this.fileHashes.get(normalizedPath);
|
|
262
547
|
}
|
|
263
548
|
/**
|
|
264
|
-
*
|
|
549
|
+
* Return tracked (currently considered indexable) relative file paths.
|
|
550
|
+
* This reflects the synchronizer snapshot under the active ignore rules.
|
|
551
|
+
*/
|
|
552
|
+
getTrackedRelativePaths() {
|
|
553
|
+
return Array.from(this.fileHashes.keys()).sort();
|
|
554
|
+
}
|
|
555
|
+
/**
|
|
556
|
+
* Delete snapshot file for a given codebase path.
|
|
265
557
|
*/
|
|
266
558
|
static async deleteSnapshot(codebasePath) {
|
|
267
|
-
const
|
|
268
|
-
const merkleDir = path.join(homeDir, '.satori', 'merkle');
|
|
269
|
-
const normalizedPath = path.resolve(codebasePath);
|
|
270
|
-
const hash = crypto.createHash('md5').update(normalizedPath).digest('hex');
|
|
271
|
-
const snapshotPath = path.join(merkleDir, `${hash}.json`);
|
|
559
|
+
const snapshotPath = FileSynchronizer.getSnapshotPathForCodebase(codebasePath);
|
|
272
560
|
try {
|
|
273
|
-
await
|
|
561
|
+
await fsp.unlink(snapshotPath);
|
|
274
562
|
console.log(`Deleted snapshot file: ${snapshotPath}`);
|
|
275
563
|
}
|
|
276
564
|
catch (error) {
|
|
@@ -279,7 +567,7 @@ class FileSynchronizer {
|
|
|
279
567
|
}
|
|
280
568
|
else {
|
|
281
569
|
console.error(`[Synchronizer] Failed to delete snapshot file ${snapshotPath}:`, error.message);
|
|
282
|
-
throw error;
|
|
570
|
+
throw error;
|
|
283
571
|
}
|
|
284
572
|
}
|
|
285
573
|
}
|