@remnic/core 10.0.56 → 10.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +8 -8
- package/dist/access-http-query.js +2 -2
- package/dist/access-http.js +3 -3
- package/dist/{chunk-FCAXGYDT.js → chunk-CTNDFKEX.js} +2 -2
- package/dist/{chunk-CC2ESKHP.js → chunk-E2YAGEFR.js} +7 -7
- package/dist/{chunk-CZS7UTXL.js → chunk-KCTL5EY7.js} +2 -2
- package/dist/{chunk-5C44AMRV.js → chunk-L5UB5BIY.js} +2 -2
- package/dist/{chunk-S3OI5DP5.js → chunk-P5FYZH5G.js} +235 -124
- package/dist/chunk-P5FYZH5G.js.map +1 -0
- package/dist/{chunk-OGR52N7A.js → chunk-W2KH27MQ.js} +3 -3
- package/dist/{chunk-GMRP3HN3.js → chunk-W3R77WHT.js} +2 -2
- package/dist/{chunk-VZ7VJGPL.js → chunk-W4QTA2H5.js} +2 -2
- package/dist/cli.js +6 -6
- package/dist/index.js +8 -8
- package/dist/namespaces/catalog-read-boundary.d.ts +42 -0
- package/dist/namespaces/catalog-read-boundary.d.ts.map +1 -0
- package/dist/namespaces/catalog-repair.d.ts +35 -0
- package/dist/namespaces/catalog-repair.d.ts.map +1 -0
- package/dist/namespaces/catalog.d.ts +31 -27
- package/dist/namespaces/catalog.d.ts.map +1 -1
- package/dist/operator-toolkit.js +3 -3
- package/dist/orchestration/maintenance.js +3 -3
- package/dist/orchestrator.js +8 -8
- package/package.json +2 -2
- package/src/namespaces/catalog-read-boundary.test.ts +121 -0
- package/src/namespaces/catalog-read-boundary.ts +159 -0
- package/src/namespaces/catalog-repair.ts +69 -0
- package/src/namespaces/catalog.test.ts +493 -10
- package/src/namespaces/catalog.ts +112 -110
- package/src/testing/subjects/catalog-read-repair.test.ts +124 -0
- package/dist/chunk-S3OI5DP5.js.map +0 -1
- /package/dist/{chunk-FCAXGYDT.js.map → chunk-CTNDFKEX.js.map} +0 -0
- /package/dist/{chunk-CC2ESKHP.js.map → chunk-E2YAGEFR.js.map} +0 -0
- /package/dist/{chunk-CZS7UTXL.js.map → chunk-KCTL5EY7.js.map} +0 -0
- /package/dist/{chunk-5C44AMRV.js.map → chunk-L5UB5BIY.js.map} +0 -0
- /package/dist/{chunk-OGR52N7A.js.map → chunk-W2KH27MQ.js.map} +0 -0
- /package/dist/{chunk-GMRP3HN3.js.map → chunk-W3R77WHT.js.map} +0 -0
- /package/dist/{chunk-VZ7VJGPL.js.map → chunk-W4QTA2H5.js.map} +0 -0
|
@@ -26,7 +26,7 @@ import {
|
|
|
26
26
|
} from "./chunk-KNA3BXXL.js";
|
|
27
27
|
|
|
28
28
|
// src/namespaces/catalog.ts
|
|
29
|
-
import
|
|
29
|
+
import path2 from "path";
|
|
30
30
|
import {
|
|
31
31
|
appendFile,
|
|
32
32
|
lstat,
|
|
@@ -35,10 +35,131 @@ import {
|
|
|
35
35
|
readdir,
|
|
36
36
|
realpath,
|
|
37
37
|
rename,
|
|
38
|
+
rm,
|
|
38
39
|
stat,
|
|
39
40
|
writeFile
|
|
40
41
|
} from "fs/promises";
|
|
41
42
|
import { StringDecoder } from "string_decoder";
|
|
43
|
+
|
|
44
|
+
// src/namespaces/catalog-read-boundary.ts
|
|
45
|
+
import path from "path";
|
|
46
|
+
async function sanitizeRecordForRead(port, record) {
|
|
47
|
+
if (record.namespace !== port.defaultNamespaceIdentity && !isSafeRouteNamespace(record.namespace)) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
if (await port.isContainedStorageDir(record.storageDir) && await port.isStorageDirForNamespace(record.namespace, record.storageDir)) {
|
|
51
|
+
return record;
|
|
52
|
+
}
|
|
53
|
+
const safe = await port.resolveSafeStorageDir(record.namespace);
|
|
54
|
+
return { ...record, storageDir: safe };
|
|
55
|
+
}
|
|
56
|
+
function storageRootOwnershipRank(port, record, resolvedStorageDir, configured) {
|
|
57
|
+
if (resolvedStorageDir === path.resolve(port.memoryDir)) {
|
|
58
|
+
return record.namespace === port.defaultNamespaceIdentity ? 0 : 3;
|
|
59
|
+
}
|
|
60
|
+
const leaf = path.basename(resolvedStorageDir);
|
|
61
|
+
const tokenOwnsRoot = namespaceIdentityToken(record.namespace) === leaf;
|
|
62
|
+
if (tokenOwnsRoot && configured.has(record.namespace)) {
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
if (record.namespace === leaf) return 1;
|
|
66
|
+
if (tokenOwnsRoot) return 2;
|
|
67
|
+
return 3;
|
|
68
|
+
}
|
|
69
|
+
function configuredNamespaceIdentities(config) {
|
|
70
|
+
return new Set(
|
|
71
|
+
[
|
|
72
|
+
config.defaultNamespace,
|
|
73
|
+
config.sharedNamespace,
|
|
74
|
+
...config.namespacePolicies.map((p) => p.name)
|
|
75
|
+
].map((n) => normalizeNamespaceIdentity(n)).filter((n) => n.length > 0)
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
function preferStorageRootOwner(port, current, candidate, resolvedStorageDir, configured) {
|
|
79
|
+
const currentRank = storageRootOwnershipRank(
|
|
80
|
+
port,
|
|
81
|
+
current,
|
|
82
|
+
resolvedStorageDir,
|
|
83
|
+
configured
|
|
84
|
+
);
|
|
85
|
+
const candidateRank = storageRootOwnershipRank(
|
|
86
|
+
port,
|
|
87
|
+
candidate,
|
|
88
|
+
resolvedStorageDir,
|
|
89
|
+
configured
|
|
90
|
+
);
|
|
91
|
+
if (candidateRank < currentRank) return candidate;
|
|
92
|
+
if (candidateRank > currentRank) return current;
|
|
93
|
+
const byName = candidate.namespace.localeCompare(current.namespace);
|
|
94
|
+
if (byName < 0) return candidate;
|
|
95
|
+
if (byName > 0) return current;
|
|
96
|
+
return candidate.identityToken.localeCompare(current.identityToken) < 0 ? candidate : current;
|
|
97
|
+
}
|
|
98
|
+
function dropDuplicateStorageRootAliases(port, records, config) {
|
|
99
|
+
const byStorageDir = /* @__PURE__ */ new Map();
|
|
100
|
+
const configured = configuredNamespaceIdentities(config);
|
|
101
|
+
for (const record of records) {
|
|
102
|
+
const resolvedStorageDir = path.resolve(record.storageDir);
|
|
103
|
+
const current = byStorageDir.get(resolvedStorageDir);
|
|
104
|
+
if (!current) {
|
|
105
|
+
byStorageDir.set(resolvedStorageDir, record);
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const owner = preferStorageRootOwner(
|
|
109
|
+
port,
|
|
110
|
+
current,
|
|
111
|
+
record,
|
|
112
|
+
resolvedStorageDir,
|
|
113
|
+
configured
|
|
114
|
+
);
|
|
115
|
+
const alias = owner === current ? record : current;
|
|
116
|
+
byStorageDir.set(resolvedStorageDir, mergeNewerTouchFields(owner, alias));
|
|
117
|
+
}
|
|
118
|
+
return [...byStorageDir.values()];
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// src/namespaces/catalog-repair.ts
|
|
122
|
+
import { createHash } from "crypto";
|
|
123
|
+
var MALFORMED_REPAIR_DROP_BACKOFF_MS = 250;
|
|
124
|
+
var MalformedCatalogRepairGate = class {
|
|
125
|
+
inFlight;
|
|
126
|
+
nextAttemptAt = 0;
|
|
127
|
+
attempts = 0;
|
|
128
|
+
/** Test-only observability: attempts that actually entered the gate. */
|
|
129
|
+
get attemptCount() {
|
|
130
|
+
return this.attempts;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Run `attempt` unless one is already in flight or the backoff window is on.
|
|
134
|
+
* Resolves when the attempt settles; never rejects (the repair is fail-open).
|
|
135
|
+
*/
|
|
136
|
+
run(attempt) {
|
|
137
|
+
if (this.inFlight || Date.now() < this.nextAttemptAt) return Promise.resolve();
|
|
138
|
+
this.attempts += 1;
|
|
139
|
+
this.inFlight = Promise.resolve().then(async () => {
|
|
140
|
+
let outcome = "failed";
|
|
141
|
+
try {
|
|
142
|
+
outcome = await attempt();
|
|
143
|
+
} catch {
|
|
144
|
+
outcome = "failed";
|
|
145
|
+
} finally {
|
|
146
|
+
this.inFlight = void 0;
|
|
147
|
+
if (outcome === "dropped") {
|
|
148
|
+
this.nextAttemptAt = Date.now() + MALFORMED_REPAIR_DROP_BACKOFF_MS;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
return this.inFlight;
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
function malformedSidecarPath(catalogPath, dropped) {
|
|
156
|
+
return `${catalogPath}.malformed-${contentHash(dropped.join("\n"))}`;
|
|
157
|
+
}
|
|
158
|
+
function contentHash(input) {
|
|
159
|
+
return createHash("sha256").update(input, "utf8").digest("hex").slice(0, 16);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// src/namespaces/catalog.ts
|
|
42
163
|
var NAMESPACE_KINDS = [
|
|
43
164
|
"default",
|
|
44
165
|
"shared",
|
|
@@ -77,7 +198,7 @@ function isCatalogEnabled(config) {
|
|
|
77
198
|
}
|
|
78
199
|
var FILE_MEMORY_DATA_CHILDREN = /* @__PURE__ */ new Set(["profile.md"]);
|
|
79
200
|
async function inspectMemoryDataMarker(rootDir, child) {
|
|
80
|
-
const childPath =
|
|
201
|
+
const childPath = path2.join(rootDir, child);
|
|
81
202
|
let entry;
|
|
82
203
|
try {
|
|
83
204
|
entry = await lstat(childPath);
|
|
@@ -225,9 +346,9 @@ var NamespaceCatalog = class {
|
|
|
225
346
|
constructor(config) {
|
|
226
347
|
this.config = config;
|
|
227
348
|
this.memoryDir = config.memoryDir;
|
|
228
|
-
this.stateDir =
|
|
229
|
-
this.catalogPath =
|
|
230
|
-
this.rebuildLockPath =
|
|
349
|
+
this.stateDir = path2.join(this.memoryDir, STATE_DIR);
|
|
350
|
+
this.catalogPath = path2.join(this.stateDir, CATALOG_FILE);
|
|
351
|
+
this.rebuildLockPath = path2.join(this.stateDir, REBUILD_LOCK_FILE);
|
|
231
352
|
this.defaultNamespaceIdentity = normalizeNamespaceIdentity(config.defaultNamespace);
|
|
232
353
|
this.compactBytesLimit = config.namespacesCatalogCompactBytes ?? 16 * 1024 * 1024;
|
|
233
354
|
this.readCoalesceMs = config.namespacesCatalogReadTouchCoalesceMs ?? 6e4;
|
|
@@ -297,6 +418,12 @@ var NamespaceCatalog = class {
|
|
|
297
418
|
compactBytesLimit;
|
|
298
419
|
readCoalesceMs;
|
|
299
420
|
writeCoalesceMs;
|
|
421
|
+
// #3298 review: single-flight + backoff gate for the malformed-catalog repair.
|
|
422
|
+
malformedRepairGate = new MalformedCatalogRepairGate();
|
|
423
|
+
/** Test-only observability: how many repair attempts actually entered the gate. */
|
|
424
|
+
get malformedRepairAttemptsForTest() {
|
|
425
|
+
return this.malformedRepairGate.attemptCount;
|
|
426
|
+
}
|
|
300
427
|
// Size after the most recent auto-compaction (0 = none yet). Hysteresis for
|
|
301
428
|
// maybeAutoCompact: a folded catalog that is itself above the limit (many
|
|
302
429
|
// distinct namespaces) would otherwise be re-folded + re-rewritten on every
|
|
@@ -315,94 +442,26 @@ var NamespaceCatalog = class {
|
|
|
315
442
|
return isCatalogEnabled(this.config);
|
|
316
443
|
}
|
|
317
444
|
// ── Public enumeration API ──────────────────────────────────────────────
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
* — return `null` so the caller drops it. The disk SCAN and the hot touch
|
|
327
|
-
* path both reject such names with the SAME default-exempt `isSafeRouteNamespace`
|
|
328
|
-
* gate, so the read boundary MUST agree, or `listNamespaces()`/`getNamespaceRecord()`
|
|
329
|
-
* would expose a namespace those paths reject (note `isStorageDirForNamespace`
|
|
330
|
-
* can still build a tokenized root even for `../evil`, so storageDir sanitation
|
|
331
|
-
* alone does not catch it). The default namespace is exempt (it may be a
|
|
332
|
-
* non-route literal), matching every other validation site.
|
|
333
|
-
*
|
|
334
|
-
* 2. UNSAFE storageDir: for an otherwise-valid namespace, apply the SAME contract
|
|
335
|
-
* as the write path — full containment (`isContainedStorageDir`: lexical +
|
|
336
|
-
* symlink/realpath) AND namespace ownership (`isStorageDirForNamespace`). When
|
|
337
|
-
* a record fails EITHER check we substitute the trusted resolved-and-safe root
|
|
338
|
-
* for that namespace (rule 42: read and write stay symmetric).
|
|
339
|
-
*/
|
|
340
|
-
async sanitizeRecordForRead(record) {
|
|
341
|
-
if (record.namespace !== this.defaultNamespaceIdentity && !isSafeRouteNamespace(record.namespace)) {
|
|
342
|
-
return null;
|
|
343
|
-
}
|
|
344
|
-
if (await this.isContainedStorageDir(record.storageDir) && await this.isStorageDirForNamespace(record.namespace, record.storageDir)) {
|
|
345
|
-
return record;
|
|
346
|
-
}
|
|
347
|
-
const safe = await this.resolveSafeStorageDir(record.namespace);
|
|
348
|
-
return { ...record, storageDir: safe };
|
|
349
|
-
}
|
|
350
|
-
storageRootOwnershipRank(record, resolvedStorageDir, configured) {
|
|
351
|
-
if (resolvedStorageDir === path.resolve(this.memoryDir)) {
|
|
352
|
-
return record.namespace === this.defaultNamespaceIdentity ? 0 : 3;
|
|
353
|
-
}
|
|
354
|
-
const leaf = path.basename(resolvedStorageDir);
|
|
355
|
-
const tokenOwnsRoot = namespaceIdentityToken(record.namespace) === leaf;
|
|
356
|
-
if (tokenOwnsRoot && configured.has(record.namespace)) {
|
|
357
|
-
return 0;
|
|
358
|
-
}
|
|
359
|
-
if (record.namespace === leaf) return 1;
|
|
360
|
-
if (tokenOwnsRoot) return 2;
|
|
361
|
-
return 3;
|
|
362
|
-
}
|
|
363
|
-
configuredNamespaceIdentities() {
|
|
364
|
-
return new Set(
|
|
365
|
-
[
|
|
366
|
-
this.config.defaultNamespace,
|
|
367
|
-
this.config.sharedNamespace,
|
|
368
|
-
...this.config.namespacePolicies.map((p) => p.name)
|
|
369
|
-
].map((n) => normalizeNamespaceIdentity(n)).filter((n) => n.length > 0)
|
|
370
|
-
);
|
|
371
|
-
}
|
|
372
|
-
preferStorageRootOwner(current, candidate, resolvedStorageDir, configured) {
|
|
373
|
-
const currentRank = this.storageRootOwnershipRank(current, resolvedStorageDir, configured);
|
|
374
|
-
const candidateRank = this.storageRootOwnershipRank(candidate, resolvedStorageDir, configured);
|
|
375
|
-
if (candidateRank < currentRank) return candidate;
|
|
376
|
-
if (candidateRank > currentRank) return current;
|
|
377
|
-
const byName = candidate.namespace.localeCompare(current.namespace);
|
|
378
|
-
if (byName < 0) return candidate;
|
|
379
|
-
if (byName > 0) return current;
|
|
380
|
-
return candidate.identityToken.localeCompare(current.identityToken) < 0 ? candidate : current;
|
|
381
|
-
}
|
|
382
|
-
dropDuplicateStorageRootAliases(records) {
|
|
383
|
-
const byStorageDir = /* @__PURE__ */ new Map();
|
|
384
|
-
const configured = this.configuredNamespaceIdentities();
|
|
385
|
-
for (const record of records) {
|
|
386
|
-
const resolvedStorageDir = path.resolve(record.storageDir);
|
|
387
|
-
const current = byStorageDir.get(resolvedStorageDir);
|
|
388
|
-
if (!current) {
|
|
389
|
-
byStorageDir.set(resolvedStorageDir, record);
|
|
390
|
-
continue;
|
|
391
|
-
}
|
|
392
|
-
const owner = this.preferStorageRootOwner(current, record, resolvedStorageDir, configured);
|
|
393
|
-
const alias = owner === current ? record : current;
|
|
394
|
-
byStorageDir.set(resolvedStorageDir, mergeNewerTouchFields(owner, alias));
|
|
395
|
-
}
|
|
396
|
-
return [...byStorageDir.values()];
|
|
445
|
+
catalogReadBoundaryPort() {
|
|
446
|
+
return {
|
|
447
|
+
defaultNamespaceIdentity: this.defaultNamespaceIdentity,
|
|
448
|
+
memoryDir: this.memoryDir,
|
|
449
|
+
isContainedStorageDir: (candidate) => this.isContainedStorageDir(candidate),
|
|
450
|
+
isStorageDirForNamespace: (namespace, candidate) => this.isStorageDirForNamespace(namespace, candidate),
|
|
451
|
+
resolveSafeStorageDir: (namespace) => this.resolveSafeStorageDir(namespace)
|
|
452
|
+
};
|
|
397
453
|
}
|
|
398
454
|
loadSanitizedRecords(health) {
|
|
399
455
|
return this.queueCritical(async () => {
|
|
400
456
|
const records = await this.loadCompacted(false, health);
|
|
457
|
+
const port = this.catalogReadBoundaryPort();
|
|
401
458
|
const sanitized = await Promise.all(
|
|
402
|
-
[...records.values()].map((r) =>
|
|
459
|
+
[...records.values()].map((r) => sanitizeRecordForRead(port, r))
|
|
403
460
|
);
|
|
404
|
-
return
|
|
405
|
-
|
|
461
|
+
return dropDuplicateStorageRootAliases(
|
|
462
|
+
port,
|
|
463
|
+
sanitized.filter((r) => r !== null),
|
|
464
|
+
this.config
|
|
406
465
|
);
|
|
407
466
|
});
|
|
408
467
|
}
|
|
@@ -410,7 +469,12 @@ var NamespaceCatalog = class {
|
|
|
410
469
|
if (!this.enabled) return [];
|
|
411
470
|
const health = {};
|
|
412
471
|
let out = await this.loadSanitizedRecords(health);
|
|
413
|
-
if (health.incomplete !== void 0)
|
|
472
|
+
if (health.incomplete !== void 0) {
|
|
473
|
+
options?.onIncomplete?.(health.incomplete);
|
|
474
|
+
}
|
|
475
|
+
if (health.malformed === true) {
|
|
476
|
+
void this.malformedRepairGate.run(() => this.rewriteAfterIncompleteRead()).catch(() => void 0);
|
|
477
|
+
}
|
|
414
478
|
if (filter?.kind) out = out.filter((r) => r.kind === filter.kind);
|
|
415
479
|
if (filter?.discoveredBy) out = out.filter((r) => r.discoveredBy === filter.discoveredBy);
|
|
416
480
|
if (filter?.writtenSince) {
|
|
@@ -596,10 +660,10 @@ var NamespaceCatalog = class {
|
|
|
596
660
|
return this.namespaceTokenDir(token);
|
|
597
661
|
}
|
|
598
662
|
namespaceTokenDir(token) {
|
|
599
|
-
if (token.length === 0 || token.includes("/") || token.includes("\\") || token.includes("..") ||
|
|
663
|
+
if (token.length === 0 || token.includes("/") || token.includes("\\") || token.includes("..") || path2.isAbsolute(token)) {
|
|
600
664
|
throw new Error(`unsafe namespace token: ${token}`);
|
|
601
665
|
}
|
|
602
|
-
return
|
|
666
|
+
return path2.join(this.memoryDir, "namespaces", token);
|
|
603
667
|
}
|
|
604
668
|
/**
|
|
605
669
|
* Whether a candidate storage dir is LEXICALLY contained: it is either the
|
|
@@ -610,11 +674,11 @@ var NamespaceCatalog = class {
|
|
|
610
674
|
* pure string check — symlink escape is checked separately via realpath.
|
|
611
675
|
*/
|
|
612
676
|
isLexicallyContained(candidate) {
|
|
613
|
-
const resolved =
|
|
614
|
-
if (resolved ===
|
|
615
|
-
const nsBase =
|
|
616
|
-
const rel =
|
|
617
|
-
return rel.length > 0 && !rel.startsWith("..") && !
|
|
677
|
+
const resolved = path2.resolve(candidate);
|
|
678
|
+
if (resolved === path2.resolve(this.memoryDir)) return true;
|
|
679
|
+
const nsBase = path2.resolve(path2.join(this.memoryDir, "namespaces"));
|
|
680
|
+
const rel = path2.relative(nsBase, resolved);
|
|
681
|
+
return rel.length > 0 && !rel.startsWith("..") && !path2.isAbsolute(rel);
|
|
618
682
|
}
|
|
619
683
|
/**
|
|
620
684
|
* Whether a candidate storage dir satisfies the catalog containment contract,
|
|
@@ -628,12 +692,12 @@ var NamespaceCatalog = class {
|
|
|
628
692
|
*/
|
|
629
693
|
async isContainedStorageDir(candidate) {
|
|
630
694
|
if (!this.isLexicallyContained(candidate)) return false;
|
|
631
|
-
if (
|
|
695
|
+
if (path2.resolve(candidate) === path2.resolve(this.memoryDir)) return true;
|
|
632
696
|
let memoryReal;
|
|
633
697
|
try {
|
|
634
698
|
memoryReal = await realpath(this.memoryDir);
|
|
635
699
|
} catch {
|
|
636
|
-
memoryReal =
|
|
700
|
+
memoryReal = path2.resolve(this.memoryDir);
|
|
637
701
|
}
|
|
638
702
|
if (await this.hasSymlinkedAncestor(candidate)) return false;
|
|
639
703
|
try {
|
|
@@ -659,15 +723,15 @@ var NamespaceCatalog = class {
|
|
|
659
723
|
* symlink-checked by the caller; this walks only the intermediate ancestors.
|
|
660
724
|
*/
|
|
661
725
|
async hasSymlinkedAncestor(candidate) {
|
|
662
|
-
const stopAt =
|
|
663
|
-
let dir =
|
|
664
|
-
const root =
|
|
665
|
-
while (dir !== stopAt && dir !== root && dir !==
|
|
726
|
+
const stopAt = path2.resolve(this.memoryDir);
|
|
727
|
+
let dir = path2.dirname(path2.resolve(candidate));
|
|
728
|
+
const root = path2.parse(dir).root;
|
|
729
|
+
while (dir !== stopAt && dir !== root && dir !== path2.dirname(dir)) {
|
|
666
730
|
try {
|
|
667
731
|
if ((await lstat(dir)).isSymbolicLink()) return true;
|
|
668
732
|
} catch {
|
|
669
733
|
}
|
|
670
|
-
dir =
|
|
734
|
+
dir = path2.dirname(dir);
|
|
671
735
|
}
|
|
672
736
|
return false;
|
|
673
737
|
}
|
|
@@ -686,10 +750,10 @@ var NamespaceCatalog = class {
|
|
|
686
750
|
* (NF21i) and the disk scan, so every containment consumer agrees.
|
|
687
751
|
*/
|
|
688
752
|
async isNearestExistingAncestorContained(candidate, memoryReal) {
|
|
689
|
-
let dir =
|
|
690
|
-
const root =
|
|
753
|
+
let dir = path2.resolve(candidate);
|
|
754
|
+
const root = path2.parse(dir).root;
|
|
691
755
|
for (; ; ) {
|
|
692
|
-
const parent =
|
|
756
|
+
const parent = path2.dirname(dir);
|
|
693
757
|
if (parent === dir || dir === root) return false;
|
|
694
758
|
let real;
|
|
695
759
|
try {
|
|
@@ -736,24 +800,24 @@ var NamespaceCatalog = class {
|
|
|
736
800
|
* namespace's root. Compared on resolved (absolute) paths.
|
|
737
801
|
*/
|
|
738
802
|
async isStorageDirForNamespace(namespace, candidate) {
|
|
739
|
-
const resolvedCandidate =
|
|
803
|
+
const resolvedCandidate = path2.resolve(candidate);
|
|
740
804
|
const valid = /* @__PURE__ */ new Set();
|
|
741
805
|
try {
|
|
742
|
-
valid.add(
|
|
806
|
+
valid.add(path2.resolve(this.namespaceTokenDir(namespaceIdentityToken(namespace))));
|
|
743
807
|
} catch {
|
|
744
808
|
}
|
|
745
809
|
try {
|
|
746
|
-
valid.add(
|
|
810
|
+
valid.add(path2.resolve(this.namespaceTokenDir(namespace)));
|
|
747
811
|
} catch {
|
|
748
812
|
}
|
|
749
813
|
try {
|
|
750
|
-
valid.add(
|
|
814
|
+
valid.add(path2.resolve(await resolveNamespaceStorageRoot(this.config, namespace)));
|
|
751
815
|
} catch {
|
|
752
816
|
}
|
|
753
817
|
if (normalizeNamespaceIdentity(namespace) === this.defaultNamespaceIdentity) {
|
|
754
|
-
valid.add(
|
|
818
|
+
valid.add(path2.resolve(this.memoryDir));
|
|
755
819
|
try {
|
|
756
|
-
valid.add(
|
|
820
|
+
valid.add(path2.resolve(await resolveDefaultNamespaceRoot(this.config)));
|
|
757
821
|
} catch {
|
|
758
822
|
}
|
|
759
823
|
}
|
|
@@ -862,7 +926,7 @@ var NamespaceCatalog = class {
|
|
|
862
926
|
} catch {
|
|
863
927
|
return false;
|
|
864
928
|
}
|
|
865
|
-
if (normalizeNamespaceIdentity(namespace) !== this.defaultNamespaceIdentity &&
|
|
929
|
+
if (normalizeNamespaceIdentity(namespace) !== this.defaultNamespaceIdentity && path2.resolve(root) === path2.resolve(this.memoryDir)) {
|
|
866
930
|
return false;
|
|
867
931
|
}
|
|
868
932
|
let stat2;
|
|
@@ -929,6 +993,40 @@ var NamespaceCatalog = class {
|
|
|
929
993
|
})
|
|
930
994
|
);
|
|
931
995
|
}
|
|
996
|
+
/**
|
|
997
|
+
* Rewrite the catalog after a read skipped malformed rows (#3297). Same fold-and-rewrite as the
|
|
998
|
+
* size-triggered compaction — the readable rows survive, the malformed ones do not — so a stuck
|
|
999
|
+
* bad row stops keeping every completion request owed. Best-effort and fail-open (rule #40): a
|
|
1000
|
+
* failed rewrite leaves the catalog unchanged and the next read reports and retries.
|
|
1001
|
+
*
|
|
1002
|
+
* Locking: the fold runs inside `queueCritical` (in-process serialization, like every other
|
|
1003
|
+
* read), and the rewrite's load→rename window runs under the held catalog file lock, so it can
|
|
1004
|
+
* never interleave with another process's touch append or rebuild. If another process already
|
|
1005
|
+
* holds the lock, the rewrite is dropped (not queued): that process's own rewrite or rebuild is
|
|
1006
|
+
* the likeliest repair, and the next incomplete read retries anyway.
|
|
1007
|
+
*
|
|
1008
|
+
* Data safety: the locked re-read must itself reach the end of the file and still find
|
|
1009
|
+
* malformed rows, and must leave at least one readable row. A failed re-read folds to an empty
|
|
1010
|
+
* map, and a catalog with nothing readable is left for `rebuildFromDisk`: rewriting either
|
|
1011
|
+
* would replace a healthy or recoverable catalog with an empty file.
|
|
1012
|
+
*/
|
|
1013
|
+
async rewriteAfterIncompleteRead() {
|
|
1014
|
+
return this.queueCritical(
|
|
1015
|
+
async () => this.withHeldCatalogLock(async (acquired) => {
|
|
1016
|
+
if (!acquired) return "dropped";
|
|
1017
|
+
const health = { malformedLines: [] };
|
|
1018
|
+
const folded = await this.loadCompacted(true, health);
|
|
1019
|
+
if (health.malformed !== true || folded.size === 0) return "failed";
|
|
1020
|
+
const dropped = health.malformedLines ?? [];
|
|
1021
|
+
if (dropped.length > 0) {
|
|
1022
|
+
const sidecar = malformedSidecarPath(this.catalogPath, dropped);
|
|
1023
|
+
await writeFile(sidecar, dropped.join("\n") + "\n", "utf8").catch(() => void 0);
|
|
1024
|
+
}
|
|
1025
|
+
await this.rewriteUnchained([...folded.values()]);
|
|
1026
|
+
return "repaired";
|
|
1027
|
+
})
|
|
1028
|
+
);
|
|
1029
|
+
}
|
|
932
1030
|
/**
|
|
933
1031
|
* Fold-and-rewrite the JSONL log when it exceeds `compactBytesLimit`
|
|
934
1032
|
* (issue #1903). MUST be called only from inside `touch`'s held-lock +
|
|
@@ -1039,7 +1137,7 @@ var NamespaceCatalog = class {
|
|
|
1039
1137
|
})
|
|
1040
1138
|
);
|
|
1041
1139
|
}
|
|
1042
|
-
const namespacesDir =
|
|
1140
|
+
const namespacesDir = path2.join(this.memoryDir, "namespaces");
|
|
1043
1141
|
let entries = [];
|
|
1044
1142
|
let namespacesDirSafe = true;
|
|
1045
1143
|
try {
|
|
@@ -1067,7 +1165,7 @@ var NamespaceCatalog = class {
|
|
|
1067
1165
|
const scannedFromTokenized = /* @__PURE__ */ new Map();
|
|
1068
1166
|
for (const entry of entries) {
|
|
1069
1167
|
const token = entry.name;
|
|
1070
|
-
const fullPath =
|
|
1168
|
+
const fullPath = path2.join(namespacesDir, token);
|
|
1071
1169
|
let stat2;
|
|
1072
1170
|
try {
|
|
1073
1171
|
stat2 = await lstat(fullPath);
|
|
@@ -1091,7 +1189,7 @@ var NamespaceCatalog = class {
|
|
|
1091
1189
|
continue;
|
|
1092
1190
|
}
|
|
1093
1191
|
const literalRecord = existing.get(token);
|
|
1094
|
-
const literalOwnsRoot = configured.has(token) || literalRecord !== void 0 &&
|
|
1192
|
+
const literalOwnsRoot = configured.has(token) || literalRecord !== void 0 && path2.resolve(literalRecord.storageDir) === path2.resolve(fullPath);
|
|
1095
1193
|
const tokenDecoded = literalOwnsRoot ? null : namespaceIdentityFromToken(token);
|
|
1096
1194
|
const rawDecoded = tokenDecoded && tokenDecoded.length > 0 ? tokenDecoded : token;
|
|
1097
1195
|
const decoded = normalizeNamespaceIdentity(rawDecoded);
|
|
@@ -1178,10 +1276,10 @@ var NamespaceCatalog = class {
|
|
|
1178
1276
|
}
|
|
1179
1277
|
if (await this.liveStorageRootExistsForRebuild(ns, memoryReal)) {
|
|
1180
1278
|
const safeDir = await this.resolveSafeStorageDir(ns);
|
|
1181
|
-
const resolvedSafe =
|
|
1279
|
+
const resolvedSafe = path2.resolve(safeDir);
|
|
1182
1280
|
let owningNamespace = null;
|
|
1183
1281
|
for (const [otherNs, otherRec] of rebuilt) {
|
|
1184
|
-
if (otherNs !== ns &&
|
|
1282
|
+
if (otherNs !== ns && path2.resolve(otherRec.storageDir) === resolvedSafe) {
|
|
1185
1283
|
owningNamespace = otherNs;
|
|
1186
1284
|
break;
|
|
1187
1285
|
}
|
|
@@ -1314,7 +1412,10 @@ var NamespaceCatalog = class {
|
|
|
1314
1412
|
};
|
|
1315
1413
|
if (!forceFresh && this.compactedCache && this.compactedCache.identity.size === identity.size && this.compactedCache.identity.mtimeMs === identity.mtimeMs && this.compactedCache.identity.ctimeMs === identity.ctimeMs && this.compactedCache.identity.ino === identity.ino) {
|
|
1316
1414
|
await handle.close().catch(() => void 0);
|
|
1317
|
-
if (health && this.compactedCache.incomplete)
|
|
1415
|
+
if (health && this.compactedCache.incomplete) {
|
|
1416
|
+
health.incomplete = this.compactedCache.incomplete;
|
|
1417
|
+
health.malformed = true;
|
|
1418
|
+
}
|
|
1318
1419
|
return this.cloneCompactedRecords(this.compactedCache.records);
|
|
1319
1420
|
}
|
|
1320
1421
|
} catch {
|
|
@@ -1336,11 +1437,13 @@ var NamespaceCatalog = class {
|
|
|
1336
1437
|
parsed = JSON.parse(trimmed);
|
|
1337
1438
|
} catch {
|
|
1338
1439
|
malformed += 1;
|
|
1440
|
+
health?.malformedLines?.push(trimmed);
|
|
1339
1441
|
return;
|
|
1340
1442
|
}
|
|
1341
1443
|
const record = coerceRecord(parsed);
|
|
1342
1444
|
if (!record) {
|
|
1343
1445
|
malformed += 1;
|
|
1446
|
+
health?.malformedLines?.push(trimmed);
|
|
1344
1447
|
return;
|
|
1345
1448
|
}
|
|
1346
1449
|
const prior = records.get(record.namespace);
|
|
@@ -1371,7 +1474,10 @@ var NamespaceCatalog = class {
|
|
|
1371
1474
|
await handle.close().catch(() => void 0);
|
|
1372
1475
|
}
|
|
1373
1476
|
const incomplete = malformed > 0 ? `${malformed} malformed catalog line(s) skipped` : void 0;
|
|
1374
|
-
if (health && incomplete)
|
|
1477
|
+
if (health && incomplete) {
|
|
1478
|
+
health.incomplete = incomplete;
|
|
1479
|
+
health.malformed = true;
|
|
1480
|
+
}
|
|
1375
1481
|
this.compactedCache = identity ? { identity, records, incomplete } : void 0;
|
|
1376
1482
|
return this.cloneCompactedRecords(records);
|
|
1377
1483
|
}
|
|
@@ -1429,8 +1535,13 @@ var NamespaceCatalog = class {
|
|
|
1429
1535
|
const body = records.map((r) => serializeRecord(r)).join("\n") + (records.length > 0 ? "\n" : "");
|
|
1430
1536
|
await mkdir(this.stateDir, { recursive: true });
|
|
1431
1537
|
const tmp = `${this.catalogPath}.${process.pid}.${Date.now()}.tmp`;
|
|
1432
|
-
|
|
1433
|
-
|
|
1538
|
+
try {
|
|
1539
|
+
await writeFile(tmp, body, "utf8");
|
|
1540
|
+
await rename(tmp, this.catalogPath);
|
|
1541
|
+
} catch (error) {
|
|
1542
|
+
await rm(tmp, { force: true }).catch(() => void 0);
|
|
1543
|
+
throw error;
|
|
1544
|
+
}
|
|
1434
1545
|
await this.refreshCacheIdentity(new Map(records.map((record) => [record.namespace, record])));
|
|
1435
1546
|
}
|
|
1436
1547
|
async refreshCacheAfterAppend(record, lineByteLength) {
|
|
@@ -1476,8 +1587,8 @@ var NamespaceCatalog = class {
|
|
|
1476
1587
|
}
|
|
1477
1588
|
};
|
|
1478
1589
|
function isPathInside(root, child) {
|
|
1479
|
-
const relative =
|
|
1480
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
1590
|
+
const relative = path2.relative(root, child);
|
|
1591
|
+
return relative === "" || !relative.startsWith("..") && !path2.isAbsolute(relative);
|
|
1481
1592
|
}
|
|
1482
1593
|
|
|
1483
1594
|
export {
|
|
@@ -1485,4 +1596,4 @@ export {
|
|
|
1485
1596
|
isNamespaceKind,
|
|
1486
1597
|
NamespaceCatalog
|
|
1487
1598
|
};
|
|
1488
|
-
//# sourceMappingURL=chunk-
|
|
1599
|
+
//# sourceMappingURL=chunk-P5FYZH5G.js.map
|