@remnic/core 9.3.653 → 9.3.654
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +17 -17
- package/dist/access-http.d.ts +4 -4
- package/dist/access-http.js +10 -10
- package/dist/access-mcp.d.ts +4 -4
- package/dist/access-mcp.js +9 -9
- package/dist/access-schema.d.ts +12 -12
- package/dist/{access-service-CdJFd3_b.d.ts → access-service-C8A5hoXJ.d.ts} +11 -2
- package/dist/access-service.d.ts +4 -4
- package/dist/access-service.js +8 -8
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +1 -1
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +3 -3
- package/dist/briefing.d.ts +1 -1
- package/dist/briefing.js +3 -3
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +1 -1
- package/dist/calibration.d.ts +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +4 -4
- package/dist/{chunk-KJDKZVF3.js → chunk-2DSTAWNZ.js} +3 -3
- package/dist/chunk-3RACUBII.js +212 -0
- package/dist/chunk-3RACUBII.js.map +1 -0
- package/dist/{chunk-Y7NWBBHV.js → chunk-6CVI6BP6.js} +2 -2
- package/dist/{chunk-R3PQUPQ4.js → chunk-6IMKOIZ6.js} +85 -3
- package/dist/chunk-6IMKOIZ6.js.map +1 -0
- package/dist/{chunk-WTI35CVJ.js → chunk-BJA6DQOC.js} +5 -5
- package/dist/{chunk-GI45G4BK.js → chunk-BP2EV6W5.js} +3 -3
- package/dist/{chunk-WLGE6KEO.js → chunk-DBM2BD22.js} +3 -3
- package/dist/{chunk-IENGGY2C.js → chunk-ENV6RDTD.js} +2 -2
- package/dist/{chunk-BEMWL2FZ.js → chunk-FVRBLJP6.js} +2 -2
- package/dist/{chunk-H3PHZLMF.js → chunk-GKKAXVAJ.js} +20 -11
- package/dist/chunk-GKKAXVAJ.js.map +1 -0
- package/dist/{chunk-NOBL7OUP.js → chunk-GPW2E4LN.js} +12 -5
- package/dist/{chunk-NOBL7OUP.js.map → chunk-GPW2E4LN.js.map} +1 -1
- package/dist/{chunk-KWM33SPU.js → chunk-JMQSYGXS.js} +2 -2
- package/dist/{chunk-QQHIQ7JD.js → chunk-JYN7QNTA.js} +87 -18
- package/dist/chunk-JYN7QNTA.js.map +1 -0
- package/dist/{chunk-AJE7FJVE.js → chunk-K6X553JB.js} +2 -2
- package/dist/{chunk-E3J6O6N7.js → chunk-LJCEWTG3.js} +19 -8
- package/dist/{chunk-E3J6O6N7.js.map → chunk-LJCEWTG3.js.map} +1 -1
- package/dist/{chunk-EW52H5EM.js → chunk-NAZWHTYV.js} +12 -5
- package/dist/chunk-NAZWHTYV.js.map +1 -0
- package/dist/{chunk-XMN6MMTU.js → chunk-NCGWXCSW.js} +2 -2
- package/dist/{chunk-C43KEWEV.js → chunk-NE2JBMLN.js} +1 -1
- package/dist/chunk-NE2JBMLN.js.map +1 -0
- package/dist/{chunk-SPMZZUEJ.js → chunk-OL2364SB.js} +2020 -368
- package/dist/chunk-OL2364SB.js.map +1 -0
- package/dist/{chunk-JF7SFXTG.js → chunk-QKK64Z6M.js} +2 -2
- package/dist/{chunk-IVYSVAC6.js → chunk-QW6JZO5P.js} +2 -2
- package/dist/{chunk-EHQLDFSH.js → chunk-RGPUQ66K.js} +2 -2
- package/dist/{chunk-CFOCZPIQ.js → chunk-T2C6QJG2.js} +2 -2
- package/dist/{chunk-V4UDXYGG.js → chunk-XWQ6ERUG.js} +2 -2
- package/dist/{chunk-BNFRL6QW.js → chunk-Y2RIIF6H.js} +2 -2
- package/dist/{chunk-C63WC454.js → chunk-YLZLPVKK.js} +22 -1
- package/dist/chunk-YLZLPVKK.js.map +1 -0
- package/dist/{chunk-RZOBQ23O.js → chunk-Z5MQI7K2.js} +2 -2
- package/dist/{chunk-PRQXUSQV.js → chunk-ZCORQM74.js} +2 -2
- package/dist/{cli-DDo7Qgs-.d.ts → cli-uQgvDFNE.d.ts} +3 -3
- package/dist/cli.d.ts +5 -5
- package/dist/cli.js +22 -22
- package/dist/compounding/engine.d.ts +1 -1
- package/dist/compounding/engine.js +3 -3
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +1 -1
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +3 -3
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +3 -3
- package/dist/consolidation-provenance-check.d.ts +1 -1
- package/dist/consolidation-undo.d.ts +1 -1
- package/dist/contradiction/index.d.ts +19 -1
- package/dist/contradiction/index.js +1 -1
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +1 -1
- package/dist/entity-retrieval.js +3 -3
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +3 -3
- package/dist/explicit-capture.js +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/identity-continuity.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/index.d.ts +8 -8
- package/dist/index.js +30 -28
- package/dist/index.js.map +1 -1
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +3 -3
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
- package/dist/maintenance/rebuild-memory-projection.js +4 -4
- package/dist/mcp-memory-inspector-app.d.ts +4 -4
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +1 -1
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +1 -1
- package/dist/namespaces/migrate.js +4 -4
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/storage.d.ts +52 -3
- package/dist/namespaces/storage.js +9 -5
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +1 -1
- package/dist/operator-toolkit.js +7 -7
- package/dist/{orchestrator-8fTZsa0y.d.ts → orchestrator-B4Y4sWQH.d.ts} +503 -3
- package/dist/orchestrator.d.ts +3 -3
- package/dist/orchestrator.js +13 -13
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-explain-renderer.js +3 -3
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-cli.js +4 -4
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray-renderer.js +3 -3
- package/dist/recall-xray.d.ts +1 -1
- package/dist/recall-xray.js +2 -2
- package/dist/{resolution-3SAP4SH2.js → resolution-IDTEBJFS.js} +2 -2
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/resume-bundles.js +2 -2
- package/dist/retrieval-agents.d.ts +1 -1
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/index.d.ts +1 -1
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-DKdYzQOg.d.ts → semantic-consolidation-BKd0Pype.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +4 -4
- package/dist/semantic-rule-promotion.js +3 -3
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +3 -3
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +1 -1
- package/dist/signal.d.ts +1 -1
- package/dist/storage.d.ts +1 -1
- package/dist/storage.js +2 -2
- package/dist/summarizer.d.ts +1 -1
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/temporal-supersession.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +1 -1
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/{types-D8yUmSik.d.ts → types-BgChEr0M.d.ts} +11 -0
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/verified-recall.js +3 -3
- package/package.json +1 -1
- package/src/access-http.ts +7 -0
- package/src/access-mcp.ts +7 -0
- package/src/access-service.ts +12 -0
- package/src/cli.ts +104 -0
- package/src/config.test.ts +40 -0
- package/src/config.ts +29 -0
- package/src/contradiction/contradiction.test.ts +284 -0
- package/src/contradiction/resolution.ts +151 -4
- package/src/explicit-capture.ts +31 -10
- package/src/index.ts +10 -0
- package/src/namespaces/catalog.test.ts +3356 -0
- package/src/namespaces/catalog.ts +2123 -0
- package/src/namespaces/storage.ts +210 -30
- package/src/orchestrator-flush.test.ts +300 -0
- package/src/orchestrator.ts +851 -240
- package/src/types.ts +11 -0
- package/dist/chunk-C43KEWEV.js.map +0 -1
- package/dist/chunk-C63WC454.js.map +0 -1
- package/dist/chunk-EW52H5EM.js.map +0 -1
- package/dist/chunk-H3PHZLMF.js.map +0 -1
- package/dist/chunk-ORGWWNJG.js +0 -131
- package/dist/chunk-ORGWWNJG.js.map +0 -1
- package/dist/chunk-QQHIQ7JD.js.map +0 -1
- package/dist/chunk-R3PQUPQ4.js.map +0 -1
- package/dist/chunk-SPMZZUEJ.js.map +0 -1
- /package/dist/{chunk-KJDKZVF3.js.map → chunk-2DSTAWNZ.js.map} +0 -0
- /package/dist/{chunk-Y7NWBBHV.js.map → chunk-6CVI6BP6.js.map} +0 -0
- /package/dist/{chunk-WTI35CVJ.js.map → chunk-BJA6DQOC.js.map} +0 -0
- /package/dist/{chunk-GI45G4BK.js.map → chunk-BP2EV6W5.js.map} +0 -0
- /package/dist/{chunk-WLGE6KEO.js.map → chunk-DBM2BD22.js.map} +0 -0
- /package/dist/{chunk-IENGGY2C.js.map → chunk-ENV6RDTD.js.map} +0 -0
- /package/dist/{chunk-BEMWL2FZ.js.map → chunk-FVRBLJP6.js.map} +0 -0
- /package/dist/{chunk-KWM33SPU.js.map → chunk-JMQSYGXS.js.map} +0 -0
- /package/dist/{chunk-AJE7FJVE.js.map → chunk-K6X553JB.js.map} +0 -0
- /package/dist/{chunk-XMN6MMTU.js.map → chunk-NCGWXCSW.js.map} +0 -0
- /package/dist/{chunk-JF7SFXTG.js.map → chunk-QKK64Z6M.js.map} +0 -0
- /package/dist/{chunk-IVYSVAC6.js.map → chunk-QW6JZO5P.js.map} +0 -0
- /package/dist/{chunk-EHQLDFSH.js.map → chunk-RGPUQ66K.js.map} +0 -0
- /package/dist/{chunk-CFOCZPIQ.js.map → chunk-T2C6QJG2.js.map} +0 -0
- /package/dist/{chunk-V4UDXYGG.js.map → chunk-XWQ6ERUG.js.map} +0 -0
- /package/dist/{chunk-BNFRL6QW.js.map → chunk-Y2RIIF6H.js.map} +0 -0
- /package/dist/{chunk-RZOBQ23O.js.map → chunk-Z5MQI7K2.js.map} +0 -0
- /package/dist/{chunk-PRQXUSQV.js.map → chunk-ZCORQM74.js.map} +0 -0
- /package/dist/{resolution-3SAP4SH2.js.map → resolution-IDTEBJFS.js.map} +0 -0
|
@@ -0,0 +1,2123 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import type { Dirent } from "node:fs";
|
|
4
|
+
import {
|
|
5
|
+
appendFile,
|
|
6
|
+
lstat,
|
|
7
|
+
mkdir,
|
|
8
|
+
open,
|
|
9
|
+
readdir,
|
|
10
|
+
readFile,
|
|
11
|
+
realpath,
|
|
12
|
+
rename,
|
|
13
|
+
stat,
|
|
14
|
+
unlink,
|
|
15
|
+
utimes,
|
|
16
|
+
writeFile,
|
|
17
|
+
} from "node:fs/promises";
|
|
18
|
+
import type { PluginConfig } from "../types.js";
|
|
19
|
+
import { isSafeRouteNamespace } from "../routing/engine.js";
|
|
20
|
+
import { namespaceIdentityFromToken, namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
|
|
21
|
+
import { resolveDefaultNamespaceRoot, resolveNamespaceStorageRoot } from "./storage.js";
|
|
22
|
+
import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Rebuildable namespace catalog (issue #1499).
|
|
26
|
+
*
|
|
27
|
+
* Purpose: a downstream, rebuildable metadata index that lets Remnic ENUMERATE
|
|
28
|
+
* the configured and dynamically-created namespaces that exist or should be
|
|
29
|
+
* maintained. Filesystem memory remains the single source of truth; the catalog
|
|
30
|
+
* is derived metadata and can always be reconstructed from disk.
|
|
31
|
+
*
|
|
32
|
+
* Storage format: `<memoryDir>/state/namespaces.jsonl` — an append-and-compact
|
|
33
|
+
* JSON-lines log. We chose this over per-namespace sidecar files because:
|
|
34
|
+
* - touches (markRead/markWrite/markMaintenance) are cheap single appends;
|
|
35
|
+
* - it is naturally audit-friendly (the raw log preserves touch history);
|
|
36
|
+
* - a single file makes enumeration trivial (no directory walk per call);
|
|
37
|
+
* - last-record-wins compaction folds the log into the current state on read,
|
|
38
|
+
* and `rebuildFromDisk` rewrites it atomically (temp file + rename).
|
|
39
|
+
*
|
|
40
|
+
* SECURITY:
|
|
41
|
+
* - The catalog stores ONLY metadata (namespace names, kinds, timestamps,
|
|
42
|
+
* resolved storage dirs). It NEVER holds raw memory content or secrets.
|
|
43
|
+
* - Catalog presence grants NO authorization. Read/write access still flows
|
|
44
|
+
* through the namespace policies in `principal.ts`; this module never makes
|
|
45
|
+
* an access decision.
|
|
46
|
+
* - All namespace tokens are validated with `isSafeRouteNamespace` (except the
|
|
47
|
+
* configured default namespace, which is exempt at the routing layer) and
|
|
48
|
+
* every storage dir is contained under `<memoryDir>/namespaces`.
|
|
49
|
+
* - `rebuildFromDisk` rejects/reports symlinked roots that escape the memory
|
|
50
|
+
* root rather than trusting them.
|
|
51
|
+
*
|
|
52
|
+
* LIFECYCLE: catalog write failures must NEVER crash a primary memory op.
|
|
53
|
+
* Callers should wrap touch calls in try/catch (or rely on the internal
|
|
54
|
+
* failure-tolerant append). The internal serialized write chain recovers from
|
|
55
|
+
* rejection so one failed append cannot poison subsequent writes.
|
|
56
|
+
*/
|
|
57
|
+
|
|
58
|
+
export type NamespaceKind =
|
|
59
|
+
| "default"
|
|
60
|
+
| "self"
|
|
61
|
+
| "shared"
|
|
62
|
+
| "project"
|
|
63
|
+
| "branch"
|
|
64
|
+
| "team-project"
|
|
65
|
+
| "explicit"
|
|
66
|
+
| "legacy";
|
|
67
|
+
|
|
68
|
+
export type NamespaceDiscoverySource = "config" | "write" | "read" | "scan" | "migration";
|
|
69
|
+
|
|
70
|
+
export interface NamespaceRecord {
|
|
71
|
+
namespace: string;
|
|
72
|
+
identityToken: string;
|
|
73
|
+
kind: NamespaceKind;
|
|
74
|
+
principal?: string;
|
|
75
|
+
projectId?: string;
|
|
76
|
+
branch?: string;
|
|
77
|
+
parentNamespace?: string;
|
|
78
|
+
createdAt: string;
|
|
79
|
+
lastReadAt?: string;
|
|
80
|
+
lastWriteAt?: string;
|
|
81
|
+
lastMaintenanceAt?: Record<string, string>;
|
|
82
|
+
storageDir: string;
|
|
83
|
+
discoveredBy: NamespaceDiscoverySource;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface NamespaceCatalogFilter {
|
|
87
|
+
kind?: NamespaceKind;
|
|
88
|
+
discoveredBy?: NamespaceDiscoverySource;
|
|
89
|
+
/** Only include namespaces written since this instant (inclusive lower bound). */
|
|
90
|
+
writtenSince?: Date;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface NamespaceTouchMetadata {
|
|
94
|
+
discoveredBy?: NamespaceDiscoverySource;
|
|
95
|
+
kind?: NamespaceKind;
|
|
96
|
+
principal?: string;
|
|
97
|
+
projectId?: string;
|
|
98
|
+
branch?: string;
|
|
99
|
+
parentNamespace?: string;
|
|
100
|
+
/** Explicit storage dir (when the caller already resolved it). */
|
|
101
|
+
storageDir?: string;
|
|
102
|
+
/** Override the touch timestamp (mainly for tests / migration replay). */
|
|
103
|
+
at?: Date;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export interface NamespaceCatalogSkippedRoot {
|
|
107
|
+
token: string;
|
|
108
|
+
reason: "symlink" | "escape" | "unsafe" | "error";
|
|
109
|
+
detail?: string;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export interface NamespaceCatalogRebuildResult {
|
|
113
|
+
dryRun: boolean;
|
|
114
|
+
records: NamespaceRecord[];
|
|
115
|
+
/** Roots reported as ambiguous/unsafe rather than silently misclassified. */
|
|
116
|
+
skipped: NamespaceCatalogSkippedRoot[];
|
|
117
|
+
/**
|
|
118
|
+
* Whether the rebuild actually rewrote the on-disk catalog (round 6, codex P2
|
|
119
|
+
* / cursor Medium — NBn3n/NBsGG). `false` for a dry-run, AND for an `--apply`
|
|
120
|
+
* that could NOT acquire the cross-process rebuild lock within the bounded wait
|
|
121
|
+
* (it ran compute-only to avoid clobbering a concurrent lock holder). Callers
|
|
122
|
+
* (CLI) must NOT report unqualified success when `applied` is false for a
|
|
123
|
+
* non-dry-run — the catalog was left unchanged and a retry is needed.
|
|
124
|
+
*/
|
|
125
|
+
applied: boolean;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const NAMESPACE_KINDS: readonly NamespaceKind[] = [
|
|
129
|
+
"default",
|
|
130
|
+
"self",
|
|
131
|
+
"shared",
|
|
132
|
+
"project",
|
|
133
|
+
"branch",
|
|
134
|
+
"team-project",
|
|
135
|
+
"explicit",
|
|
136
|
+
"legacy",
|
|
137
|
+
];
|
|
138
|
+
|
|
139
|
+
const NAMESPACE_DISCOVERY_SOURCES: readonly NamespaceDiscoverySource[] = [
|
|
140
|
+
"config",
|
|
141
|
+
"write",
|
|
142
|
+
"read",
|
|
143
|
+
"scan",
|
|
144
|
+
"migration",
|
|
145
|
+
];
|
|
146
|
+
|
|
147
|
+
const CATALOG_FILE = "namespaces.jsonl";
|
|
148
|
+
const STATE_DIR = "state";
|
|
149
|
+
const REBUILD_LOCK_FILE = "namespaces.rebuild.lock";
|
|
150
|
+
// A held lock older than this is treated as stale (a crashed rebuild) and broken.
|
|
151
|
+
const REBUILD_LOCK_STALE_MS = 30_000;
|
|
152
|
+
// Bounded acquisition: poll briefly, then proceed best-effort rather than block
|
|
153
|
+
// a CLI rebuild forever behind a busy gateway.
|
|
154
|
+
const REBUILD_LOCK_MAX_WAIT_MS = 5_000;
|
|
155
|
+
const REBUILD_LOCK_POLL_MS = 50;
|
|
156
|
+
// Heartbeat: while a rebuild holds the lock it refreshes the lock file's mtime
|
|
157
|
+
// on this interval so a long (>STALE_MS) scan is NOT mistaken for a crashed
|
|
158
|
+
// holder and broken out from under it (round 5, cursor/codex Medium/P2). Must be
|
|
159
|
+
// comfortably below STALE_MS so at least a couple of beats land per stale window.
|
|
160
|
+
const REBUILD_LOCK_HEARTBEAT_MS = 10_000;
|
|
161
|
+
|
|
162
|
+
// Children that indicate a directory holds Remnic memory data (used for legacy
|
|
163
|
+
// default-root detection and to skip empty/non-data roots during rebuild).
|
|
164
|
+
//
|
|
165
|
+
// `state` is included to MATCH the router's storage-presence check
|
|
166
|
+
// (`NamespaceStorageRouter` counts the `state` runtime child via
|
|
167
|
+
// `includeRuntimeState: true`). Without it (round 3, cursor Medium) a namespace
|
|
168
|
+
// the router actively resolves because it has only a `state/` dir would be
|
|
169
|
+
// treated as absent by rebuild and vanish from the catalog after `--apply`.
|
|
170
|
+
const MEMORY_DATA_CHILDREN = [
|
|
171
|
+
...ALL_CATEGORY_DIRS,
|
|
172
|
+
"entities",
|
|
173
|
+
"artifacts",
|
|
174
|
+
"identity",
|
|
175
|
+
"config",
|
|
176
|
+
"summaries",
|
|
177
|
+
"profile.md",
|
|
178
|
+
"state",
|
|
179
|
+
] as const;
|
|
180
|
+
|
|
181
|
+
function isCatalogEnabled(config: PluginConfig): boolean {
|
|
182
|
+
// Inert unless namespaces are enabled. namespaceCatalogEnabled defaults to
|
|
183
|
+
// true (undefined => enabled) but is only honored when namespacesEnabled.
|
|
184
|
+
if (config.namespacesEnabled !== true) return false;
|
|
185
|
+
return (config as { namespaceCatalogEnabled?: boolean }).namespaceCatalogEnabled !== false;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Marker children that MUST be a regular file rather than a directory. Everything
|
|
189
|
+
// else in MEMORY_DATA_CHILDREN is a category/data DIRECTORY that downstream
|
|
190
|
+
// indexers (`scanMemoryDir`) read — and which they reject when it is a symlink or
|
|
191
|
+
// a non-directory. `profile.md` is the sole file marker.
|
|
192
|
+
const FILE_MEMORY_DATA_CHILDREN = new Set<string>(["profile.md"]);
|
|
193
|
+
|
|
194
|
+
type MemoryDataMarkerStatus =
|
|
195
|
+
| { state: "absent" }
|
|
196
|
+
| { state: "valid" }
|
|
197
|
+
| { state: "invalid"; detail: string };
|
|
198
|
+
|
|
199
|
+
type MemoryDataRootStatus = {
|
|
200
|
+
hasData: boolean;
|
|
201
|
+
invalidMarker?: string;
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
function isNotFoundError(err: unknown): boolean {
|
|
205
|
+
return (
|
|
206
|
+
typeof err === "object" &&
|
|
207
|
+
err !== null &&
|
|
208
|
+
"code" in err &&
|
|
209
|
+
(err as { code?: string }).code === "ENOENT"
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Inspect `child` under `rootDir` as a memory-data marker (NIw0F / PR #1506).
|
|
215
|
+
* Existence alone is not enough: a bogus marker — e.g. `facts` as a symlink or a
|
|
216
|
+
* regular file instead of a real directory — passes `lstat` but makes
|
|
217
|
+
* `scanMemoryDir` throw on the symlinked/non-directory category root. Returning
|
|
218
|
+
* a distinct `invalid` status lets root scans reject a namespace when ANY known
|
|
219
|
+
* marker is malformed, even if a sibling marker such as `state/` is valid.
|
|
220
|
+
*/
|
|
221
|
+
async function inspectMemoryDataMarker(rootDir: string, child: string): Promise<MemoryDataMarkerStatus> {
|
|
222
|
+
const childPath = path.join(rootDir, child);
|
|
223
|
+
let entry;
|
|
224
|
+
try {
|
|
225
|
+
entry = await lstat(childPath);
|
|
226
|
+
} catch (err) {
|
|
227
|
+
return isNotFoundError(err)
|
|
228
|
+
? { state: "absent" }
|
|
229
|
+
: { state: "invalid", detail: `${child}: ${err instanceof Error ? err.message : String(err)}` };
|
|
230
|
+
}
|
|
231
|
+
// Reject symlinked markers outright (scan parity — never follow them).
|
|
232
|
+
if (entry.isSymbolicLink()) return { state: "invalid", detail: `${child}: symlink` };
|
|
233
|
+
if (FILE_MEMORY_DATA_CHILDREN.has(child)) {
|
|
234
|
+
// `profile.md` must be a regular file.
|
|
235
|
+
return entry.isFile()
|
|
236
|
+
? { state: "valid" }
|
|
237
|
+
: { state: "invalid", detail: `${child}: expected file` };
|
|
238
|
+
}
|
|
239
|
+
// Category/data markers must be real directories whose realpath stays inside
|
|
240
|
+
// the namespace root (no escape via a symlinked ancestor).
|
|
241
|
+
if (!entry.isDirectory()) return { state: "invalid", detail: `${child}: expected directory` };
|
|
242
|
+
try {
|
|
243
|
+
const rootReal = await realpath(rootDir);
|
|
244
|
+
const childReal = await realpath(childPath);
|
|
245
|
+
return isPathInside(rootReal, childReal)
|
|
246
|
+
? { state: "valid" }
|
|
247
|
+
: { state: "invalid", detail: `${child}: escapes namespace root` };
|
|
248
|
+
} catch (err) {
|
|
249
|
+
return { state: "invalid", detail: `${child}: ${err instanceof Error ? err.message : String(err)}` };
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async function inspectMemoryDataRoot(rootDir: string): Promise<MemoryDataRootStatus> {
|
|
254
|
+
let hasData = false;
|
|
255
|
+
for (const child of MEMORY_DATA_CHILDREN) {
|
|
256
|
+
const marker = await inspectMemoryDataMarker(rootDir, child);
|
|
257
|
+
if (marker.state === "invalid") {
|
|
258
|
+
return { hasData: false, invalidMarker: marker.detail };
|
|
259
|
+
}
|
|
260
|
+
if (marker.state === "valid") {
|
|
261
|
+
hasData = true;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
return { hasData };
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export async function hasMemoryData(rootDir: string): Promise<boolean> {
|
|
268
|
+
return (await inspectMemoryDataRoot(rootDir)).hasData;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function isValidIsoTimestamp(value: string): boolean {
|
|
272
|
+
const ms = Date.parse(value);
|
|
273
|
+
return Number.isFinite(ms);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function isNamespaceKind(value: unknown): value is NamespaceKind {
|
|
277
|
+
return typeof value === "string" && (NAMESPACE_KINDS as readonly string[]).includes(value);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function isNamespaceDiscoverySource(value: unknown): value is NamespaceDiscoverySource {
|
|
281
|
+
return typeof value === "string" && (NAMESPACE_DISCOVERY_SOURCES as readonly string[]).includes(value);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Validate a JSONL line parsed value as a usable NamespaceRecord.
|
|
286
|
+
* Rejects null / non-object / missing-field records (CLAUDE.md rule #18).
|
|
287
|
+
* Persisted enum and timestamp fields are also validated here so a syntactically
|
|
288
|
+
* valid but tampered/pre-fix line cannot surface impossible record states.
|
|
289
|
+
*/
|
|
290
|
+
function coerceRecord(value: unknown): NamespaceRecord | null {
|
|
291
|
+
if (typeof value !== "object" || value === null) return null;
|
|
292
|
+
const v = value as Record<string, unknown>;
|
|
293
|
+
if (typeof v.namespace !== "string") return null;
|
|
294
|
+
const namespace = normalizeNamespaceIdentity(v.namespace);
|
|
295
|
+
if (namespace.length === 0) return null;
|
|
296
|
+
if (typeof v.identityToken !== "string" || v.identityToken.length === 0) return null;
|
|
297
|
+
const expectedIdentityToken = namespaceIdentityToken(namespace);
|
|
298
|
+
if (v.identityToken !== expectedIdentityToken) return null;
|
|
299
|
+
if (typeof v.storageDir !== "string" || v.storageDir.length === 0) return null;
|
|
300
|
+
if (typeof v.createdAt !== "string" || v.createdAt.length === 0) return null;
|
|
301
|
+
if (!isValidIsoTimestamp(v.createdAt)) return null;
|
|
302
|
+
const kind = v.kind === undefined ? "explicit" : isNamespaceKind(v.kind) ? v.kind : null;
|
|
303
|
+
if (!kind) return null;
|
|
304
|
+
const discoveredBy =
|
|
305
|
+
v.discoveredBy === undefined
|
|
306
|
+
? "scan"
|
|
307
|
+
: isNamespaceDiscoverySource(v.discoveredBy)
|
|
308
|
+
? v.discoveredBy
|
|
309
|
+
: null;
|
|
310
|
+
if (!discoveredBy) return null;
|
|
311
|
+
const record: NamespaceRecord = {
|
|
312
|
+
namespace,
|
|
313
|
+
identityToken: expectedIdentityToken,
|
|
314
|
+
kind,
|
|
315
|
+
createdAt: v.createdAt,
|
|
316
|
+
storageDir: v.storageDir,
|
|
317
|
+
discoveredBy,
|
|
318
|
+
};
|
|
319
|
+
if (typeof v.principal === "string") record.principal = v.principal;
|
|
320
|
+
if (typeof v.projectId === "string") record.projectId = v.projectId;
|
|
321
|
+
if (typeof v.branch === "string") record.branch = v.branch;
|
|
322
|
+
if (typeof v.parentNamespace === "string") record.parentNamespace = v.parentNamespace;
|
|
323
|
+
if (typeof v.lastReadAt === "string" && isValidIsoTimestamp(v.lastReadAt)) {
|
|
324
|
+
record.lastReadAt = v.lastReadAt;
|
|
325
|
+
}
|
|
326
|
+
if (typeof v.lastWriteAt === "string" && isValidIsoTimestamp(v.lastWriteAt)) {
|
|
327
|
+
record.lastWriteAt = v.lastWriteAt;
|
|
328
|
+
}
|
|
329
|
+
if (v.lastMaintenanceAt && typeof v.lastMaintenanceAt === "object") {
|
|
330
|
+
const out: Record<string, string> = {};
|
|
331
|
+
for (const [k, val] of Object.entries(v.lastMaintenanceAt as Record<string, unknown>)) {
|
|
332
|
+
if (typeof val === "string" && isValidIsoTimestamp(val)) out[k] = val;
|
|
333
|
+
}
|
|
334
|
+
if (Object.keys(out).length > 0) record.lastMaintenanceAt = out;
|
|
335
|
+
}
|
|
336
|
+
return record;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** Later of two optional ISO timestamps (undefined-safe). */
|
|
340
|
+
function laterIso(a: string | undefined, b: string | undefined): string | undefined {
|
|
341
|
+
if (!a) return b;
|
|
342
|
+
if (!b) return a;
|
|
343
|
+
const am = Date.parse(a);
|
|
344
|
+
const bm = Date.parse(b);
|
|
345
|
+
if (!Number.isFinite(am)) return b;
|
|
346
|
+
if (!Number.isFinite(bm)) return a;
|
|
347
|
+
return bm > am ? b : a;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Fold the touch fields (lastReadAt / lastWriteAt / lastMaintenanceAt) from a
|
|
352
|
+
* freshly re-read on-disk record into the rebuilt record, taking the LATER
|
|
353
|
+
* timestamp per field (round 5 cross-process re-merge). Disk-derived fields
|
|
354
|
+
* (storageDir, kind, discoveredBy, createdAt, principal hints) are owned by the
|
|
355
|
+
* rebuilt record and left untouched — we only recover touch recency that a
|
|
356
|
+
* concurrent (possibly cross-process) writer recorded after our initial load.
|
|
357
|
+
*/
|
|
358
|
+
function mergeNewerTouchFields(base: NamespaceRecord, fresh: NamespaceRecord): NamespaceRecord {
|
|
359
|
+
const merged: NamespaceRecord = { ...base };
|
|
360
|
+
const lr = laterIso(base.lastReadAt, fresh.lastReadAt);
|
|
361
|
+
if (lr) merged.lastReadAt = lr;
|
|
362
|
+
const lw = laterIso(base.lastWriteAt, fresh.lastWriteAt);
|
|
363
|
+
if (lw) merged.lastWriteAt = lw;
|
|
364
|
+
if (base.lastMaintenanceAt || fresh.lastMaintenanceAt) {
|
|
365
|
+
const jobs: Record<string, string> = { ...(base.lastMaintenanceAt ?? {}) };
|
|
366
|
+
for (const [job, ts] of Object.entries(fresh.lastMaintenanceAt ?? {})) {
|
|
367
|
+
const latest = laterIso(jobs[job], ts);
|
|
368
|
+
if (latest) jobs[job] = latest;
|
|
369
|
+
}
|
|
370
|
+
if (Object.keys(jobs).length > 0) merged.lastMaintenanceAt = jobs;
|
|
371
|
+
}
|
|
372
|
+
return merged;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Serialize a record with sorted keys (CLAUDE.md rule #38) so byte output is
|
|
377
|
+
* stable across runs — required for idempotent rebuilds.
|
|
378
|
+
*/
|
|
379
|
+
function serializeRecord(record: NamespaceRecord): string {
|
|
380
|
+
const ordered: Record<string, unknown> = {};
|
|
381
|
+
const source = record as unknown as Record<string, unknown>;
|
|
382
|
+
for (const key of Object.keys(source).sort()) {
|
|
383
|
+
const value = source[key];
|
|
384
|
+
if (value === undefined) continue;
|
|
385
|
+
if (key === "lastMaintenanceAt" && value && typeof value === "object") {
|
|
386
|
+
const sortedJobs: Record<string, string> = {};
|
|
387
|
+
for (const jobKey of Object.keys(value as Record<string, string>).sort()) {
|
|
388
|
+
sortedJobs[jobKey] = (value as Record<string, string>)[jobKey]!;
|
|
389
|
+
}
|
|
390
|
+
ordered[key] = sortedJobs;
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
ordered[key] = value;
|
|
394
|
+
}
|
|
395
|
+
return JSON.stringify(ordered);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Infer the namespace kind from its name/structure using the same conventions
|
|
400
|
+
* as `coding-namespace.ts` (project-*, *-branch-*, team-*-project-*). Returns
|
|
401
|
+
* `explicit` when no structural signal is present. The caller can override.
|
|
402
|
+
*/
|
|
403
|
+
function inferKind(namespace: string, config: PluginConfig): NamespaceKind {
|
|
404
|
+
// Compare against NORMALIZED config names (NGnek, codex P2): the catalog seeds
|
|
405
|
+
// normalized namespace identities, so a configured name with surrounding
|
|
406
|
+
// whitespace (e.g. `sharedNamespace: "shared "`) must still classify the
|
|
407
|
+
// normalized `"shared"` as `shared`, not fall through to `explicit`.
|
|
408
|
+
if (namespace === normalizeNamespaceIdentity(config.defaultNamespace)) return "default";
|
|
409
|
+
if (namespace === normalizeNamespaceIdentity(config.sharedNamespace)) return "shared";
|
|
410
|
+
if (config.namespacePolicies.some((p) => normalizeNamespaceIdentity(p.name) === namespace)) {
|
|
411
|
+
return "explicit";
|
|
412
|
+
}
|
|
413
|
+
// Branch overlays embed "-branch-" (project-<id>-branch-<name>).
|
|
414
|
+
if (/-branch-|^project-[^-]+-branch-/.test(namespace) || namespace.includes("-branch-")) {
|
|
415
|
+
return "branch";
|
|
416
|
+
}
|
|
417
|
+
// Team-project promotions are prefixed team-*-project-*.
|
|
418
|
+
if (/^team-.*-project-/.test(namespace) || /^team-.*project-/.test(namespace)) {
|
|
419
|
+
return "team-project";
|
|
420
|
+
}
|
|
421
|
+
// Project overlays are "project-*" or "<principal>-project-*".
|
|
422
|
+
if (/^project-/.test(namespace) || /-project-/.test(namespace)) {
|
|
423
|
+
return "project";
|
|
424
|
+
}
|
|
425
|
+
return "explicit";
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
export class NamespaceCatalog {
|
|
429
|
+
private readonly memoryDir: string;
|
|
430
|
+
private readonly stateDir: string;
|
|
431
|
+
private readonly catalogPath: string;
|
|
432
|
+
private readonly rebuildLockPath: string;
|
|
433
|
+
// Per-INSTANCE lock owner id (round 6, codex P2 — NBsGP). The rebuild lock
|
|
434
|
+
// file records this id, not just `process.pid`, so two NamespaceCatalog
|
|
435
|
+
// instances in the SAME process sharing a memoryDir are NOT mistaken for each
|
|
436
|
+
// other: a touch on instance B must still wait for instance A's rebuild lock
|
|
437
|
+
// (different owner id, same PID) instead of skipping as "self-held".
|
|
438
|
+
private readonly lockOwnerId: string = randomUUID();
|
|
439
|
+
// Serialized write chain that recovers from rejection (CLAUDE.md rule #40)
|
|
440
|
+
// so a single failed append cannot permanently poison subsequent writes.
|
|
441
|
+
private writeChain: Promise<void> = Promise.resolve();
|
|
442
|
+
// Test-only seam (round 7 — NEZkA): fires inside a touch's HELD-lock critical
|
|
443
|
+
// section, after the lock is acquired but BEFORE the read→merge→append. A
|
|
444
|
+
// deterministic concurrency test installs a hook here to widen the (otherwise
|
|
445
|
+
// microscopic) window and prove that a cross-process rebuild CANNOT run its
|
|
446
|
+
// load→rename while a touch holds the lock. Never set in production code.
|
|
447
|
+
protected onTouchCriticalSectionForTest?: () => Promise<void>;
|
|
448
|
+
// Test-only seam (round 7 — NEZkA): fires inside a mutating rebuild's HELD-lock
|
|
449
|
+
// critical section, after the final cross-process re-merge `loadCompacted()` and
|
|
450
|
+
// BEFORE the atomic `rename()`. This is the EXACT window in which a check-then-
|
|
451
|
+
// append touch (the old bug) would clobber its append. A deterministic test
|
|
452
|
+
// installs a hook here to attempt a cross-instance touch in this window and
|
|
453
|
+
// assert the held mutex blocks it. Never set in production code.
|
|
454
|
+
protected onRebuildBeforeRenameForTest?: () => Promise<void>;
|
|
455
|
+
// Test-only seam (NFgCT, codex P2): fires AFTER the lockless disk scan but
|
|
456
|
+
// BEFORE the rebuild acquires the cross-process file lock for its final
|
|
457
|
+
// load→merge→rename window. A deterministic test installs a hook here to attempt
|
|
458
|
+
// a cross-instance touch DURING the scan window and assert it is NOT blocked or
|
|
459
|
+
// dropped — proving the scan no longer holds the mutex. Never set in production.
|
|
460
|
+
protected onRebuildAfterScanForTest?: () => Promise<void>;
|
|
461
|
+
// Test-only seam (NG7Bg, codex P2): fires inside `breakStaleRebuildLock` AFTER it
|
|
462
|
+
// has judged the lock stale and captured its identity, but BEFORE the final
|
|
463
|
+
// re-validation+unlink. A deterministic test installs a hook here to REPLACE the
|
|
464
|
+
// lock file (a fresh holder created a new lock in the race window) and assert the
|
|
465
|
+
// break is skipped — the replacement's active lock is not deleted. Never set in
|
|
466
|
+
// production.
|
|
467
|
+
protected onBeforeBreakStaleUnlinkForTest?: () => Promise<void>;
|
|
468
|
+
|
|
469
|
+
// Normalized (trimmed) default namespace identity (NH-FH, cursor Medium).
|
|
470
|
+
// Catalog records key namespaces by their NORMALIZED identity
|
|
471
|
+
// (`normalizeNamespaceIdentity`), but several default-namespace exemptions and
|
|
472
|
+
// memoryDir-ownership checks compared against the RAW `config.defaultNamespace`.
|
|
473
|
+
// If the configured default name carries surrounding whitespace the record key
|
|
474
|
+
// is trimmed while the comparison string is not, so the default row is
|
|
475
|
+
// misclassified, dropped at read time, or given the wrong storage root. Compare
|
|
476
|
+
// against this normalized form everywhere instead.
|
|
477
|
+
private readonly defaultNamespaceIdentity: string;
|
|
478
|
+
|
|
479
|
+
constructor(private readonly config: PluginConfig) {
|
|
480
|
+
this.memoryDir = config.memoryDir;
|
|
481
|
+
this.stateDir = path.join(this.memoryDir, STATE_DIR);
|
|
482
|
+
this.catalogPath = path.join(this.stateDir, CATALOG_FILE);
|
|
483
|
+
this.rebuildLockPath = path.join(this.stateDir, REBUILD_LOCK_FILE);
|
|
484
|
+
this.defaultNamespaceIdentity = normalizeNamespaceIdentity(config.defaultNamespace);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** Whether the catalog is active (namespaces enabled and catalog not opted out). */
|
|
488
|
+
get enabled(): boolean {
|
|
489
|
+
return isCatalogEnabled(this.config);
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
// ── Public enumeration API ──────────────────────────────────────────────
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* Sanitize a record at the enumeration boundary (round 5, cursor Medium + codex
|
|
496
|
+
* P2; round 6 — NDXHe). Reads return whatever is in `namespaces.jsonl` after
|
|
497
|
+
* schema checks only, so a tampered or pre-fix row could surface unsafe data to
|
|
498
|
+
* maintenance/QMD until a rewrite occurs. Two distinct defenses:
|
|
499
|
+
*
|
|
500
|
+
* 1. UNSAFE NAMESPACE NAME (NGZqr, codex P2): an unsafe non-default namespace
|
|
501
|
+
* (e.g. `../evil`, a name with separators, or >64 chars) is REJECTED outright
|
|
502
|
+
* — return `null` so the caller drops it. The disk SCAN and the hot touch
|
|
503
|
+
* path both reject such names with the SAME default-exempt `isSafeRouteNamespace`
|
|
504
|
+
* gate, so the read boundary MUST agree, or `listNamespaces()`/`getNamespaceRecord()`
|
|
505
|
+
* would expose a namespace those paths reject (note `isStorageDirForNamespace`
|
|
506
|
+
* can still build a tokenized root even for `../evil`, so storageDir sanitation
|
|
507
|
+
* alone does not catch it). The default namespace is exempt (it may be a
|
|
508
|
+
* non-route literal), matching every other validation site.
|
|
509
|
+
*
|
|
510
|
+
* 2. UNSAFE storageDir: for an otherwise-valid namespace, apply the SAME contract
|
|
511
|
+
* as the write path — full containment (`isContainedStorageDir`: lexical +
|
|
512
|
+
* symlink/realpath) AND namespace ownership (`isStorageDirForNamespace`). When
|
|
513
|
+
* a record fails EITHER check we substitute the trusted resolved-and-safe root
|
|
514
|
+
* for that namespace (rule 42: read and write stay symmetric).
|
|
515
|
+
*/
|
|
516
|
+
private async sanitizeRecordForRead(record: NamespaceRecord): Promise<NamespaceRecord | null> {
|
|
517
|
+
// Defense 1: drop an unsafe non-default namespace name entirely. Compare
|
|
518
|
+
// against the NORMALIZED default identity — record keys are trimmed, so a raw
|
|
519
|
+
// whitespace-padded config default would never match the default row (NH-FH).
|
|
520
|
+
if (record.namespace !== this.defaultNamespaceIdentity && !isSafeRouteNamespace(record.namespace)) {
|
|
521
|
+
return null;
|
|
522
|
+
}
|
|
523
|
+
// Defense 2: keep the record but substitute a safe storageDir when needed.
|
|
524
|
+
if (
|
|
525
|
+
(await this.isContainedStorageDir(record.storageDir)) &&
|
|
526
|
+
(await this.isStorageDirForNamespace(record.namespace, record.storageDir))
|
|
527
|
+
) {
|
|
528
|
+
return record;
|
|
529
|
+
}
|
|
530
|
+
const safe = await this.resolveSafeStorageDir(record.namespace);
|
|
531
|
+
return { ...record, storageDir: safe };
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
private storageRootOwnershipRank(
|
|
535
|
+
record: NamespaceRecord,
|
|
536
|
+
resolvedStorageDir: string,
|
|
537
|
+
configured: Set<string>,
|
|
538
|
+
): number {
|
|
539
|
+
if (resolvedStorageDir === path.resolve(this.memoryDir)) {
|
|
540
|
+
return record.namespace === this.defaultNamespaceIdentity ? 0 : 3;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
const leaf = path.basename(resolvedStorageDir);
|
|
544
|
+
const tokenOwnsRoot = namespaceIdentityToken(record.namespace) === leaf;
|
|
545
|
+
if (tokenOwnsRoot && configured.has(record.namespace)) {
|
|
546
|
+
return 0;
|
|
547
|
+
}
|
|
548
|
+
if (record.namespace === leaf) return 1;
|
|
549
|
+
if (tokenOwnsRoot) return 2;
|
|
550
|
+
return 3;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
private configuredNamespaceIdentities(): Set<string> {
|
|
554
|
+
return new Set(
|
|
555
|
+
[
|
|
556
|
+
this.config.defaultNamespace,
|
|
557
|
+
this.config.sharedNamespace,
|
|
558
|
+
...this.config.namespacePolicies.map((p) => p.name),
|
|
559
|
+
]
|
|
560
|
+
.map((n) => normalizeNamespaceIdentity(n))
|
|
561
|
+
.filter((n) => n.length > 0),
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
private preferStorageRootOwner(
|
|
566
|
+
current: NamespaceRecord,
|
|
567
|
+
candidate: NamespaceRecord,
|
|
568
|
+
resolvedStorageDir: string,
|
|
569
|
+
configured: Set<string>,
|
|
570
|
+
): NamespaceRecord {
|
|
571
|
+
const currentRank = this.storageRootOwnershipRank(current, resolvedStorageDir, configured);
|
|
572
|
+
const candidateRank = this.storageRootOwnershipRank(candidate, resolvedStorageDir, configured);
|
|
573
|
+
if (candidateRank < currentRank) return candidate;
|
|
574
|
+
if (candidateRank > currentRank) return current;
|
|
575
|
+
|
|
576
|
+
const byName = candidate.namespace.localeCompare(current.namespace);
|
|
577
|
+
if (byName < 0) return candidate;
|
|
578
|
+
if (byName > 0) return current;
|
|
579
|
+
return candidate.identityToken.localeCompare(current.identityToken) < 0 ? candidate : current;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
private dropDuplicateStorageRootAliases(records: NamespaceRecord[]): NamespaceRecord[] {
|
|
583
|
+
const byStorageDir = new Map<string, NamespaceRecord>();
|
|
584
|
+
const configured = this.configuredNamespaceIdentities();
|
|
585
|
+
for (const record of records) {
|
|
586
|
+
const resolvedStorageDir = path.resolve(record.storageDir);
|
|
587
|
+
const current = byStorageDir.get(resolvedStorageDir);
|
|
588
|
+
if (!current) {
|
|
589
|
+
byStorageDir.set(resolvedStorageDir, record);
|
|
590
|
+
continue;
|
|
591
|
+
}
|
|
592
|
+
const owner = this.preferStorageRootOwner(current, record, resolvedStorageDir, configured);
|
|
593
|
+
const alias = owner === current ? record : current;
|
|
594
|
+
byStorageDir.set(resolvedStorageDir, mergeNewerTouchFields(owner, alias));
|
|
595
|
+
}
|
|
596
|
+
return [...byStorageDir.values()];
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
private async loadSanitizedRecords(): Promise<NamespaceRecord[]> {
|
|
600
|
+
const records = await this.loadCompacted();
|
|
601
|
+
const sanitized = await Promise.all(
|
|
602
|
+
[...records.values()].map((r) => this.sanitizeRecordForRead(r)),
|
|
603
|
+
);
|
|
604
|
+
// Drop unsafe-namespace rows (sanitizer returned null) at the read boundary.
|
|
605
|
+
// Then collapse duplicate root aliases so maintenance/QMD see exactly one
|
|
606
|
+
// namespace owner for a physical storage root, matching rebuild ownership,
|
|
607
|
+
// while preserving touch recency from every alias row.
|
|
608
|
+
return this.dropDuplicateStorageRootAliases(
|
|
609
|
+
sanitized.filter((r): r is NamespaceRecord => r !== null),
|
|
610
|
+
);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
async listNamespaces(filter?: NamespaceCatalogFilter): Promise<NamespaceRecord[]> {
|
|
614
|
+
if (!this.enabled) return [];
|
|
615
|
+
let out = await this.loadSanitizedRecords();
|
|
616
|
+
if (filter?.kind) out = out.filter((r) => r.kind === filter.kind);
|
|
617
|
+
if (filter?.discoveredBy) out = out.filter((r) => r.discoveredBy === filter.discoveredBy);
|
|
618
|
+
if (filter?.writtenSince) {
|
|
619
|
+
const sinceMs = filter.writtenSince.getTime();
|
|
620
|
+
out = out.filter((r) => {
|
|
621
|
+
if (!r.lastWriteAt) return false;
|
|
622
|
+
const ms = Date.parse(r.lastWriteAt);
|
|
623
|
+
return Number.isFinite(ms) && ms >= sinceMs;
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
// Stable sort: namespace asc, identityToken as deterministic tiebreaker
|
|
627
|
+
// (CLAUDE.md rule #19 — comparator returns 0 only for truly-equal items).
|
|
628
|
+
return out.sort((a, b) => {
|
|
629
|
+
const byName = a.namespace.localeCompare(b.namespace);
|
|
630
|
+
if (byName !== 0) return byName;
|
|
631
|
+
return a.identityToken.localeCompare(b.identityToken);
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
async getNamespaceRecord(namespace: string): Promise<NamespaceRecord | null> {
|
|
636
|
+
if (!this.enabled) return null;
|
|
637
|
+
const ns = normalizeNamespaceIdentity(namespace);
|
|
638
|
+
return (await this.loadSanitizedRecords()).find((record) => record.namespace === ns) ?? null;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
// ── Touch API (cheap, failure-tolerant) ─────────────────────────────────
|
|
642
|
+
|
|
643
|
+
async markRead(namespace: string, metadata?: NamespaceTouchMetadata): Promise<void> {
|
|
644
|
+
await this.touch(namespace, "read", metadata);
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
async markWrite(namespace: string, metadata?: NamespaceTouchMetadata): Promise<void> {
|
|
648
|
+
await this.touch(namespace, "write", metadata);
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
async markMaintenance(namespace: string, jobName: string, at?: Date): Promise<void> {
|
|
652
|
+
if (typeof jobName !== "string" || jobName.trim().length === 0) {
|
|
653
|
+
throw new Error("markMaintenance requires a non-empty jobName");
|
|
654
|
+
}
|
|
655
|
+
await this.touch(namespace, "maintenance", { at }, jobName.trim());
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/**
|
|
659
|
+
* Register namespaces known purely from config (default, shared, explicit
|
|
660
|
+
* policies). Source `config`. Cheap and idempotent.
|
|
661
|
+
*/
|
|
662
|
+
async registerConfiguredNamespaces(): Promise<void> {
|
|
663
|
+
if (!this.enabled) return;
|
|
664
|
+
const names = new Set<string>([
|
|
665
|
+
this.config.defaultNamespace,
|
|
666
|
+
this.config.sharedNamespace,
|
|
667
|
+
...this.config.namespacePolicies.map((p) => p.name),
|
|
668
|
+
]);
|
|
669
|
+
for (const ns of names) {
|
|
670
|
+
if (!ns) continue;
|
|
671
|
+
// Skip unsafe configured names (e.g. a `sharedNamespace`/policy name like
|
|
672
|
+
// `../evil`) consistently with `rebuildFromDisk` (round 6, cursor Low —
|
|
673
|
+
// NBn3w). `register`→`validateNamespace` THROWS on unsafe tokens; without
|
|
674
|
+
// this guard one bad name would abort registration of all the rest. The
|
|
675
|
+
// default namespace is exempt (it may be a non-route literal). Each call is
|
|
676
|
+
// also wrapped so a single failure never blocks the remaining names.
|
|
677
|
+
// `names` carries RAW config values, so normalize before the default-exempt
|
|
678
|
+
// check — a whitespace-padded default must still be recognized (NH-FH).
|
|
679
|
+
if (normalizeNamespaceIdentity(ns) !== this.defaultNamespaceIdentity && !isSafeRouteNamespace(ns)) {
|
|
680
|
+
continue;
|
|
681
|
+
}
|
|
682
|
+
try {
|
|
683
|
+
await this.register(ns, { discoveredBy: "config" });
|
|
684
|
+
} catch {
|
|
685
|
+
// Best-effort: a single bad/unsafe name must not abort the batch.
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* Register a namespace whose storage was just resolved by the router. Used as
|
|
692
|
+
* the router's integration hook (`discoveredBy: config`). Storage dir is
|
|
693
|
+
* provided so we do not re-resolve it. Failure-tolerant. Returns whether the
|
|
694
|
+
* registration actually APPENDED (round 6, codex P2 — NEFoX), so the router's
|
|
695
|
+
* resolve-hook dedup only marks a namespace notified when it truly persisted —
|
|
696
|
+
* a dropped append (disabled catalog or rebuild-lock-timeout drop) returns
|
|
697
|
+
* `false` and is retried on the next resolve.
|
|
698
|
+
*/
|
|
699
|
+
async registerResolved(namespace: string, storageDir: string): Promise<boolean> {
|
|
700
|
+
if (!this.enabled) return false;
|
|
701
|
+
return this.register(namespace, { discoveredBy: "config", storageDir });
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
/**
|
|
705
|
+
* Generic register/touch without changing read/write timestamps unless the
|
|
706
|
+
* source implies it. Validates the namespace and resolves a storage dir.
|
|
707
|
+
* Returns whether the touch actually appended.
|
|
708
|
+
*/
|
|
709
|
+
private async register(namespace: string, metadata: NamespaceTouchMetadata): Promise<boolean> {
|
|
710
|
+
return this.touch(namespace, "register", metadata);
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
private validateNamespace(namespace: string): string {
|
|
714
|
+
const ns = normalizeNamespaceIdentity(namespace);
|
|
715
|
+
if (ns.length === 0) throw new Error("empty namespace");
|
|
716
|
+
// The configured default namespace is exempt from isSafeRouteNamespace at
|
|
717
|
+
// the routing layer; honor the same exemption here, but everything still
|
|
718
|
+
// resolves through the contained storage-dir helper below.
|
|
719
|
+
if (ns !== this.defaultNamespaceIdentity && !isSafeRouteNamespace(ns)) {
|
|
720
|
+
throw new Error(`unsafe namespace: ${ns}`);
|
|
721
|
+
}
|
|
722
|
+
return ns;
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* Resolve the on-disk storage dir for a namespace WITHOUT trusting caller
|
|
727
|
+
* input. The default namespace may use the legacy memoryDir root; everything
|
|
728
|
+
* else lives under `<memoryDir>/namespaces/<token>`. Containment is enforced
|
|
729
|
+
* by rejecting separators/parent-refs in the token.
|
|
730
|
+
*/
|
|
731
|
+
private resolveStorageDir(namespace: string): string {
|
|
732
|
+
if (normalizeNamespaceIdentity(namespace) === this.defaultNamespaceIdentity) {
|
|
733
|
+
// Default may resolve to the legacy memoryDir root OR a tokenized dir; we
|
|
734
|
+
// report memoryDir here as the canonical default root for the catalog.
|
|
735
|
+
// rebuildFromDisk refines this when a tokenized default dir holds data.
|
|
736
|
+
return this.memoryDir;
|
|
737
|
+
}
|
|
738
|
+
const token = namespaceIdentityToken(namespace);
|
|
739
|
+
return this.namespaceTokenDir(token);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
private namespaceTokenDir(token: string): string {
|
|
743
|
+
if (
|
|
744
|
+
token.length === 0 ||
|
|
745
|
+
token.includes("/") ||
|
|
746
|
+
token.includes("\\") ||
|
|
747
|
+
token.includes("..") ||
|
|
748
|
+
path.isAbsolute(token)
|
|
749
|
+
) {
|
|
750
|
+
throw new Error(`unsafe namespace token: ${token}`);
|
|
751
|
+
}
|
|
752
|
+
return path.join(this.memoryDir, "namespaces", token);
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
/**
|
|
756
|
+
* Whether a candidate storage dir is LEXICALLY contained: it is either the
|
|
757
|
+
* legacy default root (`memoryDir`) or a strict descendant of
|
|
758
|
+
* `<memoryDir>/namespaces/`. The router legitimately resolves a namespace to
|
|
759
|
+
* EITHER the tokenized dir or a legacy raw-name dir under `namespaces/`, so we
|
|
760
|
+
* accept any contained child rather than a single exact token path. This is a
|
|
761
|
+
* pure string check — symlink escape is checked separately via realpath.
|
|
762
|
+
*/
|
|
763
|
+
private isLexicallyContained(candidate: string): boolean {
|
|
764
|
+
const resolved = path.resolve(candidate);
|
|
765
|
+
if (resolved === path.resolve(this.memoryDir)) return true;
|
|
766
|
+
const nsBase = path.resolve(path.join(this.memoryDir, "namespaces"));
|
|
767
|
+
const rel = path.relative(nsBase, resolved);
|
|
768
|
+
// Must be a strict descendant of namespaces/ (non-empty, no parent escape).
|
|
769
|
+
return rel.length > 0 && !rel.startsWith("..") && !path.isAbsolute(rel);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
/**
|
|
773
|
+
* Whether a candidate storage dir satisfies the catalog containment contract,
|
|
774
|
+
* including SYMLINK-escape rejection (round 5, codex P2). A lexically-contained
|
|
775
|
+
* path that is actually a symlink to an outside directory would let maintenance
|
|
776
|
+
* or QMD follow it outside `memoryDir`. We mirror `rebuildFromDisk`'s posture:
|
|
777
|
+
* the path must be lexically contained AND, if it exists on disk, neither the
|
|
778
|
+
* path itself a symlink nor its realpath escaping the memory root. Non-existent
|
|
779
|
+
* paths pass the realpath stage (nothing to follow yet) but still must be
|
|
780
|
+
* lexically contained.
|
|
781
|
+
*/
|
|
782
|
+
private async isContainedStorageDir(candidate: string): Promise<boolean> {
|
|
783
|
+
if (!this.isLexicallyContained(candidate)) return false;
|
|
784
|
+
// The default/legacy memoryDir root is trusted as-is.
|
|
785
|
+
if (path.resolve(candidate) === path.resolve(this.memoryDir)) return true;
|
|
786
|
+
let memoryReal: string;
|
|
787
|
+
try {
|
|
788
|
+
memoryReal = await realpath(this.memoryDir);
|
|
789
|
+
} catch {
|
|
790
|
+
memoryReal = path.resolve(this.memoryDir);
|
|
791
|
+
}
|
|
792
|
+
// Reject a candidate beneath any SYMLINKED ancestor (codex NVuq5): even when
|
|
793
|
+
// the symlink currently resolves back inside memoryDir, the disk scanner
|
|
794
|
+
// rejects such a root, and a later retarget of the link would let
|
|
795
|
+
// maintenance/QMD follow the persisted path outside memoryDir. Mirror the
|
|
796
|
+
// scanner so touch/config seeding cannot persist a root under a symlinked
|
|
797
|
+
// namespace ancestor (the leaf itself is symlink-checked below).
|
|
798
|
+
if (await this.hasSymlinkedAncestor(candidate)) return false;
|
|
799
|
+
try {
|
|
800
|
+
const stat = await lstat(candidate);
|
|
801
|
+
if (stat.isSymbolicLink()) return false;
|
|
802
|
+
// Reject an EXISTING non-directory root (NF21i, codex P2; CLAUDE.md rule
|
|
803
|
+
// #24). A regular file (or socket/fifo) at `<memoryDir>/namespaces/<token>`
|
|
804
|
+
// is lexically contained and its realpath stays inside memoryDir, so the
|
|
805
|
+
// realpath check below would ACCEPT it — but a storage root must be a
|
|
806
|
+
// directory. Recording a file as a namespace root yields a broken install
|
|
807
|
+
// that only fails later when maintenance/QMD/mkdir treat it as a dir. The
|
|
808
|
+
// disk scan already skips non-directory entries; mirror that here so every
|
|
809
|
+
// containment consumer (resolve/touch/fallback/live-recheck) agrees.
|
|
810
|
+
if (!stat.isDirectory()) return false;
|
|
811
|
+
} catch {
|
|
812
|
+
// The leaf does not exist yet. Lexical containment is NOT sufficient: an
|
|
813
|
+
// EXISTING ancestor (e.g. `<memoryDir>/namespaces`) could be a symlink to
|
|
814
|
+
// outside memoryDir, so a future mkdir/maintenance/QMD op would follow the
|
|
815
|
+
// persisted root outside the root (round 6, codex P2 — NDo79). Verify the
|
|
816
|
+
// nearest EXISTING ancestor's realpath still resolves inside memoryDir.
|
|
817
|
+
return this.isNearestExistingAncestorContained(candidate, memoryReal);
|
|
818
|
+
}
|
|
819
|
+
try {
|
|
820
|
+
const real = await realpath(candidate);
|
|
821
|
+
return isPathInside(memoryReal, real);
|
|
822
|
+
} catch {
|
|
823
|
+
return false;
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
/**
|
|
828
|
+
* Reject a candidate whose path crosses a SYMLINKED ancestor strictly between
|
|
829
|
+
* memoryDir and the leaf (codex NVuq5). `realpath`-based containment accepts a
|
|
830
|
+
* symlinked `<memoryDir>/namespaces` that currently resolves back inside
|
|
831
|
+
* memoryDir, but the disk scanner rejects such a root and a later retarget would
|
|
832
|
+
* escape the memory tree — so refuse it here too. The leaf itself is
|
|
833
|
+
* symlink-checked by the caller; this walks only the intermediate ancestors.
|
|
834
|
+
*/
|
|
835
|
+
private async hasSymlinkedAncestor(candidate: string): Promise<boolean> {
|
|
836
|
+
const stopAt = path.resolve(this.memoryDir);
|
|
837
|
+
let dir = path.dirname(path.resolve(candidate));
|
|
838
|
+
const root = path.parse(dir).root;
|
|
839
|
+
while (dir !== stopAt && dir !== root && dir !== path.dirname(dir)) {
|
|
840
|
+
try {
|
|
841
|
+
if ((await lstat(dir)).isSymbolicLink()) return true;
|
|
842
|
+
} catch {
|
|
843
|
+
// Ancestor does not exist yet — it cannot be a symlink; keep walking up.
|
|
844
|
+
}
|
|
845
|
+
dir = path.dirname(dir);
|
|
846
|
+
}
|
|
847
|
+
return false;
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Walk up from a not-yet-existing candidate to the nearest ancestor that exists
|
|
852
|
+
* on disk and verify its realpath stays inside `memoryReal` (round 6, codex P2
|
|
853
|
+
* — NDo79). Rejects a non-existent leaf whose existing parent chain escapes
|
|
854
|
+
* memoryDir via a symlink. Stops at memoryDir's resolved root.
|
|
855
|
+
*
|
|
856
|
+
* The nearest existing ancestor must also be a DIRECTORY (NHIdt, codex P2): if
|
|
857
|
+
* an existing parent such as `<memoryDir>/namespaces` is a regular FILE (or
|
|
858
|
+
* socket/fifo), `realpath(parent)` still succeeds and resolves inside memoryDir,
|
|
859
|
+
* so a containment-only check would ACCEPT a leaf that can never be created — you
|
|
860
|
+
* cannot mkdir a child under a file. We `lstat` the nearest existing ancestor and
|
|
861
|
+
* reject when it is not a directory, mirroring the leaf non-directory rejection
|
|
862
|
+
* (NF21i) and the disk scan, so every containment consumer agrees.
|
|
863
|
+
*/
|
|
864
|
+
private async isNearestExistingAncestorContained(
|
|
865
|
+
candidate: string,
|
|
866
|
+
memoryReal: string,
|
|
867
|
+
): Promise<boolean> {
|
|
868
|
+
let dir = path.resolve(candidate);
|
|
869
|
+
const root = path.parse(dir).root;
|
|
870
|
+
for (;;) {
|
|
871
|
+
const parent = path.dirname(dir);
|
|
872
|
+
// Reached the filesystem root without finding an existing ancestor.
|
|
873
|
+
if (parent === dir || dir === root) return false;
|
|
874
|
+
let real: string;
|
|
875
|
+
try {
|
|
876
|
+
real = await realpath(parent);
|
|
877
|
+
} catch {
|
|
878
|
+
// Parent does not exist yet either — keep walking up.
|
|
879
|
+
dir = parent;
|
|
880
|
+
continue;
|
|
881
|
+
}
|
|
882
|
+
// The nearest EXISTING ancestor must resolve inside the memory root...
|
|
883
|
+
if (!(isPathInside(memoryReal, real) || real === memoryReal)) return false;
|
|
884
|
+
// ...AND be a directory: a non-directory ancestor (e.g. a file occupying
|
|
885
|
+
// `namespaces`) cannot hold the not-yet-created leaf (NHIdt).
|
|
886
|
+
try {
|
|
887
|
+
const stat = await lstat(real);
|
|
888
|
+
return stat.isDirectory();
|
|
889
|
+
} catch {
|
|
890
|
+
// The ancestor vanished between realpath and lstat — treat as not usable.
|
|
891
|
+
return false;
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
/**
|
|
897
|
+
* Resolve the storage dir to persist for a touch, validating any caller-
|
|
898
|
+
* provided `metadata.storageDir` against the catalog containment contract
|
|
899
|
+
* (round 4 + round 5, codex P2). `markWrite`/`registerResolved` accept an
|
|
900
|
+
* explicit storageDir, but persisting it verbatim would let a bad hook or
|
|
901
|
+
* external consumer write an arbitrary path — including one outside `memoryDir`
|
|
902
|
+
* or a symlink that escapes it — into the catalog, handing maintenance/QMD an
|
|
903
|
+
* unsafe root. We accept an explicit (or previously-stored) dir ONLY when it
|
|
904
|
+
* stays contained under memoryDir (lexically AND via realpath); otherwise we
|
|
905
|
+
* drop it and fall back to the trusted resolved dir.
|
|
906
|
+
*/
|
|
907
|
+
private async resolveTouchStorageDir(
|
|
908
|
+
namespace: string,
|
|
909
|
+
explicit: string | undefined,
|
|
910
|
+
existingDir: string | undefined,
|
|
911
|
+
): Promise<string> {
|
|
912
|
+
// An explicit storageDir is accepted ONLY when it is both contained AND
|
|
913
|
+
// actually belongs to THIS namespace (round 6, codex P2 — NDATT). Containment
|
|
914
|
+
// alone let a caller pass another namespace's tree (e.g.
|
|
915
|
+
// `markWrite("project-a", { storageDir: ".../namespaces/<project-b-token>" })`)
|
|
916
|
+
// or `memoryDir` for a non-default namespace; `listNamespaces()` would then
|
|
917
|
+
// tell maintenance/QMD that `project-a` lives in another namespace's (or the
|
|
918
|
+
// default) tree — a cross-namespace root confusion. We reject a mismatched
|
|
919
|
+
// explicit root and fall back to the namespace's own resolved root.
|
|
920
|
+
if (
|
|
921
|
+
explicit !== undefined &&
|
|
922
|
+
(await this.isContainedStorageDir(explicit)) &&
|
|
923
|
+
(await this.isStorageDirForNamespace(namespace, explicit))
|
|
924
|
+
) {
|
|
925
|
+
return explicit;
|
|
926
|
+
}
|
|
927
|
+
// Don't let a record poisoned by a pre-fix out-of-containment write keep an
|
|
928
|
+
// unsafe dir alive across touches — only preserve a contained existing dir
|
|
929
|
+
// that also belongs to this namespace.
|
|
930
|
+
if (
|
|
931
|
+
existingDir !== undefined &&
|
|
932
|
+
(await this.isContainedStorageDir(existingDir)) &&
|
|
933
|
+
(await this.isStorageDirForNamespace(namespace, existingDir))
|
|
934
|
+
) {
|
|
935
|
+
return existingDir;
|
|
936
|
+
}
|
|
937
|
+
return this.resolveSafeStorageDir(namespace);
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
/**
|
|
941
|
+
* Whether `candidate` is a legitimate storage root FOR `namespace` (round 6,
|
|
942
|
+
* codex P2 — NDATT). Accepts the namespace's router-resolved root, its canonical
|
|
943
|
+
* lexical tokenized dir, and (for the default namespace only) memoryDir. This
|
|
944
|
+
* prevents a contained-but-CROSS-NAMESPACE path — another namespace's tree, or
|
|
945
|
+
* memoryDir for a non-default namespace — from being persisted as this
|
|
946
|
+
* namespace's root. Compared on resolved (absolute) paths.
|
|
947
|
+
*/
|
|
948
|
+
private async isStorageDirForNamespace(namespace: string, candidate: string): Promise<boolean> {
|
|
949
|
+
const resolvedCandidate = path.resolve(candidate);
|
|
950
|
+
const valid = new Set<string>();
|
|
951
|
+
// The namespace's canonical lexical TOKENIZED dir is always a valid root.
|
|
952
|
+
try {
|
|
953
|
+
valid.add(path.resolve(this.namespaceTokenDir(namespaceIdentityToken(namespace))));
|
|
954
|
+
} catch {
|
|
955
|
+
// Unsafe token cannot build a lexical dir; fall through to other roots.
|
|
956
|
+
}
|
|
957
|
+
// The namespace's legacy RAW-NAME dir (`namespaces/<rawname>`) is also a
|
|
958
|
+
// valid root — the router serves data from it when present, even before any
|
|
959
|
+
// dir exists on disk. Both forms belong to THIS namespace, never another's.
|
|
960
|
+
try {
|
|
961
|
+
valid.add(path.resolve(this.namespaceTokenDir(namespace)));
|
|
962
|
+
} catch {
|
|
963
|
+
// Unsafe raw name cannot build a lexical dir; rely on the other roots.
|
|
964
|
+
}
|
|
965
|
+
// The router-resolved root (whichever of the above it currently serves, a
|
|
966
|
+
// migrated default, etc.).
|
|
967
|
+
try {
|
|
968
|
+
valid.add(path.resolve(await resolveNamespaceStorageRoot(this.config, namespace)));
|
|
969
|
+
} catch {
|
|
970
|
+
// Router resolution failed; rely on the lexical/default roots below.
|
|
971
|
+
}
|
|
972
|
+
// memoryDir is a valid root ONLY for the default namespace.
|
|
973
|
+
if (normalizeNamespaceIdentity(namespace) === this.defaultNamespaceIdentity) {
|
|
974
|
+
valid.add(path.resolve(this.memoryDir));
|
|
975
|
+
try {
|
|
976
|
+
valid.add(path.resolve(await resolveDefaultNamespaceRoot(this.config)));
|
|
977
|
+
} catch {
|
|
978
|
+
// ignore; memoryDir already covers the common default case.
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
return valid.has(resolvedCandidate);
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
/**
|
|
985
|
+
* Resolve the canonical storage dir for a namespace as the LIVE ROUTER would,
|
|
986
|
+
* but NEVER return a path that escapes the memory root.
|
|
987
|
+
*
|
|
988
|
+
* Router alignment (round 4, cursor Medium): a read/register touch with no
|
|
989
|
+
* explicit storageDir previously used the lexical `resolveStorageDir`, which
|
|
990
|
+
* always picks `<memoryDir>/namespaces/<token>` (or `memoryDir` for the
|
|
991
|
+
* default). That diverges from `NamespaceStorageRouter`, which can route to a
|
|
992
|
+
* legacy raw-name dir or a migrated default root — so a recall touch could
|
|
993
|
+
* record a contained-but-WRONG root that maintenance/rebuild then targets. We
|
|
994
|
+
* now delegate to the shared `resolveNamespaceStorageRoot` (the very helper the
|
|
995
|
+
* router uses) so the catalog records the same on-disk root the router serves.
|
|
996
|
+
*
|
|
997
|
+
* Containment (round 5, codex P2): the resolved path can still be a symlink
|
|
998
|
+
* escaping memoryDir, so we run the full (lexical + realpath) containment
|
|
999
|
+
* contract. When it FAILS we fall back to a NAMESPACE-SPECIFIC safe root, NOT
|
|
1000
|
+
* a blanket `memoryDir`. Recording `memoryDir` for a non-default namespace
|
|
1001
|
+
* would point enumeration/maintenance at the DEFAULT namespace's tree (round 5,
|
|
1002
|
+
* cursor/codex Medium/P2) — a cross-namespace fanout error. The correct safe
|
|
1003
|
+
* root is the namespace's own lexical tokenized dir
|
|
1004
|
+
* (`<memoryDir>/namespaces/<token>`), which is always contained and is that
|
|
1005
|
+
* namespace's canonical location (we record the lexical PATH as metadata; we do
|
|
1006
|
+
* not follow the escaping symlink). Only the default namespace — or a token so
|
|
1007
|
+
* unsafe even the lexical dir cannot be built — falls back to `memoryDir`.
|
|
1008
|
+
*/
|
|
1009
|
+
private async resolveSafeStorageDir(namespace: string): Promise<string> {
|
|
1010
|
+
let resolved: string;
|
|
1011
|
+
try {
|
|
1012
|
+
resolved = await resolveNamespaceStorageRoot(this.config, namespace);
|
|
1013
|
+
} catch {
|
|
1014
|
+
return this.safeFallbackStorageDir(namespace);
|
|
1015
|
+
}
|
|
1016
|
+
if (await this.isContainedStorageDir(resolved)) return resolved;
|
|
1017
|
+
return this.safeFallbackStorageDir(namespace);
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/**
|
|
1021
|
+
* The namespace-specific contained fallback root, used when the router-resolved
|
|
1022
|
+
* root fails containment (round 5, cursor/codex Medium/P2).
|
|
1023
|
+
*
|
|
1024
|
+
* Preference order:
|
|
1025
|
+
* 1. The namespace's OWN lexical tokenized dir (`namespaces/<token>`) — so a
|
|
1026
|
+
* non-default namespace is NOT pointed at the DEFAULT namespace's `memoryDir`
|
|
1027
|
+
* tree (which would misdirect maintenance fanout). Returned only when the
|
|
1028
|
+
* token dir itself stays CONTAINED (it is not a symlink, and its realpath
|
|
1029
|
+
* does not escape memoryDir — e.g. via a symlinked `namespaces/` parent).
|
|
1030
|
+
* 2. `memoryDir` as a LAST resort — for the default namespace, an unsafe token
|
|
1031
|
+
* that cannot build a contained path, OR the irreparable case where the
|
|
1032
|
+
* token dir's realpath escapes the root (so even its lexical path resolves
|
|
1033
|
+
* outside). NF21m note (codex P2): we deliberately do NOT record the lexical
|
|
1034
|
+
* token dir in that irreparable case — its realpath escapes memoryDir, and
|
|
1035
|
+
* the NDo79 contract REQUIRES that an escaping path is never persisted (a
|
|
1036
|
+
* later mkdir/maintenance/QMD op would follow it outside the root). Since no
|
|
1037
|
+
* contained namespace-specific path exists, containment wins: `memoryDir` is
|
|
1038
|
+
* the only safe root left. A namespace whose token dir's realpath escapes is
|
|
1039
|
+
* an irreparable on-disk state; recording the contained default root is
|
|
1040
|
+
* strictly safer than persisting an escaping one. The common case where the
|
|
1041
|
+
* token dir IS contained is handled by branch 1, so a healthy non-default
|
|
1042
|
+
* namespace never reaches `memoryDir`.
|
|
1043
|
+
*/
|
|
1044
|
+
private async safeFallbackStorageDir(namespace: string): Promise<string> {
|
|
1045
|
+
if (normalizeNamespaceIdentity(namespace) === this.defaultNamespaceIdentity) return this.memoryDir;
|
|
1046
|
+
let tokenDir: string;
|
|
1047
|
+
try {
|
|
1048
|
+
tokenDir = this.namespaceTokenDir(namespaceIdentityToken(namespace));
|
|
1049
|
+
} catch {
|
|
1050
|
+
return this.memoryDir;
|
|
1051
|
+
}
|
|
1052
|
+
if (await this.isContainedStorageDir(tokenDir)) return tokenDir;
|
|
1053
|
+
return this.memoryDir;
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* Re-check, NOW, whether a namespace's storage root currently EXISTS on disk
|
|
1058
|
+
* with the SAME safety the directory scan uses (NFJV8, codex P2).
|
|
1059
|
+
*
|
|
1060
|
+
* The rebuild's final re-merge runs under the held lock and folds the freshly
|
|
1061
|
+
* re-read log (`latest`) into the scanned `rebuilt` set. A namespace present in
|
|
1062
|
+
* `latest` (a live touch row) but ABSENT from `rebuilt` is normally PURGED as
|
|
1063
|
+
* deleted (the NATqU "disk scan is authoritative" rule). But there is a TOCTOU
|
|
1064
|
+
* window: a dynamic namespace can be CREATED on disk AFTER `rebuildFromDisk()`
|
|
1065
|
+
* already enumerated `namespaces/` but BEFORE this re-merge. The scan snapshot
|
|
1066
|
+
* missed its new root, yet a gateway `markWrite` already appended a row for it.
|
|
1067
|
+
* Blindly purging that row would rewrite the catalog WITHOUT a live namespace
|
|
1068
|
+
* that now has data on disk, so `writtenSince`/maintenance/QMD consumers miss
|
|
1069
|
+
* it until another touch or rebuild.
|
|
1070
|
+
*
|
|
1071
|
+
* So before purging, we re-resolve the namespace's safe storage root (the same
|
|
1072
|
+
* router-aligned, containment-checked path the scan would have catalogued) and
|
|
1073
|
+
* confirm it is a real, contained, non-symlink directory that actually holds
|
|
1074
|
+
* memory data RIGHT NOW. If so the namespace was created-after-scan and is LIVE
|
|
1075
|
+
* — KEEP its row. This is the precise inverse of NATqU and does NOT reintroduce
|
|
1076
|
+
* it: a touch on a REMOVED root re-checks as ABSENT (no data on disk) and is
|
|
1077
|
+
* still purged; only a root that EXISTS on a fresh re-check is kept.
|
|
1078
|
+
*
|
|
1079
|
+
* Mirrors the per-entry scan checks (symlink rejection + realpath containment +
|
|
1080
|
+
* `hasMemoryData`) so a symlinked/escaping root is never resurrected.
|
|
1081
|
+
*/
|
|
1082
|
+
private async liveStorageRootExistsForRebuild(
|
|
1083
|
+
namespace: string,
|
|
1084
|
+
memoryReal: string | null,
|
|
1085
|
+
): Promise<boolean> {
|
|
1086
|
+
let root: string;
|
|
1087
|
+
try {
|
|
1088
|
+
// Use the SAME router-aligned, containment-enforcing resolver the catalog
|
|
1089
|
+
// uses everywhere else. It never returns an escaping path (falls back to a
|
|
1090
|
+
// namespace-specific contained root on containment failure).
|
|
1091
|
+
root = await this.resolveSafeStorageDir(namespace);
|
|
1092
|
+
} catch {
|
|
1093
|
+
return false;
|
|
1094
|
+
}
|
|
1095
|
+
// NH3Xy (codex P2): for a NON-default namespace, a generic fallback root is
|
|
1096
|
+
// NOT proof of liveness. When the namespace's own token root was skipped by
|
|
1097
|
+
// the scan as a symlink/escape, `resolveSafeStorageDir` can fall back to the
|
|
1098
|
+
// DEFAULT namespace's `memoryDir`; `hasMemoryData()` on that shared default
|
|
1099
|
+
// tree then returns true whenever the default namespace has any data, which
|
|
1100
|
+
// would wrongly KEEP a stale project row now pointing at the default tree
|
|
1101
|
+
// instead of purging the skipped namespace. Only the namespace's OWN root may
|
|
1102
|
+
// attest its liveness — so if a non-default namespace resolved to `memoryDir`,
|
|
1103
|
+
// it has no independent contained root and must be treated as absent (purge).
|
|
1104
|
+
if (
|
|
1105
|
+
normalizeNamespaceIdentity(namespace) !== this.defaultNamespaceIdentity &&
|
|
1106
|
+
path.resolve(root) === path.resolve(this.memoryDir)
|
|
1107
|
+
) {
|
|
1108
|
+
return false;
|
|
1109
|
+
}
|
|
1110
|
+
let stat;
|
|
1111
|
+
try {
|
|
1112
|
+
stat = await lstat(root);
|
|
1113
|
+
} catch {
|
|
1114
|
+
// Root does not exist on disk → genuinely absent → allow the purge.
|
|
1115
|
+
return false;
|
|
1116
|
+
}
|
|
1117
|
+
// Reject a symlinked root rather than resurrecting it (scan parity).
|
|
1118
|
+
if (stat.isSymbolicLink()) return false;
|
|
1119
|
+
if (!stat.isDirectory()) return false;
|
|
1120
|
+
// Realpath must stay inside the memory root (scan parity).
|
|
1121
|
+
try {
|
|
1122
|
+
const real = await realpath(root);
|
|
1123
|
+
if (memoryReal && !isPathInside(memoryReal, real)) return false;
|
|
1124
|
+
} catch {
|
|
1125
|
+
return false;
|
|
1126
|
+
}
|
|
1127
|
+
// Only treat the root as a live namespace when it actually holds memory data,
|
|
1128
|
+
// exactly as the scan does (empty shells are not catalogued).
|
|
1129
|
+
return hasMemoryData(root);
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
/**
|
|
1133
|
+
* Record a namespace touch. Returns whether the touch actually APPENDED to the
|
|
1134
|
+
* log (round 6, codex P2 — NEFoX): a disabled catalog or a dropped append (the
|
|
1135
|
+
* NAUf7 rebuild-lock-timeout drop) returns `false`, so callers (e.g. the router
|
|
1136
|
+
* resolve-hook dedup) can avoid marking a dropped registration as completed and
|
|
1137
|
+
* suppressing its retry.
|
|
1138
|
+
*/
|
|
1139
|
+
private async touch(
|
|
1140
|
+
namespace: string,
|
|
1141
|
+
kind: "read" | "write" | "maintenance" | "register",
|
|
1142
|
+
metadata?: NamespaceTouchMetadata,
|
|
1143
|
+
jobName?: string,
|
|
1144
|
+
): Promise<boolean> {
|
|
1145
|
+
if (!this.enabled) return false;
|
|
1146
|
+
// Validate up front (outside the chain) so caller-facing rejections — e.g.
|
|
1147
|
+
// an unsafe namespace token — surface immediately and deterministically,
|
|
1148
|
+
// not interleaved with serialized I/O.
|
|
1149
|
+
const ns = this.validateNamespace(namespace);
|
|
1150
|
+
const nowIso = (metadata?.at ?? new Date()).toISOString();
|
|
1151
|
+
|
|
1152
|
+
// Run the read → merge → append as a single serialized critical section so
|
|
1153
|
+
// two concurrent touches for the same namespace cannot both observe the same
|
|
1154
|
+
// stale record and then have the later append win compaction while dropping
|
|
1155
|
+
// the earlier touch's fields (CLAUDE.md rule #40 — the chain also recovers
|
|
1156
|
+
// from rejection). Reading inside the chain guarantees each touch sees the
|
|
1157
|
+
// most recent appended state, including any concurrent read/write/register.
|
|
1158
|
+
// Cross-process serialization (round 7, codex P2 — NEZkA: HELD MUTEX). A CLI
|
|
1159
|
+
// `rebuild --apply` holds the rebuild lock across its final `loadCompacted()`
|
|
1160
|
+
// → atomic `rename`. Previously a touch only POLLED (`waitForRebuildLockClear`)
|
|
1161
|
+
// for the lock before reading/appending WITHOUT holding it — a check-then-act
|
|
1162
|
+
// gap: a touch could see no lock, a rebuild could then acquire the lock + run
|
|
1163
|
+
// its final `loadCompacted()`, and the touch's later append would be clobbered
|
|
1164
|
+
// by the rebuild's `rename()`. We now make the touch HOLD the SAME advisory
|
|
1165
|
+
// lock for the WHOLE read → merge → append window. While the touch holds the
|
|
1166
|
+
// lock, a rebuild in another process blocks on it (and vice-versa), so no
|
|
1167
|
+
// append can land between a rebuild's final load and its rename. `queueCritical`
|
|
1168
|
+
// serializes this within ONE process (so the OS lock is never self-contended in
|
|
1169
|
+
// process); the file lock adds the missing CROSS-process exclusion. If the touch
|
|
1170
|
+
// cannot ACQUIRE the lock within the bounded wait (another process's rebuild is
|
|
1171
|
+
// mid-flight), it DROPS the append: the catalog is rebuildable best-effort
|
|
1172
|
+
// metadata, so skipping one touch is acceptable; it NEVER blocks forever, NEVER
|
|
1173
|
+
// appends without the lock, and NEVER crashes the primary memory op.
|
|
1174
|
+
return this.queueCritical(async () =>
|
|
1175
|
+
this.withHeldCatalogLock(async (acquired) => {
|
|
1176
|
+
// Could not hold the lock (a cross-process rebuild is in its load→rename
|
|
1177
|
+
// window). DROP rather than append into that window (the lost-append race
|
|
1178
|
+
// this lock exists to prevent). Returning false also lets the router's
|
|
1179
|
+
// resolve-hook dedup retry a dropped registration later.
|
|
1180
|
+
if (!acquired) return false;
|
|
1181
|
+
|
|
1182
|
+
// Test-only seam: widen the held-lock window so a concurrency test can
|
|
1183
|
+
// attempt a cross-process rebuild here and assert it is BLOCKED by this
|
|
1184
|
+
// held lock (no-op in production).
|
|
1185
|
+
if (this.onTouchCriticalSectionForTest) {
|
|
1186
|
+
await this.onTouchCriticalSectionForTest();
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
const records = await this.loadCompacted();
|
|
1190
|
+
const existing = records.get(ns);
|
|
1191
|
+
|
|
1192
|
+
// Containment-check any explicit storageDir before persisting it (round 4
|
|
1193
|
+
// + round 5, codex P2). Never trust a caller-provided path verbatim;
|
|
1194
|
+
// reject lexical escapes AND symlinks that escape via realpath.
|
|
1195
|
+
const storageDir = await this.resolveTouchStorageDir(
|
|
1196
|
+
ns,
|
|
1197
|
+
metadata?.storageDir,
|
|
1198
|
+
existing?.storageDir,
|
|
1199
|
+
);
|
|
1200
|
+
// Provenance (discoveredBy) and createdAt are CREATION-ONLY fields. Once a
|
|
1201
|
+
// record exists they are preserved, so a routine routing/recall touch (or
|
|
1202
|
+
// the router's `config` register hook firing on a cache hit) can never
|
|
1203
|
+
// clobber the original discovery source — e.g. a `write`-discovered record
|
|
1204
|
+
// is not reset to `config` by a later resolve. Touch fields (lastReadAt /
|
|
1205
|
+
// lastWriteAt / lastMaintenanceAt) still update on every touch below.
|
|
1206
|
+
const record: NamespaceRecord = existing
|
|
1207
|
+
? { ...existing }
|
|
1208
|
+
: {
|
|
1209
|
+
namespace: ns,
|
|
1210
|
+
identityToken: namespaceIdentityToken(ns),
|
|
1211
|
+
kind: metadata?.kind ?? inferKind(ns, this.config),
|
|
1212
|
+
createdAt: nowIso,
|
|
1213
|
+
storageDir,
|
|
1214
|
+
discoveredBy:
|
|
1215
|
+
metadata?.discoveredBy ??
|
|
1216
|
+
(kind === "register" ? "config" : kind === "maintenance" ? "scan" : kind),
|
|
1217
|
+
};
|
|
1218
|
+
|
|
1219
|
+
// Update mutable fields. storageDir, kind, and the principal/project hints
|
|
1220
|
+
// may legitimately change over a namespace's lifetime, so they upsert.
|
|
1221
|
+
record.storageDir = storageDir;
|
|
1222
|
+
if (metadata?.kind) record.kind = metadata.kind;
|
|
1223
|
+
if (metadata?.principal !== undefined) record.principal = metadata.principal;
|
|
1224
|
+
if (metadata?.projectId !== undefined) record.projectId = metadata.projectId;
|
|
1225
|
+
if (metadata?.branch !== undefined) record.branch = metadata.branch;
|
|
1226
|
+
if (metadata?.parentNamespace !== undefined)
|
|
1227
|
+
record.parentNamespace = metadata.parentNamespace;
|
|
1228
|
+
// PROVENANCE (creation-only, with one upgrade — round 6, codex P2 NBPmT):
|
|
1229
|
+
// `discoveredBy` is otherwise preserved for existing records (a routine
|
|
1230
|
+
// read/register/resolve never relabels it). The single exception is a real
|
|
1231
|
+
// WRITE upgrading a record that was only PRE-REGISTERED by the router's
|
|
1232
|
+
// `onResolve` hook (`discoveredBy: "config"`) before any data was written.
|
|
1233
|
+
// Without this upgrade, `listNamespaces({ discoveredBy: "write" })` misses
|
|
1234
|
+
// namespaces that were genuinely written, because `storageFor()` fires
|
|
1235
|
+
// `registerResolved()` (config) before `recordCatalogWrite()` runs. We
|
|
1236
|
+
// upgrade ONLY config→write — never downgrade write/read, never relabel a
|
|
1237
|
+
// read-discovered record — so the authoritative "this namespace has been
|
|
1238
|
+
// written" signal is recorded.
|
|
1239
|
+
if (kind === "write" && existing && record.discoveredBy === "config") {
|
|
1240
|
+
record.discoveredBy = "write";
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
if (kind === "read") record.lastReadAt = nowIso;
|
|
1244
|
+
if (kind === "write") record.lastWriteAt = nowIso;
|
|
1245
|
+
if (kind === "maintenance" && jobName) {
|
|
1246
|
+
record.lastMaintenanceAt = { ...(record.lastMaintenanceAt ?? {}), [jobName]: nowIso };
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
await this.appendUnchained(record);
|
|
1250
|
+
return true;
|
|
1251
|
+
}),
|
|
1252
|
+
);
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
// ── Rebuild from disk ────────────────────────────────────────────────────
|
|
1256
|
+
|
|
1257
|
+
async rebuildFromDisk(
|
|
1258
|
+
options?: { dryRun?: boolean },
|
|
1259
|
+
): Promise<NamespaceCatalogRebuildResult> {
|
|
1260
|
+
const dryRun = options?.dryRun === true;
|
|
1261
|
+
if (!this.enabled) {
|
|
1262
|
+
return { dryRun, records: [], skipped: [], applied: false };
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
// CONCURRENCY (Issue A — round 2): the entire scan → merge → rewrite runs
|
|
1266
|
+
// inside ONE serialized critical section on the shared write chain. This
|
|
1267
|
+
// closes the round-1 residual risk where a hot-path markRead/markWrite/
|
|
1268
|
+
// registerResolved append could land AFTER the snapshot but BEFORE the
|
|
1269
|
+
// atomic rewrite and then be discarded by the rewrite. Because touches also
|
|
1270
|
+
// run through `queueCritical`, no append can interleave between the load
|
|
1271
|
+
// (which now reads the latest persisted state, including touches that
|
|
1272
|
+
// landed before this section started) and the rewrite. A `--dry-run` still
|
|
1273
|
+
// takes the section for a consistent read but performs no mutation.
|
|
1274
|
+
//
|
|
1275
|
+
// Deadlock note: the rewrite inside this section uses the unchained
|
|
1276
|
+
// `rewriteUnchained` helper (mirroring `appendUnchained`) rather than a
|
|
1277
|
+
// helper that re-enters `queueCritical` — re-entering the chain from inside
|
|
1278
|
+
// a held turn would await the very entry this section holds.
|
|
1279
|
+
//
|
|
1280
|
+
// CROSS-PROCESS (round 5, codex P2): `queueCritical` only serializes this
|
|
1281
|
+
// process's instance. A CLI `rebuild --apply` and the live gateway are
|
|
1282
|
+
// SEPARATE processes with independent write chains, so a gateway append can
|
|
1283
|
+
// still land between the CLI's load and its atomic rename. For the mutating
|
|
1284
|
+
// path we additionally take a cross-process file lock AND re-merge the latest
|
|
1285
|
+
// on-disk touches under that lock immediately before the rewrite (see
|
|
1286
|
+
// `rebuildInsideChain`). A dry-run never mutates, so it skips the lock.
|
|
1287
|
+
if (dryRun) {
|
|
1288
|
+
return this.queueCritical(async () => this.rebuildInsideChain(dryRun, false));
|
|
1289
|
+
}
|
|
1290
|
+
// A mutating rebuild HOLDS the same advisory lock that touches now hold (round
|
|
1291
|
+
// 7, codex P2 — NEZkA). Because the touch path acquires this lock across its
|
|
1292
|
+
// read→append window and the rebuild holds it across its final
|
|
1293
|
+
// `loadCompacted()` → `rename()`, the two are mutually exclusive cross-process:
|
|
1294
|
+
// no touch append can land between a rebuild's final load and its rename.
|
|
1295
|
+
//
|
|
1296
|
+
// SCOPED MUTEX (NFgCT, codex P2): the lock is acquired ONLY around the final
|
|
1297
|
+
// load→merge→rename window, NOT the (potentially long) disk scan. The scan does
|
|
1298
|
+
// not mutate, so holding the lock across it merely forces concurrent gateway
|
|
1299
|
+
// touches to wait — and they DROP their append after `REBUILD_LOCK_MAX_WAIT_MS`,
|
|
1300
|
+
// losing real `lastWriteAt`/new-namespace data the rewrite then misses. Keeping
|
|
1301
|
+
// the scan lockless shrinks the window in which a touch must contend with the
|
|
1302
|
+
// rebuild to just the final critical section, which is brief. `rebuildInsideChain`
|
|
1303
|
+
// acquires `withHeldCatalogLock` itself, immediately before its re-merge+rewrite.
|
|
1304
|
+
//
|
|
1305
|
+
// LOCK ORDERING (round 7 — NEZkA): the file lock is acquired INSIDE
|
|
1306
|
+
// `queueCritical`, identically to the touch path (`queueCritical` → file lock),
|
|
1307
|
+
// NOT around it. A consistent acquire order is what prevents an in-process
|
|
1308
|
+
// deadlock between a same-instance touch and rebuild: `queueCritical` fully
|
|
1309
|
+
// serializes the two turns in this process, so when one turn holds the file
|
|
1310
|
+
// lock the other is not even running — the OS lock is never self-contended
|
|
1311
|
+
// in-process and a same-instance touch never stalls/drops behind its own
|
|
1312
|
+
// rebuild. The file lock therefore adds ONLY the missing cross-process
|
|
1313
|
+
// exclusion. `rebuildInsideChain` still runs entirely inside `queueCritical`;
|
|
1314
|
+
// it just narrows the cross-process file lock to the final rewrite window.
|
|
1315
|
+
return this.queueCritical(async () => this.rebuildInsideChain(dryRun, true));
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
/**
|
|
1319
|
+
* Body of `rebuildFromDisk`, run inside a single `queueCritical` turn. MUST
|
|
1320
|
+
* only be invoked from within the serialized chain so the load and the
|
|
1321
|
+
* rewrite are atomic with respect to concurrent touches (in-process).
|
|
1322
|
+
*
|
|
1323
|
+
* `wantMutate` is true for an `--apply` (the caller intends to rewrite). The
|
|
1324
|
+
* cross-process file lock is acquired LATE — only around the final
|
|
1325
|
+
* load→merge→rename window (NFgCT, codex P2) — never across the disk scan, so a
|
|
1326
|
+
* long scan does not force concurrent gateway touches to wait (and drop their
|
|
1327
|
+
* append). Whether the rewrite actually happened is reported via the result's
|
|
1328
|
+
* `applied`: true only when `wantMutate` AND the lock was acquired.
|
|
1329
|
+
*/
|
|
1330
|
+
private async rebuildInsideChain(
|
|
1331
|
+
dryRun: boolean,
|
|
1332
|
+
wantMutate: boolean,
|
|
1333
|
+
): Promise<NamespaceCatalogRebuildResult> {
|
|
1334
|
+
// Read the LATEST persisted state inside the chain so any touch that landed
|
|
1335
|
+
// before this turn is folded in (and re-merged into the rewrite below).
|
|
1336
|
+
const existing = await this.loadCompacted();
|
|
1337
|
+
const skipped: NamespaceCatalogSkippedRoot[] = [];
|
|
1338
|
+
const rebuilt = new Map<string, NamespaceRecord>();
|
|
1339
|
+
const nowIso = new Date().toISOString();
|
|
1340
|
+
|
|
1341
|
+
let memoryReal: string | null = null;
|
|
1342
|
+
try {
|
|
1343
|
+
memoryReal = await realpath(this.memoryDir);
|
|
1344
|
+
} catch {
|
|
1345
|
+
memoryReal = this.memoryDir;
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
// 1) Configured namespaces always belong in the catalog.
|
|
1349
|
+
//
|
|
1350
|
+
// NORMALIZE FIRST (NGnek, codex P2): the live router normalizes every namespace
|
|
1351
|
+
// via `normalizeNamespaceIdentity` (a trim) in `storageFor()` before resolving
|
|
1352
|
+
// storage, and `isSafeRouteNamespace` also trims before validating. So a
|
|
1353
|
+
// configured name with harmless surrounding whitespace (e.g.
|
|
1354
|
+
// `sharedNamespace: "shared "` or a policy name copied with a trailing space)
|
|
1355
|
+
// would otherwise seed a catalog row for the RAW string and resolve a
|
|
1356
|
+
// `namespaces/shared ` root the live reads/writes never use — pointing
|
|
1357
|
+
// maintenance/QMD at the wrong directory after `rebuild --apply`. We normalize
|
|
1358
|
+
// configured names here so the catalog seeds the SAME identity the router uses
|
|
1359
|
+
// (rule #42: read/write resolve through the same normalization). The default
|
|
1360
|
+
// namespace is normalized too and compared via its normalized form (`defaultNs`)
|
|
1361
|
+
// wherever a configured/scanned name is matched against it below.
|
|
1362
|
+
const defaultNs = normalizeNamespaceIdentity(this.config.defaultNamespace);
|
|
1363
|
+
const configured = new Set<string>(
|
|
1364
|
+
[
|
|
1365
|
+
this.config.defaultNamespace,
|
|
1366
|
+
this.config.sharedNamespace,
|
|
1367
|
+
...this.config.namespacePolicies.map((p) => p.name),
|
|
1368
|
+
]
|
|
1369
|
+
.map((n) => normalizeNamespaceIdentity(n))
|
|
1370
|
+
.filter((n) => n.length > 0),
|
|
1371
|
+
);
|
|
1372
|
+
|
|
1373
|
+
// 2) Default-root alignment (Issue C — round 2): the catalog's default
|
|
1374
|
+
// record MUST point at the SAME root the runtime router resolves, or
|
|
1375
|
+
// maintenance/QMD consumers would read a different default root than
|
|
1376
|
+
// live reads. We delegate to the shared `resolveDefaultNamespaceRoot`
|
|
1377
|
+
// (the very helper the router uses) instead of reimplementing divergent
|
|
1378
|
+
// "prefer tokenized dir if it has data" logic — while legacy data lives
|
|
1379
|
+
// directly under memoryDir, this returns memoryDir, matching runtime.
|
|
1380
|
+
const resolvedDefaultRoot = await resolveDefaultNamespaceRoot(this.config);
|
|
1381
|
+
// CONTAINMENT (round 6, codex P2 — NEOFS): `resolveDefaultNamespaceRoot()` can
|
|
1382
|
+
// return a `namespaces/<default-token>` symlink escaping memoryDir when the
|
|
1383
|
+
// legacy default root is empty. The default record must never carry an
|
|
1384
|
+
// escaping `storageDir`; fall back to the trusted `memoryDir` root when the
|
|
1385
|
+
// resolved one fails containment. Computed ONCE so every later use (the
|
|
1386
|
+
// configured-seeding step and the scan's default-dir re-apply) stays safe.
|
|
1387
|
+
const defaultStorageDir = (await this.isContainedStorageDir(resolvedDefaultRoot))
|
|
1388
|
+
? resolvedDefaultRoot
|
|
1389
|
+
: this.memoryDir;
|
|
1390
|
+
const legacyDefaultHasData = defaultStorageDir === this.memoryDir;
|
|
1391
|
+
|
|
1392
|
+
for (const ns of configured) {
|
|
1393
|
+
if (!ns) continue;
|
|
1394
|
+
// SAFETY (round 6, codex P2 — NBPmO): `parseConfig` intentionally preserves
|
|
1395
|
+
// unsafe namespace strings (e.g. a `sharedNamespace`/`namespacePolicies[]`
|
|
1396
|
+
// name like `../evil`) so sinks reject them. The hot touch/scan paths
|
|
1397
|
+
// already reject via `isSafeRouteNamespace`; rebuild must NOT be the path
|
|
1398
|
+
// that admits an unsafe configured namespace into the catalog. The default
|
|
1399
|
+
// namespace is exempt (it may be a non-route literal), matching the scan
|
|
1400
|
+
// loop's exemption below.
|
|
1401
|
+
if (ns !== defaultNs && !isSafeRouteNamespace(ns)) {
|
|
1402
|
+
let token: string;
|
|
1403
|
+
try {
|
|
1404
|
+
token = namespaceIdentityToken(ns);
|
|
1405
|
+
} catch {
|
|
1406
|
+
token = ns;
|
|
1407
|
+
}
|
|
1408
|
+
skipped.push({ token, reason: "unsafe", detail: ns });
|
|
1409
|
+
continue;
|
|
1410
|
+
}
|
|
1411
|
+
// ROUTER ALIGNMENT (round 6, codex P2 — NDxiS): seed a configured
|
|
1412
|
+
// non-default namespace with the SAME root the runtime router resolves, not
|
|
1413
|
+
// a blanket tokenized dir. `resolveNamespaceStorageRoot` returns the legacy
|
|
1414
|
+
// RAW root when it exists and only prefers the tokenized root when that has
|
|
1415
|
+
// storage markers — so a configured namespace with an empty legacy raw root
|
|
1416
|
+
// (e.g. `namespaces/shared`) is catalogued at the runtime path, keeping
|
|
1417
|
+
// maintenance/QMD aligned with live reads. Falls back to the lexical token
|
|
1418
|
+
// dir if router resolution fails.
|
|
1419
|
+
let storageDir: string;
|
|
1420
|
+
if (ns === defaultNs) {
|
|
1421
|
+
storageDir = defaultStorageDir;
|
|
1422
|
+
} else {
|
|
1423
|
+
try {
|
|
1424
|
+
storageDir = await resolveNamespaceStorageRoot(this.config, ns);
|
|
1425
|
+
} catch {
|
|
1426
|
+
storageDir = this.namespaceTokenDir(namespaceIdentityToken(ns));
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
// CONTAINMENT (round 6, codex P2 — NCzT4/NEOFS): verify the seeded path does
|
|
1430
|
+
// not ESCAPE memoryDir before recording it. The scan below rejects
|
|
1431
|
+
// escaping/symlinked roots, but this seeding runs FIRST, so without this
|
|
1432
|
+
// check rebuild would persist an escaping `storageDir`. `isContainedStorageDir`
|
|
1433
|
+
// enforces the full lexical + symlink + realpath contract and allows a
|
|
1434
|
+
// not-yet-created path (a brand-new configured namespace seeds its canonical
|
|
1435
|
+
// root). The DEFAULT namespace is also checked (NEOFS): if
|
|
1436
|
+
// `resolveDefaultNamespaceRoot()` returns a `namespaces/<default-token>`
|
|
1437
|
+
// symlink escaping memoryDir, we must NOT persist it. The default cannot be
|
|
1438
|
+
// "skipped" (it must always exist), so it falls back to the trusted
|
|
1439
|
+
// `memoryDir` root; a non-default namespace is skipped (escape).
|
|
1440
|
+
if (!(await this.isContainedStorageDir(storageDir))) {
|
|
1441
|
+
if (ns === defaultNs) {
|
|
1442
|
+
storageDir = this.memoryDir;
|
|
1443
|
+
} else {
|
|
1444
|
+
skipped.push({ token: namespaceIdentityToken(ns), reason: "escape", detail: storageDir });
|
|
1445
|
+
continue;
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
rebuilt.set(
|
|
1449
|
+
ns,
|
|
1450
|
+
this.mergeForRebuild(existing.get(ns), {
|
|
1451
|
+
namespace: ns,
|
|
1452
|
+
identityToken: namespaceIdentityToken(ns),
|
|
1453
|
+
kind: inferKind(ns, this.config),
|
|
1454
|
+
createdAt: existing.get(ns)?.createdAt ?? nowIso,
|
|
1455
|
+
storageDir,
|
|
1456
|
+
discoveredBy: "config",
|
|
1457
|
+
}),
|
|
1458
|
+
);
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
// 3) Scan the namespaces/ directory for tokenized roots.
|
|
1462
|
+
const namespacesDir = path.join(this.memoryDir, "namespaces");
|
|
1463
|
+
let entries: Dirent[] = [];
|
|
1464
|
+
// CONTAINMENT (round 8, codex P2 — NE9K_): check the `namespaces` ROOT itself
|
|
1465
|
+
// BEFORE `readdir` follows it. If `<memoryDir>/namespaces` is a symlink (or its
|
|
1466
|
+
// realpath escapes memoryDir), `readdir()` would enumerate an arbitrary outside
|
|
1467
|
+
// tree — leaking names or spending time on a huge directory — even though the
|
|
1468
|
+
// catalog rejects symlinked/escaping per-entry roots. The per-entry lstat/realpath
|
|
1469
|
+
// checks below run AFTER the readdir, so they cannot prevent following an
|
|
1470
|
+
// escaping ROOT. We lstat the root: if it is a symlink, OR its realpath escapes
|
|
1471
|
+
// memoryDir, we DO NOT read it and report it as a single unsafe scan root.
|
|
1472
|
+
let namespacesDirSafe = true;
|
|
1473
|
+
try {
|
|
1474
|
+
const rootStat = await lstat(namespacesDir);
|
|
1475
|
+
if (rootStat.isSymbolicLink()) {
|
|
1476
|
+
namespacesDirSafe = false;
|
|
1477
|
+
} else {
|
|
1478
|
+
const realNamespacesDir = await realpath(namespacesDir);
|
|
1479
|
+
if (memoryReal && !isPathInside(memoryReal, realNamespacesDir)) {
|
|
1480
|
+
namespacesDirSafe = false;
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
} catch {
|
|
1484
|
+
// The `namespaces` dir does not exist yet (or lstat failed): nothing to scan,
|
|
1485
|
+
// and there is no symlink to follow. Treat as an empty, safe scan.
|
|
1486
|
+
namespacesDirSafe = true;
|
|
1487
|
+
}
|
|
1488
|
+
if (!namespacesDirSafe) {
|
|
1489
|
+
skipped.push({ token: "namespaces", reason: "symlink", detail: namespacesDir });
|
|
1490
|
+
} else {
|
|
1491
|
+
try {
|
|
1492
|
+
entries = await readdir(namespacesDir, { withFileTypes: true });
|
|
1493
|
+
} catch {
|
|
1494
|
+
entries = [];
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
// Dual-root alignment (round 5, cursor Medium): when both a legacy raw-name
|
|
1499
|
+
// dir and a tokenized dir hold data for the SAME namespace, the router
|
|
1500
|
+
// prefers the tokenized root. Track which scanned namespaces were already
|
|
1501
|
+
// sourced from their tokenized dir so a later legacy-named `readdir` entry
|
|
1502
|
+
// cannot overwrite the tokenized record (and vice-versa: a tokenized entry
|
|
1503
|
+
// always wins over a previously-set legacy one).
|
|
1504
|
+
const scannedFromTokenized = new Set<string>();
|
|
1505
|
+
|
|
1506
|
+
for (const entry of entries) {
|
|
1507
|
+
const token = entry.name;
|
|
1508
|
+
const fullPath = path.join(namespacesDir, token);
|
|
1509
|
+
// Reject symlinks / escaping roots rather than trusting them.
|
|
1510
|
+
let stat;
|
|
1511
|
+
try {
|
|
1512
|
+
stat = await lstat(fullPath);
|
|
1513
|
+
} catch (err) {
|
|
1514
|
+
skipped.push({ token, reason: "error", detail: err instanceof Error ? err.message : String(err) });
|
|
1515
|
+
continue;
|
|
1516
|
+
}
|
|
1517
|
+
if (stat.isSymbolicLink()) {
|
|
1518
|
+
skipped.push({ token, reason: "symlink", detail: fullPath });
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
if (!stat.isDirectory()) continue;
|
|
1522
|
+
// Containment: realpath must stay inside the memory root.
|
|
1523
|
+
try {
|
|
1524
|
+
const real = await realpath(fullPath);
|
|
1525
|
+
if (memoryReal && !isPathInside(memoryReal, real)) {
|
|
1526
|
+
skipped.push({ token, reason: "escape", detail: real });
|
|
1527
|
+
continue;
|
|
1528
|
+
}
|
|
1529
|
+
} catch (err) {
|
|
1530
|
+
skipped.push({ token, reason: "error", detail: err instanceof Error ? err.message : String(err) });
|
|
1531
|
+
continue;
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
// Decode the namespace from the dir name. A configured dir name is used
|
|
1535
|
+
// verbatim. Otherwise decode a genuine tokenized dir back to its identity,
|
|
1536
|
+
// falling back to the raw dir name when it is not a decodable token.
|
|
1537
|
+
//
|
|
1538
|
+
// NDATN note (round 6, codex P2): a raw dir literally named like a CANONICAL
|
|
1539
|
+
// token (e.g. `namespaces/ns-616c706861`, the canonical token of `alpha`) is
|
|
1540
|
+
// inherently ambiguous from disk alone — the bytes are identical whether the
|
|
1541
|
+
// namespace is `alpha` (in its tokenized dir) or the literal `ns-616c706861`
|
|
1542
|
+
// (in a raw dir). Decoding a canonical token is the correct default. The
|
|
1543
|
+
// unambiguous fix lives on the WRITE path, where the caller knows the true
|
|
1544
|
+
// namespace and records it verbatim (NCQI0); the scanner cannot recover a
|
|
1545
|
+
// name the encoding cannot distinguish, so we keep the canonical decode.
|
|
1546
|
+
//
|
|
1547
|
+
// NRcCD (round 9, codex P2 — same class as namespaceFromStorageDir/NRCve):
|
|
1548
|
+
// the canonical decode is WRONG when a namespace LITERALLY named like the
|
|
1549
|
+
// token already OWNS this root. A dynamic namespace served from a legacy raw
|
|
1550
|
+
// root `namespaces/ns-616c706861` (named verbatim `ns-616c706861`) records a
|
|
1551
|
+
// catalog row from the write path; that row is in `existing` (the prior
|
|
1552
|
+
// load) here. If we still decoded to `alpha`, this scan would emit an `alpha`
|
|
1553
|
+
// row at `fullPath`, and the final live-row remerge in `finishRebuild` would
|
|
1554
|
+
// re-add the literal `ns-616c706861` row (its root still has data) — leaving
|
|
1555
|
+
// TWO catalog rows at the SAME `storageDir`, fanning QMD/maintenance out under
|
|
1556
|
+
// the wrong namespace. So, mirroring `namespaceFromStorageDir`'s "config/catalog
|
|
1557
|
+
// match before decode" rule, prefer the LITERAL dir name when it is already a
|
|
1558
|
+
// KNOWN namespace — configured OR present as a live/cataloged row in `existing`
|
|
1559
|
+
// — and DO NOT also emit the decoded alias for that same root. A genuine
|
|
1560
|
+
// tokenized dir with no literal owner (no `existing` row keyed by the raw
|
|
1561
|
+
// token) still decodes as before.
|
|
1562
|
+
// Root ownership (codex r3499938974): preserving the literal must be
|
|
1563
|
+
// ROOT-based, not just key-based. A STALE cataloged row merely NAMED like
|
|
1564
|
+
// the token (but whose storageDir is NOT this `fullPath`) must NOT win — a
|
|
1565
|
+
// real dynamic `alpha` write served from this tokenized root would then be
|
|
1566
|
+
// rebuilt under the stale literal name and the fresh `alpha` row dropped by
|
|
1567
|
+
// the owned-by-other guard. So only prefer the literal when a CONFIGURED
|
|
1568
|
+
// name matches OR an existing cataloged row named `token` actually OWNS this
|
|
1569
|
+
// `fullPath`. A genuine tokenized root with no literal owner decodes.
|
|
1570
|
+
const literalRecord = existing.get(token);
|
|
1571
|
+
const literalOwnsRoot =
|
|
1572
|
+
configured.has(token) ||
|
|
1573
|
+
(literalRecord !== undefined &&
|
|
1574
|
+
path.resolve(literalRecord.storageDir) === path.resolve(fullPath));
|
|
1575
|
+
// Match `storageFor()`'s canonical namespace identity. A raw root whose
|
|
1576
|
+
// spelling trims to another namespace (for example `namespaces/shared `)
|
|
1577
|
+
// is not a routeable live root and must not be catalogued from disk.
|
|
1578
|
+
const tokenDecoded = literalOwnsRoot ? null : namespaceIdentityFromToken(token);
|
|
1579
|
+
const rawDecoded = tokenDecoded && tokenDecoded.length > 0 ? tokenDecoded : token;
|
|
1580
|
+
const decoded = normalizeNamespaceIdentity(rawDecoded);
|
|
1581
|
+
if (decoded.length === 0 || rawDecoded !== decoded) {
|
|
1582
|
+
skipped.push({ token, reason: "unsafe", detail: rawDecoded });
|
|
1583
|
+
continue;
|
|
1584
|
+
}
|
|
1585
|
+
if (decoded !== defaultNs && !isSafeRouteNamespace(decoded)) {
|
|
1586
|
+
skipped.push({ token, reason: "unsafe", detail: decoded });
|
|
1587
|
+
continue;
|
|
1588
|
+
}
|
|
1589
|
+
// Only catalog roots that actually hold memory data (skip empty shells).
|
|
1590
|
+
// A malformed PRESENT marker is different from an absent marker: if
|
|
1591
|
+
// `facts/` is a file/symlink but `state/` is valid, cataloging the root
|
|
1592
|
+
// would later make catalog-driven QMD scan the bad category directory and
|
|
1593
|
+
// throw. Reject the whole root on the first malformed known marker.
|
|
1594
|
+
const memoryData = await inspectMemoryDataRoot(fullPath);
|
|
1595
|
+
if (memoryData.invalidMarker) {
|
|
1596
|
+
skipped.push({
|
|
1597
|
+
token,
|
|
1598
|
+
reason: "unsafe",
|
|
1599
|
+
detail: `invalid memory marker: ${memoryData.invalidMarker}`,
|
|
1600
|
+
});
|
|
1601
|
+
continue;
|
|
1602
|
+
}
|
|
1603
|
+
if (!memoryData.hasData) continue;
|
|
1604
|
+
|
|
1605
|
+
// Default-root alignment (Issue C): never let a tokenized default dir
|
|
1606
|
+
// overwrite the configured default's storageDir with `fullPath`. The
|
|
1607
|
+
// default record's root is owned by `resolveDefaultNamespaceRoot` above,
|
|
1608
|
+
// which mirrors the router. We still keep the default record (set in
|
|
1609
|
+
// step 1) but skip clobbering its root here.
|
|
1610
|
+
if (decoded === defaultNs) {
|
|
1611
|
+
const def = rebuilt.get(defaultNs);
|
|
1612
|
+
if (def) {
|
|
1613
|
+
def.storageDir = defaultStorageDir;
|
|
1614
|
+
def.kind = "default";
|
|
1615
|
+
}
|
|
1616
|
+
continue;
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
// Dual-root preference: mirror the router, which uses the tokenized root
|
|
1620
|
+
// over a legacy raw-name root when the tokenized one has data. `entry.name`
|
|
1621
|
+
// is the on-disk dir name; it is the tokenized dir iff it equals the
|
|
1622
|
+
// namespace's identity token. If we already recorded this namespace from
|
|
1623
|
+
// its tokenized dir, a later legacy-named entry must not clobber it.
|
|
1624
|
+
const isTokenizedEntry = token === namespaceIdentityToken(decoded);
|
|
1625
|
+
if (rebuilt.has(decoded) && scannedFromTokenized.has(decoded) && !isTokenizedEntry) {
|
|
1626
|
+
continue;
|
|
1627
|
+
}
|
|
1628
|
+
if (isTokenizedEntry) scannedFromTokenized.add(decoded);
|
|
1629
|
+
|
|
1630
|
+
const prior = existing.get(decoded);
|
|
1631
|
+
rebuilt.set(
|
|
1632
|
+
decoded,
|
|
1633
|
+
this.mergeForRebuild(prior, {
|
|
1634
|
+
namespace: decoded,
|
|
1635
|
+
identityToken: namespaceIdentityToken(decoded),
|
|
1636
|
+
kind: inferKind(decoded, this.config),
|
|
1637
|
+
createdAt: prior?.createdAt ?? nowIso,
|
|
1638
|
+
storageDir: fullPath,
|
|
1639
|
+
// Configured-and-present namespaces keep config provenance; purely
|
|
1640
|
+
// discovered ones are scan.
|
|
1641
|
+
discoveredBy: configured.has(decoded) ? "config" : prior?.discoveredBy ?? "scan",
|
|
1642
|
+
}),
|
|
1643
|
+
);
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
// Mark legacy default root explicitly when applicable.
|
|
1647
|
+
if (legacyDefaultHasData && defaultStorageDir === this.memoryDir) {
|
|
1648
|
+
const def = rebuilt.get(defaultNs);
|
|
1649
|
+
if (def) def.kind = "default";
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
// ── Final critical section (SCOPED MUTEX — NFgCT, codex P2) ──────────────
|
|
1653
|
+
// The disk scan above ran LOCKLESS (it only reads). Now, for a mutating
|
|
1654
|
+
// rebuild, acquire the cross-process file lock ONLY for the
|
|
1655
|
+
// load→merge→rename window — the brief section a concurrent touch must be
|
|
1656
|
+
// excluded from. `canMutate` is true iff we ACTUALLY hold the lock: if
|
|
1657
|
+
// acquisition timed out (`acquired === false`) we run compute-only and never
|
|
1658
|
+
// re-merge+rewrite unlocked (which would race a concurrent lock holder and
|
|
1659
|
+
// recreate the lost-append window). A dry-run skips the lock entirely.
|
|
1660
|
+
if (!wantMutate) {
|
|
1661
|
+
return this.finishRebuild(rebuilt, skipped, dryRun, false, memoryReal, nowIso);
|
|
1662
|
+
}
|
|
1663
|
+
// Test-only seam: the SCAN is now complete but the cross-process lock has NOT
|
|
1664
|
+
// yet been acquired (NFgCT). A concurrency test attempts a cross-instance touch
|
|
1665
|
+
// here and asserts it is NOT blocked/dropped — proving the scan is lockless.
|
|
1666
|
+
if (this.onRebuildAfterScanForTest) {
|
|
1667
|
+
await this.onRebuildAfterScanForTest();
|
|
1668
|
+
}
|
|
1669
|
+
return this.withHeldCatalogLock((acquired) =>
|
|
1670
|
+
this.finishRebuild(rebuilt, skipped, dryRun, acquired, memoryReal, nowIso),
|
|
1671
|
+
);
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
/**
|
|
1675
|
+
* Final load→merge→rename window of a rebuild, factored out so the caller can
|
|
1676
|
+
* run it WITHIN the cross-process file lock (NFgCT, codex P2) without holding
|
|
1677
|
+
* that lock across the preceding disk scan. Re-reads the latest on-disk state,
|
|
1678
|
+
* folds concurrent touches, then (when `canMutate`) atomically rewrites the log.
|
|
1679
|
+
*
|
|
1680
|
+
* `canMutate` records that the cross-process lock was actually held. The
|
|
1681
|
+
* re-merge + rewrite run only when it is true — a dry-run, or an unlocked apply
|
|
1682
|
+
* (lock-acquisition timeout), computes records but does NOT rename, so it can
|
|
1683
|
+
* never clobber a concurrent lock holder's window. `applied` mirrors `canMutate`.
|
|
1684
|
+
*/
|
|
1685
|
+
private async finishRebuild(
|
|
1686
|
+
rebuilt: Map<string, NamespaceRecord>,
|
|
1687
|
+
skipped: NamespaceCatalogSkippedRoot[],
|
|
1688
|
+
dryRun: boolean,
|
|
1689
|
+
canMutate: boolean,
|
|
1690
|
+
memoryReal: string | null,
|
|
1691
|
+
nowIso: string,
|
|
1692
|
+
): Promise<NamespaceCatalogRebuildResult> {
|
|
1693
|
+
if (canMutate) {
|
|
1694
|
+
// CROSS-PROCESS re-merge (round 5, codex P2): under the rebuild lock,
|
|
1695
|
+
// re-read the on-disk log ONE more time and fold any touch fields that
|
|
1696
|
+
// landed AFTER our initial `loadCompacted()` (e.g. a gateway markWrite in
|
|
1697
|
+
// another process) into the rebuilt records — last-write-wins per touch
|
|
1698
|
+
// field. This recovers cross-process appends that completed during the
|
|
1699
|
+
// scan, which the in-process `queueCritical` alone cannot see. Only runs
|
|
1700
|
+
// when we hold the lock (round 6, codex P2 — NBPmY): an unlocked rebuild
|
|
1701
|
+
// must not re-merge then rename, or it races a concurrent lock holder.
|
|
1702
|
+
const latest = await this.loadCompacted();
|
|
1703
|
+
for (const [ns, fresh] of latest) {
|
|
1704
|
+
const current = rebuilt.get(ns);
|
|
1705
|
+
if (!current) {
|
|
1706
|
+
// AUTHORITATIVE PURGE (round 6, cursor Medium — NATqU): the disk scan
|
|
1707
|
+
// is the single source of truth for which namespaces EXIST. A namespace
|
|
1708
|
+
// absent from `rebuilt` was NOT discovered on disk (its root is
|
|
1709
|
+
// empty/deleted) and is NOT configured, so the rebuild is purging it.
|
|
1710
|
+
// We must NOT resurrect it from the log — not even when a CONCURRENT
|
|
1711
|
+
// best-effort `markRead`/`markWrite` touched it after our snapshot. A
|
|
1712
|
+
// touch on a dynamic namespace whose on-disk root was removed only
|
|
1713
|
+
// bumps a timestamp; re-inserting that row (with its stale `storageDir`)
|
|
1714
|
+
// would defeat the purge the rebuild is meant to perform.
|
|
1715
|
+
//
|
|
1716
|
+
// CREATED-AFTER-SCAN RE-CHECK (NFJV8, codex P2): there is a TOCTOU
|
|
1717
|
+
// window where a dynamic namespace is CREATED on disk AFTER the scan
|
|
1718
|
+
// enumerated `namespaces/` but BEFORE this re-merge. Its new root was
|
|
1719
|
+
// missed by the snapshot, yet a gateway `markWrite` already landed a row
|
|
1720
|
+
// in `latest`. Purging that row would drop a LIVE namespace that now has
|
|
1721
|
+
// data on disk. So before purging, re-check the namespace's storage root
|
|
1722
|
+
// RIGHT NOW (with the same symlink/realpath/containment + memory-data
|
|
1723
|
+
// safety the scan uses). If it currently EXISTS with data, the namespace
|
|
1724
|
+
// was created-after-scan and is live — KEEP its row. This is the precise
|
|
1725
|
+
// inverse of NATqU, not a regression of it: a touch on a REMOVED root
|
|
1726
|
+
// re-checks as absent and is still purged below; only a root that EXISTS
|
|
1727
|
+
// on a fresh re-check is kept.
|
|
1728
|
+
//
|
|
1729
|
+
// SAFETY REVALIDATION (NGLz5, codex P2): the `ns` key comes from the
|
|
1730
|
+
// UNTRUSTED log (`latest`), which may carry an unsafe namespace row from a
|
|
1731
|
+
// pre-fix or tampered catalog. The disk SCAN validates every decoded
|
|
1732
|
+
// namespace with `isSafeRouteNamespace` (default exempt) and SKIPS unsafe
|
|
1733
|
+
// ones — so an unsafe namespace is absent from `rebuilt` by design, NOT
|
|
1734
|
+
// because it was deleted. Without re-applying that exact check here, a
|
|
1735
|
+
// matching tokenized dir on disk would let this branch RESURRECT the
|
|
1736
|
+
// unsafe row, and `--apply` would rewrite the catalog with a namespace the
|
|
1737
|
+
// hot touch/config/scan paths all reject — leaving maintenance/QMD able to
|
|
1738
|
+
// enumerate an unsafe namespace after a rebuild that appeared to skip it.
|
|
1739
|
+
// Apply the SAME default-exempt safety gate before the live-root recheck;
|
|
1740
|
+
// an unsafe row is dropped (fall through to purge), never kept.
|
|
1741
|
+
if (ns !== this.defaultNamespaceIdentity && !isSafeRouteNamespace(ns)) {
|
|
1742
|
+
continue;
|
|
1743
|
+
}
|
|
1744
|
+
if (await this.liveStorageRootExistsForRebuild(ns, memoryReal)) {
|
|
1745
|
+
// Created-after-scan: keep the live row. Re-resolve its storageDir to
|
|
1746
|
+
// the safe (router-aligned, contained) root so we never persist a
|
|
1747
|
+
// touch's stale/escaping `storageDir`.
|
|
1748
|
+
const safeDir = await this.resolveSafeStorageDir(ns);
|
|
1749
|
+
// DUAL-ROOT GUARD (codex NR-td): if another rebuilt row already OWNS
|
|
1750
|
+
// this exact storageDir (e.g. the decoded/configured owner of a
|
|
1751
|
+
// token-shaped root that the disk scan resolved), do NOT also resurrect
|
|
1752
|
+
// this stale alias from the untrusted log — that leaves TWO catalog rows
|
|
1753
|
+
// pointing at one root and fans maintenance/QMD out over the wrong
|
|
1754
|
+
// namespace. Enforce at most one row per storageDir: the scan's owner
|
|
1755
|
+
// wins, the alias is dropped (falls through to purge) after folding
|
|
1756
|
+
// its touch fields into the owner so recency filters/maintenance do
|
|
1757
|
+
// not miss a real write.
|
|
1758
|
+
const resolvedSafe = path.resolve(safeDir);
|
|
1759
|
+
let owningNamespace: string | null = null;
|
|
1760
|
+
for (const [otherNs, otherRec] of rebuilt) {
|
|
1761
|
+
if (otherNs !== ns && path.resolve(otherRec.storageDir) === resolvedSafe) {
|
|
1762
|
+
owningNamespace = otherNs;
|
|
1763
|
+
break;
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
if (owningNamespace) {
|
|
1767
|
+
const owner = rebuilt.get(owningNamespace);
|
|
1768
|
+
if (owner) rebuilt.set(owningNamespace, mergeNewerTouchFields(owner, fresh));
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
rebuilt.set(ns, {
|
|
1772
|
+
...fresh,
|
|
1773
|
+
storageDir: safeDir,
|
|
1774
|
+
identityToken: namespaceIdentityToken(ns),
|
|
1775
|
+
kind: fresh.kind ?? inferKind(ns, this.config),
|
|
1776
|
+
createdAt: fresh.createdAt ?? nowIso,
|
|
1777
|
+
});
|
|
1778
|
+
continue;
|
|
1779
|
+
}
|
|
1780
|
+
// Confirmed absent on disk. Losing a touch timestamp for a deleted root
|
|
1781
|
+
// is acceptable (the catalog is rebuildable best-effort metadata);
|
|
1782
|
+
// resurrecting a purged record is not. Drop it.
|
|
1783
|
+
continue;
|
|
1784
|
+
}
|
|
1785
|
+
// SURVIVING namespace (still present in the authoritative disk scan):
|
|
1786
|
+
// fold in any newer touch fields that landed cross-process after our
|
|
1787
|
+
// initial snapshot so a concurrent gateway markWrite is not lost.
|
|
1788
|
+
rebuilt.set(ns, mergeNewerTouchFields(current, fresh));
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
|
|
1792
|
+
const records = [...rebuilt.values()].sort((a, b) => {
|
|
1793
|
+
const byName = a.namespace.localeCompare(b.namespace);
|
|
1794
|
+
if (byName !== 0) return byName;
|
|
1795
|
+
return a.identityToken.localeCompare(b.identityToken);
|
|
1796
|
+
});
|
|
1797
|
+
|
|
1798
|
+
// Only rewrite when we actually hold the cross-process lock (round 6, codex
|
|
1799
|
+
// P2 — NBPmY). A dry-run never mutates; an unlocked rebuild (acquisition
|
|
1800
|
+
// timed out) returns the computed records WITHOUT renaming over the log, so
|
|
1801
|
+
// it can never clobber a concurrent lock holder's window.
|
|
1802
|
+
if (canMutate) {
|
|
1803
|
+
// Test-only seam: the load→rename window where the old check-then-append
|
|
1804
|
+
// touch could be clobbered. A concurrency test attempts a cross-instance
|
|
1805
|
+
// touch here and asserts the held lock blocks it (no-op in production).
|
|
1806
|
+
if (this.onRebuildBeforeRenameForTest) {
|
|
1807
|
+
await this.onRebuildBeforeRenameForTest();
|
|
1808
|
+
}
|
|
1809
|
+
await this.rewriteUnchained(records);
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
// `applied` is true only when we actually rewrote the log: never for a
|
|
1813
|
+
// dry-run, and never for an `--apply` that ran compute-only because it could
|
|
1814
|
+
// not acquire the lock (canMutate=false). Surfaces the real mutation state so
|
|
1815
|
+
// the CLI does not report success on a skipped rewrite (NBn3n/NBsGG).
|
|
1816
|
+
return { dryRun, records, skipped, applied: canMutate };
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
// ── Cross-process catalog write lock (held mutex) ────────────────────────
|
|
1820
|
+
|
|
1821
|
+
/**
|
|
1822
|
+
* Run `fn` while HOLDING the shared cross-process advisory lock (round 5, codex
|
|
1823
|
+
* P2; generalized round 7 — NEZkA). This is the SINGLE mutex shared by BOTH the
|
|
1824
|
+
* touch read→merge→append window AND the rebuild final load→merge→rename window,
|
|
1825
|
+
* so a touch and a rebuild in different processes are mutually exclusive over
|
|
1826
|
+
* their respective critical sections — closing the check-then-append gap where a
|
|
1827
|
+
* polled-only touch could append into a rebuild's load→rename window.
|
|
1828
|
+
*
|
|
1829
|
+
* Acquisition is atomic via `open(..., "wx")`. A lock older than
|
|
1830
|
+
* `REBUILD_LOCK_STALE_MS` is treated as a crashed holder and broken. After
|
|
1831
|
+
* `REBUILD_LOCK_MAX_WAIT_MS` of contention we proceed best-effort WITHOUT the
|
|
1832
|
+
* lock rather than block forever. The lock is always released in `finally`.
|
|
1833
|
+
*
|
|
1834
|
+
* IN-PROCESS SAFETY: every caller invokes this from inside (or wrapping) the
|
|
1835
|
+
* per-process `queueCritical` chain, which serializes all catalog mutations in
|
|
1836
|
+
* THIS process. So within one process only one logical holder attempts OS-lock
|
|
1837
|
+
* acquisition at a time — the file lock is never self-contended in-process, and
|
|
1838
|
+
* the lock is acquired and released within a single in-process turn. The file
|
|
1839
|
+
* lock adds only the missing CROSS-process exclusion.
|
|
1840
|
+
*
|
|
1841
|
+
* HEARTBEAT (round 5, cursor/codex Medium/P2): while WE hold the lock a timer
|
|
1842
|
+
* refreshes its mtime every `REBUILD_LOCK_HEARTBEAT_MS`, so a legitimately long
|
|
1843
|
+
* holder (> `REBUILD_LOCK_STALE_MS`) is not treated as a crashed holder and
|
|
1844
|
+
* unlinked by another process — which would let overlapping windows lose
|
|
1845
|
+
* appends. Heartbeat failures are swallowed; the timer is always cleared in
|
|
1846
|
+
* `finally`.
|
|
1847
|
+
*
|
|
1848
|
+
* ACQUISITION RESULT (round 6, codex P2 — NBPmY): `fn` receives whether WE
|
|
1849
|
+
* actually hold the lock. When acquisition TIMED OUT (another holder is active),
|
|
1850
|
+
* a MUTATING rebuild must NOT perform its load/rename window unlocked, and a
|
|
1851
|
+
* touch must NOT append unlocked — both would recreate the lost-append race. The
|
|
1852
|
+
* caller uses `acquired` to run compute-only (rebuild) or DROP the append
|
|
1853
|
+
* (touch) when unlocked.
|
|
1854
|
+
*/
|
|
1855
|
+
private async withHeldCatalogLock<T>(fn: (acquired: boolean) => Promise<T>): Promise<T> {
|
|
1856
|
+
const acquired = await this.acquireRebuildLock();
|
|
1857
|
+
let heartbeat: ReturnType<typeof setInterval> | undefined;
|
|
1858
|
+
if (acquired) {
|
|
1859
|
+
heartbeat = setInterval(() => {
|
|
1860
|
+
const now = new Date();
|
|
1861
|
+
// Refresh mtime so age-based stale detection sees an active holder.
|
|
1862
|
+
utimes(this.rebuildLockPath, now, now).catch(() => undefined);
|
|
1863
|
+
}, REBUILD_LOCK_HEARTBEAT_MS);
|
|
1864
|
+
// Don't keep the event loop alive solely for the heartbeat.
|
|
1865
|
+
heartbeat.unref?.();
|
|
1866
|
+
}
|
|
1867
|
+
try {
|
|
1868
|
+
return await fn(acquired);
|
|
1869
|
+
} finally {
|
|
1870
|
+
if (heartbeat) clearInterval(heartbeat);
|
|
1871
|
+
if (acquired) {
|
|
1872
|
+
try {
|
|
1873
|
+
// Release ONLY the lock still owned by THIS instance (round 6, codex
|
|
1874
|
+
// P2 — NCzT6). If this rebuild paused long enough that another process
|
|
1875
|
+
// treated our lock as stale, unlinked it, and acquired a REPLACEMENT,
|
|
1876
|
+
// an unconditional unlink here would delete that other holder's active
|
|
1877
|
+
// lock — letting writers/another rebuild proceed during its load/rename
|
|
1878
|
+
// window and recreating the lost-append race. Verify ownership first.
|
|
1879
|
+
if (await this.rebuildLockHeldBySelf()) {
|
|
1880
|
+
await unlink(this.rebuildLockPath);
|
|
1881
|
+
}
|
|
1882
|
+
} catch {
|
|
1883
|
+
// Best-effort release; a stale lock will be broken on next rebuild.
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
/** Try to acquire the rebuild lock; returns true if WE created it. */
|
|
1890
|
+
private async acquireRebuildLock(): Promise<boolean> {
|
|
1891
|
+
const deadline = Date.now() + REBUILD_LOCK_MAX_WAIT_MS;
|
|
1892
|
+
await mkdir(this.stateDir, { recursive: true });
|
|
1893
|
+
for (;;) {
|
|
1894
|
+
try {
|
|
1895
|
+
const handle = await open(this.rebuildLockPath, "wx");
|
|
1896
|
+
try {
|
|
1897
|
+
// Record PID, this instance's owner id, and a timestamp. The owner id
|
|
1898
|
+
// distinguishes same-process instances (NBsGP).
|
|
1899
|
+
await handle.writeFile(
|
|
1900
|
+
`${process.pid} ${this.lockOwnerId} ${new Date().toISOString()}\n`,
|
|
1901
|
+
"utf8",
|
|
1902
|
+
);
|
|
1903
|
+
} catch {
|
|
1904
|
+
// Ignore write failures — the exclusive create already gave us the lock.
|
|
1905
|
+
} finally {
|
|
1906
|
+
await handle.close();
|
|
1907
|
+
}
|
|
1908
|
+
return true;
|
|
1909
|
+
} catch (err) {
|
|
1910
|
+
if ((err as NodeJS.ErrnoException)?.code !== "EEXIST") {
|
|
1911
|
+
// Unexpected FS error — proceed best-effort without the lock.
|
|
1912
|
+
return false;
|
|
1913
|
+
}
|
|
1914
|
+
// Lock exists: break it if stale, otherwise wait briefly.
|
|
1915
|
+
await this.breakStaleRebuildLock();
|
|
1916
|
+
if (Date.now() >= deadline) return false;
|
|
1917
|
+
await new Promise((r) => setTimeout(r, REBUILD_LOCK_POLL_MS));
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
/**
|
|
1923
|
+
* Remove the lock file if its mtime is older than the stale threshold.
|
|
1924
|
+
*
|
|
1925
|
+
* REPLACEMENT-SAFE (NG7Bg, codex P2): a plain `stat` → `unlink` has a TOCTOU
|
|
1926
|
+
* window — two processes can both observe the SAME stale lock; one removes it and
|
|
1927
|
+
* creates a FRESH lock, and the other's later `unlink` then deletes that fresh
|
|
1928
|
+
* holder's ACTIVE lock based on the stale identity it read earlier, leaving the
|
|
1929
|
+
* fresh holder running its critical section with no visible lock and reopening the
|
|
1930
|
+
* lost-update race the mutex prevents. We therefore capture the lock's IDENTITY
|
|
1931
|
+
* (its full content line: `<pid> <owner-uuid> <iso>`) when we judge it stale, then
|
|
1932
|
+
* RE-READ immediately before unlinking and only remove it when the content is
|
|
1933
|
+
* byte-identical AND still stale. A replacement lock has a different owner id /
|
|
1934
|
+
* timestamp, so its content differs and we leave it untouched. We never unlink a
|
|
1935
|
+
* lock whose mtime is now fresh (a heartbeat refreshed it) or whose identity
|
|
1936
|
+
* changed (a replacement was created). This is best-effort: any mismatch/vanish
|
|
1937
|
+
* simply skips the break and the caller polls again.
|
|
1938
|
+
*/
|
|
1939
|
+
private async breakStaleRebuildLock(): Promise<void> {
|
|
1940
|
+
let staleIdentity: string;
|
|
1941
|
+
try {
|
|
1942
|
+
const info = await stat(this.rebuildLockPath);
|
|
1943
|
+
if (Date.now() - info.mtimeMs <= REBUILD_LOCK_STALE_MS) {
|
|
1944
|
+
// Not stale (e.g. a live holder's heartbeat keeps it fresh) — leave it.
|
|
1945
|
+
return;
|
|
1946
|
+
}
|
|
1947
|
+
// Capture the exact identity we judged stale, so we can confirm it has not
|
|
1948
|
+
// been replaced before we unlink.
|
|
1949
|
+
staleIdentity = await readFile(this.rebuildLockPath, "utf8");
|
|
1950
|
+
} catch {
|
|
1951
|
+
// Lock vanished (released by holder) or stat/read failed — nothing to do.
|
|
1952
|
+
return;
|
|
1953
|
+
}
|
|
1954
|
+
// Test-only seam: simulate a replacement lock being created in the race window
|
|
1955
|
+
// between the staleness judgment and the unlink (NG7Bg). No-op in production.
|
|
1956
|
+
if (this.onBeforeBreakStaleUnlinkForTest) {
|
|
1957
|
+
await this.onBeforeBreakStaleUnlinkForTest();
|
|
1958
|
+
}
|
|
1959
|
+
try {
|
|
1960
|
+
// Re-validate immediately before unlinking: the lock must still carry the
|
|
1961
|
+
// SAME identity AND still be stale. If a replacement lock was created in the
|
|
1962
|
+
// window (different owner/timestamp) or a heartbeat refreshed the mtime, do
|
|
1963
|
+
// NOT unlink — that would delete another process's ACTIVE lock.
|
|
1964
|
+
const current = await readFile(this.rebuildLockPath, "utf8");
|
|
1965
|
+
if (current !== staleIdentity) return; // replaced — leave the fresh lock
|
|
1966
|
+
const recheck = await stat(this.rebuildLockPath);
|
|
1967
|
+
if (Date.now() - recheck.mtimeMs <= REBUILD_LOCK_STALE_MS) return; // refreshed
|
|
1968
|
+
await unlink(this.rebuildLockPath).catch(() => undefined);
|
|
1969
|
+
} catch {
|
|
1970
|
+
// The lock changed/vanished between checks — another process handled it.
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
/**
|
|
1975
|
+
* Whether the rebuild lock file was written by THIS instance (round 6, codex
|
|
1976
|
+
* P2 — NBsGP). Matches the per-instance owner id, NOT just `process.pid`: two
|
|
1977
|
+
* NamespaceCatalog instances in the same process share a PID, so a PID-only
|
|
1978
|
+
* check would wrongly treat instance A's lock as self-held by instance B and
|
|
1979
|
+
* let B's touch skip the wait and append into A's rebuild window. Falls back to
|
|
1980
|
+
* the legacy PID-only form for lock files written before owner ids existed.
|
|
1981
|
+
*/
|
|
1982
|
+
private async rebuildLockHeldBySelf(): Promise<boolean> {
|
|
1983
|
+
try {
|
|
1984
|
+
const body = await readFile(this.rebuildLockPath, "utf8");
|
|
1985
|
+
const parts = body.trim().split(/\s+/);
|
|
1986
|
+
const pid = Number.parseInt(parts[0] ?? "", 10);
|
|
1987
|
+
const ownerId = parts[1];
|
|
1988
|
+
// New format: "<pid> <uuid> <iso>". A UUID at parts[1] uniquely identifies
|
|
1989
|
+
// the writing INSTANCE; only the same instance is self. The strict UUID
|
|
1990
|
+
// shape avoids mistaking a legacy "<pid> <iso>" timestamp (also hyphenated)
|
|
1991
|
+
// for an owner id.
|
|
1992
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
1993
|
+
if (ownerId && UUID_RE.test(ownerId)) {
|
|
1994
|
+
return ownerId === this.lockOwnerId;
|
|
1995
|
+
}
|
|
1996
|
+
// Legacy format: "<pid> <iso>" (no owner id). Best-effort PID match.
|
|
1997
|
+
return Number.isFinite(pid) && pid === process.pid;
|
|
1998
|
+
} catch {
|
|
1999
|
+
return false;
|
|
2000
|
+
}
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
/**
|
|
2004
|
+
* Merge a prior record's preserved metadata (timestamps, principal hints)
|
|
2005
|
+
* onto a freshly-discovered record. Disk-derived fields (storageDir, kind)
|
|
2006
|
+
* take precedence from the new record.
|
|
2007
|
+
*
|
|
2008
|
+
* PROVENANCE (round 3, cursor Low): `discoveredBy` and `createdAt` are
|
|
2009
|
+
* CREATION-ONLY — identical to the touch path's invariant. A rebuild must NOT
|
|
2010
|
+
* reset a namespace first seen via a `write`/`read` touch back to `config`
|
|
2011
|
+
* just because it is also listed in policies. So when a prior record exists we
|
|
2012
|
+
* carry its `discoveredBy` forward; only brand-new records keep the fresh
|
|
2013
|
+
* (config/scan) provenance.
|
|
2014
|
+
*/
|
|
2015
|
+
private mergeForRebuild(prior: NamespaceRecord | undefined, fresh: NamespaceRecord): NamespaceRecord {
|
|
2016
|
+
if (!prior) return fresh;
|
|
2017
|
+
const merged: NamespaceRecord = {
|
|
2018
|
+
...fresh,
|
|
2019
|
+
createdAt: prior.createdAt ?? fresh.createdAt,
|
|
2020
|
+
discoveredBy: prior.discoveredBy ?? fresh.discoveredBy,
|
|
2021
|
+
};
|
|
2022
|
+
if (prior.lastReadAt) merged.lastReadAt = prior.lastReadAt;
|
|
2023
|
+
if (prior.lastWriteAt) merged.lastWriteAt = prior.lastWriteAt;
|
|
2024
|
+
if (prior.lastMaintenanceAt) merged.lastMaintenanceAt = { ...prior.lastMaintenanceAt };
|
|
2025
|
+
if (prior.principal !== undefined) merged.principal = prior.principal;
|
|
2026
|
+
if (prior.projectId !== undefined) merged.projectId = prior.projectId;
|
|
2027
|
+
if (prior.branch !== undefined) merged.branch = prior.branch;
|
|
2028
|
+
if (prior.parentNamespace !== undefined) merged.parentNamespace = prior.parentNamespace;
|
|
2029
|
+
return merged;
|
|
2030
|
+
}
|
|
2031
|
+
|
|
2032
|
+
// ── Persistence ──────────────────────────────────────────────────────────
|
|
2033
|
+
|
|
2034
|
+
/** Load the JSONL log and fold it into current state (last-record-wins). */
|
|
2035
|
+
private async loadCompacted(): Promise<Map<string, NamespaceRecord>> {
|
|
2036
|
+
const records = new Map<string, NamespaceRecord>();
|
|
2037
|
+
let raw: string;
|
|
2038
|
+
try {
|
|
2039
|
+
raw = await readFile(this.catalogPath, "utf8");
|
|
2040
|
+
} catch {
|
|
2041
|
+
return records;
|
|
2042
|
+
}
|
|
2043
|
+
for (const line of raw.split("\n")) {
|
|
2044
|
+
const trimmed = line.trim();
|
|
2045
|
+
if (trimmed.length === 0) continue;
|
|
2046
|
+
let parsed: unknown;
|
|
2047
|
+
try {
|
|
2048
|
+
parsed = JSON.parse(trimmed);
|
|
2049
|
+
} catch {
|
|
2050
|
+
// Skip corrupt lines (CLAUDE.md rule #18 robustness).
|
|
2051
|
+
continue;
|
|
2052
|
+
}
|
|
2053
|
+
const record = coerceRecord(parsed);
|
|
2054
|
+
if (!record) continue;
|
|
2055
|
+
// Field-level touch merge during compaction (round 6, codex P2 — ND6Cz).
|
|
2056
|
+
// Touches run on PER-PROCESS write chains, so two processes (a gateway write
|
|
2057
|
+
// racing a CLI/second-server read or maintenance touch) can each load the
|
|
2058
|
+
// same prior record and append a full snapshot. Plain last-record-wins
|
|
2059
|
+
// compaction would then discard the earlier snapshot's `lastReadAt` /
|
|
2060
|
+
// `lastWriteAt` / `lastMaintenanceAt`, erasing a real touch and skewing
|
|
2061
|
+
// `writtenSince`. We instead take the LATER record as the base (most recent
|
|
2062
|
+
// identity/disk-derived state) and fold in the MAX of each touch field from
|
|
2063
|
+
// both, so no cross-process touch recency is lost without locking the hot
|
|
2064
|
+
// touch path. A destructive overwrite of real memory is never at stake here
|
|
2065
|
+
// — only best-effort recency metadata.
|
|
2066
|
+
const prior = records.get(record.namespace);
|
|
2067
|
+
records.set(record.namespace, prior ? mergeNewerTouchFields(record, prior) : record);
|
|
2068
|
+
}
|
|
2069
|
+
return records;
|
|
2070
|
+
}
|
|
2071
|
+
|
|
2072
|
+
/**
|
|
2073
|
+
* Serialize an arbitrary read-modify-write critical section through the single
|
|
2074
|
+
* write chain. Every catalog mutation (touch read+merge+append, full rewrite)
|
|
2075
|
+
* runs through this so they are mutually exclusive: a touch always reads the
|
|
2076
|
+
* latest persisted state before appending, and a rebuild rewrite cannot
|
|
2077
|
+
* interleave with a touch's append. The chain recovers from rejection
|
|
2078
|
+
* (CLAUDE.md rule #40) — one failed section never poisons subsequent ones —
|
|
2079
|
+
* while still surfacing the error to that section's awaited promise.
|
|
2080
|
+
*/
|
|
2081
|
+
private queueCritical<T>(fn: () => Promise<T>): Promise<T> {
|
|
2082
|
+
const run = this.writeChain.then(fn);
|
|
2083
|
+
// Keep the chain alive after a rejection so later sections still run.
|
|
2084
|
+
this.writeChain = run.then(
|
|
2085
|
+
() => undefined,
|
|
2086
|
+
() => undefined,
|
|
2087
|
+
);
|
|
2088
|
+
return run;
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
/**
|
|
2092
|
+
* Append a single record to the JSONL log WITHOUT re-serializing through the
|
|
2093
|
+
* write chain. MUST only be called from inside a `queueCritical(...)` section
|
|
2094
|
+
* (which already holds the serialized turn); calling it directly would bypass
|
|
2095
|
+
* the read-before-append ordering that prevents lost-field races.
|
|
2096
|
+
*/
|
|
2097
|
+
private async appendUnchained(record: NamespaceRecord): Promise<void> {
|
|
2098
|
+
const line = serializeRecord(record) + "\n";
|
|
2099
|
+
await mkdir(this.stateDir, { recursive: true });
|
|
2100
|
+
await appendFile(this.catalogPath, line, "utf8");
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
/**
|
|
2104
|
+
* Atomic temp-file + rename rewrite (CLAUDE.md rule #54: write temp, then
|
|
2105
|
+
* rename — never delete-before-write) WITHOUT re-entering the write chain.
|
|
2106
|
+
* MUST only be called from inside a `queueCritical(...)` turn (e.g. the
|
|
2107
|
+
* rebuild critical section, which already holds the serialized turn so its
|
|
2108
|
+
* load and rewrite are atomic against concurrent touches). Re-entering the
|
|
2109
|
+
* chain from within a held turn would deadlock.
|
|
2110
|
+
*/
|
|
2111
|
+
private async rewriteUnchained(records: NamespaceRecord[]): Promise<void> {
|
|
2112
|
+
const body = records.map((r) => serializeRecord(r)).join("\n") + (records.length > 0 ? "\n" : "");
|
|
2113
|
+
await mkdir(this.stateDir, { recursive: true });
|
|
2114
|
+
const tmp = `${this.catalogPath}.${process.pid}.${Date.now()}.tmp`;
|
|
2115
|
+
await writeFile(tmp, body, "utf8");
|
|
2116
|
+
await rename(tmp, this.catalogPath);
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
function isPathInside(root: string, child: string): boolean {
|
|
2121
|
+
const relative = path.relative(root, child);
|
|
2122
|
+
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
|
|
2123
|
+
}
|