@remnic/core 9.52.1 → 9.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.d.ts +5 -5
- package/dist/access-admin-ops-surface.js +22 -22
- package/dist/access-authorization-probe.d.ts +5 -5
- package/dist/access-authorization-probe.js +23 -23
- package/dist/access-boundary.d.ts +5 -5
- package/dist/access-boundary.js +22 -22
- package/dist/access-cli.js +54 -54
- package/dist/access-extraction-force-flush.d.ts +5 -5
- package/dist/access-extraction-force-flush.js +22 -22
- package/dist/access-http-lcm-compaction.d.ts +5 -5
- package/dist/access-http-lifecycle-flush.d.ts +5 -5
- package/dist/access-http-offline-stream.d.ts +5 -5
- package/dist/access-http-query.js +22 -22
- package/dist/access-http.d.ts +5 -5
- package/dist/access-http.js +29 -29
- package/dist/access-identity-continuity-surface.d.ts +4 -4
- package/dist/access-identity-continuity-surface.js +22 -22
- package/dist/access-lcm-surface.d.ts +5 -5
- package/dist/access-lcm-surface.js +22 -22
- package/dist/access-mcp.d.ts +5 -5
- package/dist/access-mcp.js +27 -27
- package/dist/access-namespace-preflight.js +22 -22
- package/dist/access-observe-write-surface.d.ts +5 -5
- package/dist/access-observe-write-surface.js +22 -22
- package/dist/access-offline-manifest.d.ts +4 -4
- package/dist/access-offline-manifest.js +8 -4
- package/dist/access-operations-batch.js +24 -24
- package/dist/access-operations.d.ts +5 -5
- package/dist/access-operations.js +26 -26
- package/dist/access-recall-concurrency.d.ts +5 -5
- package/dist/access-recall-concurrency.js +22 -22
- package/dist/access-recall-response.d.ts +5 -5
- package/dist/access-recall-response.js +22 -22
- package/dist/access-recall-surface.d.ts +5 -5
- package/dist/access-recall-surface.js +22 -22
- package/dist/{access-service-Cbk3bNbD.d.ts → access-service-CgCjvbQu.d.ts} +3 -3
- package/dist/access-service-helpers.d.ts +5 -5
- package/dist/access-service.d.ts +5 -5
- package/dist/access-service.js +22 -22
- package/dist/access-surface-catalog.d.ts +5 -5
- package/dist/access-wearables-meetings-surface.d.ts +2 -2
- package/dist/active-recall.js +4 -4
- package/dist/{auto-sync-QFQGOEO2.js → auto-sync-JXEW7444.js} +6 -6
- package/dist/bootstrap.d.ts +4 -4
- package/dist/briefing.d.ts +1 -1
- package/dist/briefing.js +12 -12
- package/dist/buffer.d.ts +1 -1
- package/dist/causal-consolidation.js +13 -13
- package/dist/{chunk-MLXZD4WM.js → chunk-2GFK22OZ.js} +2 -2
- package/dist/{chunk-D5YOZ66N.js → chunk-3DWTDC54.js} +20 -20
- package/dist/{chunk-RANKJI5T.js → chunk-3JLPGJ43.js} +2 -2
- package/dist/chunk-5FVVY2EB.js +27 -0
- package/dist/chunk-5FVVY2EB.js.map +1 -0
- package/dist/{chunk-GXSOUIA4.js → chunk-5WQMORU5.js} +4 -4
- package/dist/{chunk-J272WKX2.js → chunk-6HPHEFXT.js} +2 -2
- package/dist/{chunk-B6LZ4IBH.js → chunk-6LAPLZOS.js} +2 -2
- package/dist/{chunk-WD5B5AIE.js → chunk-6R4G7N4S.js} +118 -173
- package/dist/chunk-6R4G7N4S.js.map +1 -0
- package/dist/{chunk-VXCNQYTC.js → chunk-APPFNGX6.js} +2 -2
- package/dist/{chunk-RMHMUUBW.js → chunk-AY43LBE4.js} +104 -48
- package/dist/{chunk-RMHMUUBW.js.map → chunk-AY43LBE4.js.map} +1 -1
- package/dist/{chunk-GXFSQ6SD.js → chunk-C4ZNGVK3.js} +2 -2
- package/dist/{chunk-JSEJH2YQ.js → chunk-CGVHPCP2.js} +2 -2
- package/dist/{chunk-BQIFZ2MP.js → chunk-CX5SW2EW.js} +41 -24
- package/dist/chunk-CX5SW2EW.js.map +1 -0
- package/dist/{chunk-TEMSWHQD.js → chunk-DZD576ZU.js} +5 -5
- package/dist/{chunk-AMTEYNYK.js → chunk-EVWWT3KE.js} +4 -4
- package/dist/{chunk-XAPMEU3K.js → chunk-FHFERRYE.js} +2 -2
- package/dist/{chunk-P2KBDJJ6.js → chunk-G2LUMAUF.js} +4 -3
- package/dist/{chunk-P2KBDJJ6.js.map → chunk-G2LUMAUF.js.map} +1 -1
- package/dist/{chunk-AXIDQG5A.js → chunk-G4T7L557.js} +5 -5
- package/dist/{chunk-7M6NYQWB.js → chunk-G7Z6EENO.js} +4 -4
- package/dist/{chunk-YXKVLYXU.js → chunk-HWRD23DW.js} +2 -2
- package/dist/{chunk-SOEN6KCF.js → chunk-IBKR2TFR.js} +2 -2
- package/dist/{chunk-R3DJVOSM.js → chunk-JDSVLVCG.js} +8 -1
- package/dist/chunk-JDSVLVCG.js.map +1 -0
- package/dist/{chunk-F4EU6VUC.js → chunk-JDZPNCMA.js} +2 -2
- package/dist/{chunk-DLG5V6ND.js → chunk-K4QS7LXU.js} +4 -7
- package/dist/chunk-K4QS7LXU.js.map +1 -0
- package/dist/{chunk-BAKXNQ3C.js → chunk-KRTY3ENI.js} +2 -2
- package/dist/{chunk-NK5RGJ65.js → chunk-L4DLLKPN.js} +2 -2
- package/dist/{chunk-VFSAWRMZ.js → chunk-LJS7KUPV.js} +2 -2
- package/dist/{chunk-JJRVPWFY.js → chunk-MLPTFODV.js} +166 -37
- package/dist/chunk-MLPTFODV.js.map +1 -0
- package/dist/{chunk-NOEMGE56.js → chunk-NH5IR2WM.js} +2 -2
- package/dist/{chunk-5V6IXHPI.js → chunk-NSVDUN76.js} +2 -2
- package/dist/{chunk-IARXXZU4.js → chunk-P7XI2AQ3.js} +2 -2
- package/dist/{chunk-W7Y5JDVB.js → chunk-PDDTI7IC.js} +4 -4
- package/dist/{chunk-QINJITDS.js → chunk-PX2LSHYM.js} +2 -2
- package/dist/{chunk-O5FFBO2M.js → chunk-QAVH6X2A.js} +2 -2
- package/dist/{chunk-HDIUYMSB.js → chunk-QWGE2PUH.js} +6 -6
- package/dist/{chunk-X57M2Q4I.js → chunk-RDJRFDJY.js} +3 -3
- package/dist/{chunk-RYLEDKQ2.js → chunk-RUMNY5DT.js} +3 -3
- package/dist/{chunk-BKIFTXHR.js → chunk-RZ4XGKOJ.js} +2 -2
- package/dist/{chunk-66SE3BNH.js → chunk-SUPEYLH2.js} +2 -2
- package/dist/{chunk-BF6SXNQJ.js → chunk-T2JHZWH6.js} +20 -20
- package/dist/{chunk-VHFXP27N.js → chunk-THPPCISA.js} +60 -23
- package/dist/chunk-THPPCISA.js.map +1 -0
- package/dist/{chunk-QPOHMO6M.js → chunk-UES4BHA2.js} +2 -2
- package/dist/{chunk-X7XEDOSL.js → chunk-VLL43QEQ.js} +8 -8
- package/dist/{chunk-QBSFOKY6.js → chunk-VOELBUS4.js} +4 -4
- package/dist/{chunk-2LY6GIIE.js → chunk-WDBX6VER.js} +2 -2
- package/dist/{chunk-BMEYMKWX.js → chunk-WM65LU67.js} +2 -2
- package/dist/{chunk-WLRD2TXF.js → chunk-WOPA3SGI.js} +4 -4
- package/dist/{chunk-5UZ6UCOZ.js → chunk-XUAMCLY4.js} +3 -3
- package/dist/{chunk-7HXZJAID.js → chunk-YGLY6QFV.js} +3 -3
- package/dist/{chunk-2OIQDTDC.js → chunk-ZPJ4QSGP.js} +2 -2
- package/dist/{chunk-3OY7KXHM.js → chunk-ZYLC7NFO.js} +3 -3
- package/dist/{cli-DMAFI0qt.d.ts → cli-D-HiEYhl.d.ts} +3 -3
- package/dist/cli.d.ts +6 -6
- package/dist/cli.js +41 -41
- package/dist/compounding/engine.d.ts +1 -1
- package/dist/compounding/engine.js +13 -13
- package/dist/config.js +4 -4
- package/dist/connectors/codex-materialize-runner.js +12 -12
- package/dist/connectors/index.js +12 -12
- package/dist/consolidation-provenance-check.d.ts +1 -1
- package/dist/consolidation-undo.d.ts +1 -1
- package/dist/content-hash.d.ts +12 -2
- package/dist/content-hash.js +9 -3
- package/dist/contradiction/index.d.ts +1 -1
- package/dist/contradiction/index.js +3 -3
- package/dist/convergence-refresh.d.ts +1 -1
- package/dist/convergence-refresh.js +14 -14
- package/dist/corpus-watermark.js +14 -14
- package/dist/entity-retrieval.d.ts +1 -1
- package/dist/entity-retrieval.js +12 -12
- package/dist/explicit-capture.d.ts +4 -4
- package/dist/explicit-capture.js +8 -8
- package/dist/external-wiki-access.d.ts +5 -5
- package/dist/external-wiki-access.js +23 -23
- package/dist/external-wiki-mcp-tools.d.ts +5 -5
- package/dist/extraction-normalization.js +2 -2
- package/dist/extraction-source-grounding.js +3 -3
- package/dist/extraction.js +16 -16
- package/dist/{forget-HDS4N6CG.js → forget-EHGGJZLX.js} +3 -2
- package/dist/{forget-HDS4N6CG.js.map → forget-EHGGJZLX.js.map} +1 -1
- package/dist/harmonic-construction.js +2 -2
- package/dist/harmonic-retrieval.js +4 -4
- package/dist/index.d.ts +12 -12
- package/dist/index.js +66 -66
- package/dist/maintenance/memory-governance.js +12 -12
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.d.ts +1 -1
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +12 -12
- package/dist/maintenance/rebuild-memory-projection.d.ts +1 -1
- package/dist/maintenance/rebuild-memory-projection.js +13 -13
- package/dist/{maintenance-kBII2qUI.d.ts → maintenance-CN_Ct-Hr.d.ts} +1 -1
- package/dist/mcp-memory-inspector-app.d.ts +5 -5
- package/dist/memory-worth-outcomes.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +1 -1
- package/dist/namespaces/migrate.js +13 -13
- package/dist/namespaces/storage.d.ts +1 -1
- package/dist/namespaces/storage.js +12 -12
- package/dist/operator-doctor-corpus.js +15 -15
- package/dist/operator-toolkit.d.ts +1 -1
- package/dist/operator-toolkit.js +20 -20
- package/dist/orchestration/compression-guideline-coordinator.d.ts +1 -1
- package/dist/orchestration/maintenance.d.ts +2 -2
- package/dist/orchestration/maintenance.js +17 -17
- package/dist/{orchestrator-DHy5LtTy.d.ts → orchestrator-B3Xe8qu0.d.ts} +3 -3
- package/dist/orchestrator.d.ts +4 -4
- package/dist/orchestrator.js +54 -54
- package/dist/provenance.js +2 -2
- package/dist/reconcile/manifest.d.ts +3 -1
- package/dist/reconcile/manifest.js +7 -3
- package/dist/{resolution-MDPNOEG6.js → resolution-MMOD5WC4.js} +4 -4
- package/dist/resume-bundles.js +5 -5
- package/dist/salvage-envelope.js +3 -3
- package/dist/schemas.d.ts +38 -38
- package/dist/semantic-consolidation.js +13 -13
- package/dist/semantic-rule-promotion.js +12 -12
- package/dist/semantic-rule-verifier.js +12 -12
- package/dist/{service-PT2_z7xm.d.ts → service-CQGEqL1g.d.ts} +1 -1
- package/dist/source-agent-qualifier.js +14 -14
- package/dist/{storage-BmVudhk4.d.ts → storage-DmkCjo0l.d.ts} +36 -32
- package/dist/storage.d.ts +1 -1
- package/dist/storage.js +14 -14
- package/dist/structured-attributes.d.ts +3 -1
- package/dist/structured-attributes.js +3 -1
- package/dist/tier-migration.d.ts +1 -1
- package/dist/transfer/types.d.ts +12 -12
- package/dist/verified-recall.js +12 -12
- package/dist/write-envelope.js +2 -2
- package/package.json +2 -2
- package/src/access-offline-manifest.ts +1 -0
- package/src/access-service-offline-file-content.test.ts +1 -0
- package/src/content-hash.ts +34 -1
- package/src/dedup/index.test.ts +56 -38
- package/src/dedup/index.ts +85 -38
- package/src/lifecycle/tombstones.test.ts +763 -16
- package/src/lifecycle/tombstones.ts +217 -50
- package/src/reconcile/manifest.test.ts +158 -1
- package/src/reconcile/manifest.ts +76 -22
- package/src/storage/content-hash-index.ts +56 -40
- package/src/storage/entity-ref-repair.ts +3 -3
- package/src/storage/memory-read-store.ts +9 -0
- package/src/storage/tombstone-blocked-capture-index.ts +4 -9
- package/src/storage-contract/content-hash.test.ts +13 -0
- package/src/storage-fact-hash-index-defer.test.ts +1 -1
- package/src/storage.ts +107 -93
- package/src/structured-attributes.ts +11 -0
- package/dist/chunk-BQIFZ2MP.js.map +0 -1
- package/dist/chunk-DLG5V6ND.js.map +0 -1
- package/dist/chunk-JJRVPWFY.js.map +0 -1
- package/dist/chunk-R3DJVOSM.js.map +0 -1
- package/dist/chunk-VHFXP27N.js.map +0 -1
- package/dist/chunk-W5H6ZRPT.js +0 -14
- package/dist/chunk-W5H6ZRPT.js.map +0 -1
- package/dist/chunk-WD5B5AIE.js.map +0 -1
- /package/dist/{auto-sync-QFQGOEO2.js.map → auto-sync-JXEW7444.js.map} +0 -0
- /package/dist/{chunk-MLXZD4WM.js.map → chunk-2GFK22OZ.js.map} +0 -0
- /package/dist/{chunk-D5YOZ66N.js.map → chunk-3DWTDC54.js.map} +0 -0
- /package/dist/{chunk-RANKJI5T.js.map → chunk-3JLPGJ43.js.map} +0 -0
- /package/dist/{chunk-GXSOUIA4.js.map → chunk-5WQMORU5.js.map} +0 -0
- /package/dist/{chunk-J272WKX2.js.map → chunk-6HPHEFXT.js.map} +0 -0
- /package/dist/{chunk-B6LZ4IBH.js.map → chunk-6LAPLZOS.js.map} +0 -0
- /package/dist/{chunk-VXCNQYTC.js.map → chunk-APPFNGX6.js.map} +0 -0
- /package/dist/{chunk-GXFSQ6SD.js.map → chunk-C4ZNGVK3.js.map} +0 -0
- /package/dist/{chunk-JSEJH2YQ.js.map → chunk-CGVHPCP2.js.map} +0 -0
- /package/dist/{chunk-TEMSWHQD.js.map → chunk-DZD576ZU.js.map} +0 -0
- /package/dist/{chunk-AMTEYNYK.js.map → chunk-EVWWT3KE.js.map} +0 -0
- /package/dist/{chunk-XAPMEU3K.js.map → chunk-FHFERRYE.js.map} +0 -0
- /package/dist/{chunk-AXIDQG5A.js.map → chunk-G4T7L557.js.map} +0 -0
- /package/dist/{chunk-7M6NYQWB.js.map → chunk-G7Z6EENO.js.map} +0 -0
- /package/dist/{chunk-YXKVLYXU.js.map → chunk-HWRD23DW.js.map} +0 -0
- /package/dist/{chunk-SOEN6KCF.js.map → chunk-IBKR2TFR.js.map} +0 -0
- /package/dist/{chunk-F4EU6VUC.js.map → chunk-JDZPNCMA.js.map} +0 -0
- /package/dist/{chunk-BAKXNQ3C.js.map → chunk-KRTY3ENI.js.map} +0 -0
- /package/dist/{chunk-NK5RGJ65.js.map → chunk-L4DLLKPN.js.map} +0 -0
- /package/dist/{chunk-VFSAWRMZ.js.map → chunk-LJS7KUPV.js.map} +0 -0
- /package/dist/{chunk-NOEMGE56.js.map → chunk-NH5IR2WM.js.map} +0 -0
- /package/dist/{chunk-5V6IXHPI.js.map → chunk-NSVDUN76.js.map} +0 -0
- /package/dist/{chunk-IARXXZU4.js.map → chunk-P7XI2AQ3.js.map} +0 -0
- /package/dist/{chunk-W7Y5JDVB.js.map → chunk-PDDTI7IC.js.map} +0 -0
- /package/dist/{chunk-QINJITDS.js.map → chunk-PX2LSHYM.js.map} +0 -0
- /package/dist/{chunk-O5FFBO2M.js.map → chunk-QAVH6X2A.js.map} +0 -0
- /package/dist/{chunk-HDIUYMSB.js.map → chunk-QWGE2PUH.js.map} +0 -0
- /package/dist/{chunk-X57M2Q4I.js.map → chunk-RDJRFDJY.js.map} +0 -0
- /package/dist/{chunk-RYLEDKQ2.js.map → chunk-RUMNY5DT.js.map} +0 -0
- /package/dist/{chunk-BKIFTXHR.js.map → chunk-RZ4XGKOJ.js.map} +0 -0
- /package/dist/{chunk-66SE3BNH.js.map → chunk-SUPEYLH2.js.map} +0 -0
- /package/dist/{chunk-BF6SXNQJ.js.map → chunk-T2JHZWH6.js.map} +0 -0
- /package/dist/{chunk-QPOHMO6M.js.map → chunk-UES4BHA2.js.map} +0 -0
- /package/dist/{chunk-X7XEDOSL.js.map → chunk-VLL43QEQ.js.map} +0 -0
- /package/dist/{chunk-QBSFOKY6.js.map → chunk-VOELBUS4.js.map} +0 -0
- /package/dist/{chunk-2LY6GIIE.js.map → chunk-WDBX6VER.js.map} +0 -0
- /package/dist/{chunk-BMEYMKWX.js.map → chunk-WM65LU67.js.map} +0 -0
- /package/dist/{chunk-WLRD2TXF.js.map → chunk-WOPA3SGI.js.map} +0 -0
- /package/dist/{chunk-5UZ6UCOZ.js.map → chunk-XUAMCLY4.js.map} +0 -0
- /package/dist/{chunk-7HXZJAID.js.map → chunk-YGLY6QFV.js.map} +0 -0
- /package/dist/{chunk-2OIQDTDC.js.map → chunk-ZPJ4QSGP.js.map} +0 -0
- /package/dist/{chunk-3OY7KXHM.js.map → chunk-ZYLC7NFO.js.map} +0 -0
- /package/dist/{resolution-MDPNOEG6.js.map → resolution-MMOD5WC4.js.map} +0 -0
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { computeLegacyContentHash, normalizeLegacyContent } from "../content-hash.js";
|
|
2
3
|
import { inferMemoryStatus } from "../memory-lifecycle-ledger-utils.js";
|
|
3
4
|
import { ContentHashIndex, type ContentHashPathEntry } from "../storage/content-hash-index.js";
|
|
5
|
+
import { DEFAULT_CITATION_FORMAT, stripCitationForTemplate } from "../source-attribution.js";
|
|
6
|
+
import { storedContentIdentityCandidates } from "../structured-attributes.js";
|
|
4
7
|
import type { MemoryFrontmatter, MemoryStatus } from "../types.js";
|
|
5
8
|
import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
|
|
6
9
|
import {
|
|
@@ -14,11 +17,13 @@ import {
|
|
|
14
17
|
|
|
15
18
|
export const RECONCILE_MANIFEST_FORMAT = "remnic-reconcile-manifest";
|
|
16
19
|
export const RECONCILE_MANIFEST_SCHEMA_VERSION = 1;
|
|
20
|
+
const CONTENT_HASH_NORMALIZER_VERSION = 4;
|
|
17
21
|
|
|
18
22
|
export interface ReconcileMemoryIdentity {
|
|
19
23
|
id: string;
|
|
20
24
|
category: string;
|
|
21
25
|
contentHash: string;
|
|
26
|
+
normalizerVersion?: number;
|
|
22
27
|
status: MemoryStatus;
|
|
23
28
|
}
|
|
24
29
|
|
|
@@ -43,6 +48,7 @@ export interface BuildReconcileManifestOptions {
|
|
|
43
48
|
readFile: (file: ReconcileFileState) => Promise<Buffer | string | null>;
|
|
44
49
|
parseMemory: ReconcileMemoryParser;
|
|
45
50
|
cachedFiles?: Iterable<ReconcileManifestFile>;
|
|
51
|
+
citationTemplate?: string;
|
|
46
52
|
}
|
|
47
53
|
|
|
48
54
|
const SHA256_PATTERN = /^[a-f0-9]{64}$/i;
|
|
@@ -61,19 +67,38 @@ function parsedMemoryIdentity(
|
|
|
61
67
|
filePath: string,
|
|
62
68
|
raw: Buffer | string,
|
|
63
69
|
parseMemory: ReconcileMemoryParser,
|
|
70
|
+
citationTemplate: string,
|
|
64
71
|
): ReconcileMemoryIdentity | undefined {
|
|
65
72
|
if (!isMemoryPath(filePath)) return undefined;
|
|
66
73
|
const parsed = parseMemory(Buffer.isBuffer(raw) ? raw.toString("utf8") : raw);
|
|
67
74
|
if (!parsed?.frontmatter.id) return undefined;
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
75
|
+
|
|
76
|
+
const storedHash =
|
|
77
|
+
parsed.frontmatter.contentHash &&
|
|
78
|
+
SHA256_PATTERN.test(parsed.frontmatter.contentHash)
|
|
79
|
+
? parsed.frontmatter.contentHash.toLowerCase()
|
|
80
|
+
: undefined;
|
|
81
|
+
const candidates = storedContentIdentityCandidates(parsed.content, (content) =>
|
|
82
|
+
stripCitationForTemplate(content, citationTemplate)
|
|
83
|
+
);
|
|
84
|
+
const bodyHash = ContentHashIndex.computeHash(candidates[0] ?? "");
|
|
85
|
+
const currentMatch = candidates.find(
|
|
86
|
+
(content) => ContentHashIndex.computeHash(content) === storedHash
|
|
87
|
+
);
|
|
88
|
+
const legacyMatch = [...candidates].reverse().find(
|
|
89
|
+
(content) => computeLegacyContentHash(content) === storedHash
|
|
90
|
+
);
|
|
91
|
+
const contentHash = currentMatch
|
|
92
|
+
? ContentHashIndex.computeHash(currentMatch)
|
|
93
|
+
: legacyMatch
|
|
94
|
+
? ContentHashIndex.computeHash(legacyMatch)
|
|
95
|
+
: storedHash ?? bodyHash;
|
|
96
|
+
|
|
73
97
|
return {
|
|
74
98
|
id: parsed.frontmatter.id,
|
|
75
99
|
category: parsed.frontmatter.category,
|
|
76
100
|
contentHash,
|
|
101
|
+
normalizerVersion: CONTENT_HASH_NORMALIZER_VERSION,
|
|
77
102
|
status: inferMemoryStatus(parsed.frontmatter, filePath),
|
|
78
103
|
};
|
|
79
104
|
}
|
|
@@ -82,6 +107,7 @@ export async function buildReconcileManifestFile(
|
|
|
82
107
|
file: ReconcileFileState,
|
|
83
108
|
readFile: (file: ReconcileFileState) => Promise<Buffer | string | null>,
|
|
84
109
|
parseMemory: ReconcileMemoryParser,
|
|
110
|
+
citationTemplate = DEFAULT_CITATION_FORMAT,
|
|
85
111
|
): Promise<ReconcileManifestFile> {
|
|
86
112
|
let raw: Buffer | string | null = null;
|
|
87
113
|
if (isMemoryPath(file.path)) {
|
|
@@ -94,7 +120,7 @@ export async function buildReconcileManifestFile(
|
|
|
94
120
|
if (raw !== null && createHash("sha256").update(raw).digest("hex") !== file.sha256.toLowerCase()) {
|
|
95
121
|
raw = null;
|
|
96
122
|
}
|
|
97
|
-
const memory = raw === null ? undefined : parsedMemoryIdentity(file.path, raw, parseMemory);
|
|
123
|
+
const memory = raw === null ? undefined : parsedMemoryIdentity(file.path, raw, parseMemory, citationTemplate);
|
|
98
124
|
return { ...file, ...(memory ? { memory } : {}) };
|
|
99
125
|
}
|
|
100
126
|
|
|
@@ -107,12 +133,21 @@ export async function buildReconcileManifest(options: BuildReconcileManifestOpti
|
|
|
107
133
|
const files: ReconcileManifestFile[] = [];
|
|
108
134
|
for (const file of options.files) {
|
|
109
135
|
const cached = cachedByPath.get(file.path);
|
|
110
|
-
if (
|
|
136
|
+
if (
|
|
137
|
+
cached?.sha256.toLowerCase() === file.sha256.toLowerCase() &&
|
|
138
|
+
(cached.memory === undefined ||
|
|
139
|
+
cached.memory.normalizerVersion === CONTENT_HASH_NORMALIZER_VERSION)
|
|
140
|
+
) {
|
|
111
141
|
files.push({ ...file, ...(cached.memory ? { memory: cached.memory } : {}) });
|
|
112
142
|
continue;
|
|
113
143
|
}
|
|
114
144
|
|
|
115
|
-
files.push(await buildReconcileManifestFile(
|
|
145
|
+
files.push(await buildReconcileManifestFile(
|
|
146
|
+
file,
|
|
147
|
+
options.readFile,
|
|
148
|
+
options.parseMemory,
|
|
149
|
+
options.citationTemplate,
|
|
150
|
+
));
|
|
116
151
|
}
|
|
117
152
|
files.sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));
|
|
118
153
|
return {
|
|
@@ -132,10 +167,17 @@ function activeFactByPath(manifest: ReconcileManifest | undefined): Map<string,
|
|
|
132
167
|
return result;
|
|
133
168
|
}
|
|
134
169
|
|
|
135
|
-
function
|
|
170
|
+
function memoryIdentityHashes(memory: ReconcileMemoryIdentity): string[] {
|
|
171
|
+
return [memory.contentHash];
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function contentHashRows(
|
|
175
|
+
files: Iterable<ActiveFactManifestFile>,
|
|
176
|
+
contentHash: string,
|
|
177
|
+
): ContentHashPathEntry[] {
|
|
136
178
|
const rows: ContentHashPathEntry[] = [];
|
|
137
179
|
for (const file of files) {
|
|
138
|
-
rows.push({ path: file.path, contentHash
|
|
180
|
+
rows.push({ path: file.path, contentHash });
|
|
139
181
|
}
|
|
140
182
|
return rows;
|
|
141
183
|
}
|
|
@@ -194,35 +236,47 @@ export function collapseActiveFactDuplicates(
|
|
|
194
236
|
const peerByHash = new Map<string, ActiveFactManifestFile[]>();
|
|
195
237
|
|
|
196
238
|
for (const file of localByPath.values()) {
|
|
197
|
-
const hash
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
239
|
+
for (const hash of memoryIdentityHashes(file.memory)) {
|
|
240
|
+
const bucket = localByHash.get(hash) ?? [];
|
|
241
|
+
bucket.push(file);
|
|
242
|
+
localByHash.set(hash, bucket);
|
|
243
|
+
}
|
|
201
244
|
}
|
|
202
245
|
for (const file of peerByPath.values()) {
|
|
203
|
-
const hash
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
246
|
+
for (const hash of memoryIdentityHashes(file.memory)) {
|
|
247
|
+
const bucket = peerByHash.get(hash) ?? [];
|
|
248
|
+
bucket.push(file);
|
|
249
|
+
peerByHash.set(hash, bucket);
|
|
250
|
+
}
|
|
207
251
|
}
|
|
208
252
|
|
|
209
253
|
const removed = new Set<ReconcilePlanEntry>();
|
|
210
254
|
const replacements: ReconcilePlanEntry[] = [];
|
|
255
|
+
const processedPathPairs = new Set<string>();
|
|
211
256
|
for (const hash of new Set([...localByHash.keys(), ...peerByHash.keys()])) {
|
|
212
257
|
const localCandidates = (localByHash.get(hash) ?? []).filter((file) => {
|
|
213
258
|
const opposite = peerFilesByPath.get(file.path);
|
|
214
259
|
const activeOpposite = peerByPath.get(file.path);
|
|
215
|
-
return opposite === undefined ||
|
|
260
|
+
return opposite === undefined || (
|
|
261
|
+
activeOpposite !== undefined &&
|
|
262
|
+
memoryIdentityHashes(activeOpposite.memory).includes(hash)
|
|
263
|
+
);
|
|
216
264
|
});
|
|
217
265
|
const peerCandidates = (peerByHash.get(hash) ?? []).filter((file) => {
|
|
218
266
|
const opposite = localFilesByPath.get(file.path);
|
|
219
267
|
const activeOpposite = localByPath.get(file.path);
|
|
220
|
-
return opposite === undefined ||
|
|
268
|
+
return opposite === undefined || (
|
|
269
|
+
activeOpposite !== undefined &&
|
|
270
|
+
memoryIdentityHashes(activeOpposite.memory).includes(hash)
|
|
271
|
+
);
|
|
221
272
|
});
|
|
222
|
-
const localPath = ContentHashIndex.resolvePathByHash(hash, contentHashRows(localCandidates));
|
|
223
|
-
const peerPath = ContentHashIndex.resolvePathByHash(hash, contentHashRows(peerCandidates));
|
|
273
|
+
const localPath = ContentHashIndex.resolvePathByHash(hash, contentHashRows(localCandidates, hash));
|
|
274
|
+
const peerPath = ContentHashIndex.resolvePathByHash(hash, contentHashRows(peerCandidates, hash));
|
|
224
275
|
|
|
225
276
|
if (localPath && peerPath && localPath !== peerPath) {
|
|
277
|
+
const pathPair = `${localPath}\0${peerPath}`;
|
|
278
|
+
if (processedPathPairs.has(pathPair)) continue;
|
|
279
|
+
processedPathPairs.add(pathPair);
|
|
226
280
|
const localFile = localByPath.get(localPath);
|
|
227
281
|
const peerFile = peerByPath.get(peerPath);
|
|
228
282
|
if (!localFile || !peerFile) continue;
|
|
@@ -5,15 +5,11 @@
|
|
|
5
5
|
// part of its public API.
|
|
6
6
|
import { mkdir, stat } from "node:fs/promises";
|
|
7
7
|
import path from "node:path";
|
|
8
|
+
import { computeContentHash, normalizeContent } from "../content-hash.js";
|
|
8
9
|
import { log } from "../logger.js";
|
|
10
|
+
import { SecureStoreLockedError, readMaybeEncryptedFile, writeMaybeEncryptedFile } from "../secure-store/secure-fs.js";
|
|
9
11
|
import { isErrnoCode } from "../utils/errno.js";
|
|
10
12
|
import { withHeldFileLock } from "../utils/serialize-mutations.js";
|
|
11
|
-
import { normalizeContent, computeContentHash } from "../content-hash.js";
|
|
12
|
-
import {
|
|
13
|
-
SecureStoreLockedError,
|
|
14
|
-
readMaybeEncryptedFile,
|
|
15
|
-
writeMaybeEncryptedFile,
|
|
16
|
-
} from "../secure-store/secure-fs.js";
|
|
17
13
|
|
|
18
14
|
/**
|
|
19
15
|
* Stale threshold (ms) for the per-file advisory lock guarding a fact-hash
|
|
@@ -23,6 +19,23 @@ import {
|
|
|
23
19
|
* without breaking a legitimately in-progress publish.
|
|
24
20
|
*/
|
|
25
21
|
const CONTENT_HASH_INDEX_LOCK_STALE_MS = 30_000;
|
|
22
|
+
/** Version marker for the normalized Unicode hash index format. */
|
|
23
|
+
const CONTENT_HASH_INDEX_FORMAT_HEADER = "# remnic-content-hash-index:v2";
|
|
24
|
+
|
|
25
|
+
function parseHashIndex(raw: string): { versioned: boolean; hashes: Set<string> } {
|
|
26
|
+
const lines = raw
|
|
27
|
+
.split("\n")
|
|
28
|
+
.map((line) => line.trim())
|
|
29
|
+
.filter((line) => line.length > 0);
|
|
30
|
+
if (lines[0] !== CONTENT_HASH_INDEX_FORMAT_HEADER) {
|
|
31
|
+
return { versioned: false, hashes: new Set() };
|
|
32
|
+
}
|
|
33
|
+
return { versioned: true, hashes: new Set(lines.slice(1)) };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function serializeHashIndex(hashes: Set<string>): string {
|
|
37
|
+
return `${CONTENT_HASH_INDEX_FORMAT_HEADER}\n${[...hashes].join("\n")}\n`;
|
|
38
|
+
}
|
|
26
39
|
|
|
27
40
|
/**
|
|
28
41
|
* Durable-retry policy for a deferred content-hash reconcile save whose
|
|
@@ -70,7 +83,7 @@ export interface ContentHashIndexLockOptions {
|
|
|
70
83
|
export class FactHashIndexNotAuthoritativeError extends Error {
|
|
71
84
|
constructor(stateDir: string) {
|
|
72
85
|
super(
|
|
73
|
-
`fact-hash index is not authoritative: the cross-process rebuild lock for ${stateDir} could not be acquired within the bounded retry budget
|
|
86
|
+
`fact-hash index is not authoritative: the cross-process rebuild lock for ${stateDir} could not be acquired within the bounded retry budget`
|
|
74
87
|
);
|
|
75
88
|
this.name = "FactHashIndexNotAuthoritativeError";
|
|
76
89
|
}
|
|
@@ -98,6 +111,8 @@ export interface ContentHashPathEntry {
|
|
|
98
111
|
export class ContentHashIndex {
|
|
99
112
|
private hashes: Set<string> = new Set();
|
|
100
113
|
private dirty = false;
|
|
114
|
+
/** True when load() found the pre-Unicode, markerless on-disk format. */
|
|
115
|
+
private formatMigrationRequired = false;
|
|
101
116
|
/**
|
|
102
117
|
* Hashes explicitly removed since the last successful save (issue #1909 review
|
|
103
118
|
* round 8 thread 3). Tracked separately so `saveMergingWithDisk` can be
|
|
@@ -140,7 +155,6 @@ export class ContentHashIndex {
|
|
|
140
155
|
* without an O(file-size) read on the hot path.
|
|
141
156
|
*/
|
|
142
157
|
private lastSyncedFingerprint: DiskFingerprint | null = null;
|
|
143
|
-
|
|
144
158
|
constructor(
|
|
145
159
|
stateDir: string,
|
|
146
160
|
secureStoreKeyProvider: () => Buffer | null = () => null,
|
|
@@ -154,16 +168,24 @@ export class ContentHashIndex {
|
|
|
154
168
|
this.memoryDir = memoryDir;
|
|
155
169
|
this.lockOptions = lockOptions;
|
|
156
170
|
}
|
|
171
|
+
/** Whether a caller with an authoritative corpus must rebuild this legacy index. */
|
|
172
|
+
get requiresFormatMigration(): boolean {
|
|
173
|
+
return this.formatMigrationRequired;
|
|
174
|
+
}
|
|
157
175
|
|
|
158
176
|
/** Load existing hashes from disk. Safe to call multiple times. */
|
|
159
177
|
async load(): Promise<void> {
|
|
160
178
|
try {
|
|
161
179
|
const raw = await readMaybeEncryptedFile(this.filePath, this.secureStoreKeyProvider(), this.memoryDir);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
180
|
+
const parsed = parseHashIndex(raw);
|
|
181
|
+
this.formatMigrationRequired = !parsed.versioned;
|
|
182
|
+
if (!parsed.versioned) {
|
|
183
|
+
// Hashes from the pre-NFC format are unsafe after normalization changes.
|
|
184
|
+
// Ignore them until an authoritative corpus rebuild publishes v2.
|
|
185
|
+
this.hashes.clear();
|
|
186
|
+
log.warn(`content-hash index: ignored legacy unversioned index at ${this.filePath}`);
|
|
187
|
+
} else {
|
|
188
|
+
for (const hash of parsed.hashes) this.hashes.add(hash);
|
|
167
189
|
}
|
|
168
190
|
log.debug(`content-hash index: loaded ${this.hashes.size} hashes`);
|
|
169
191
|
} catch (err) {
|
|
@@ -292,14 +314,9 @@ export class ContentHashIndex {
|
|
|
292
314
|
// what we published; late deltas stay pending. Mirrors saveMergingWithDisk.
|
|
293
315
|
const publishedAdded = new Set<string>(this.added);
|
|
294
316
|
const publishedRemoved = new Set<string>(this.removed);
|
|
295
|
-
const serialized =
|
|
296
|
-
await writeMaybeEncryptedFile(
|
|
297
|
-
|
|
298
|
-
serialized,
|
|
299
|
-
this.secureStoreWriteKeyProvider(),
|
|
300
|
-
{},
|
|
301
|
-
this.memoryDir
|
|
302
|
-
);
|
|
317
|
+
const serialized = serializeHashIndex(this.hashes);
|
|
318
|
+
await writeMaybeEncryptedFile(this.filePath, serialized, this.secureStoreWriteKeyProvider(), {}, this.memoryDir);
|
|
319
|
+
this.formatMigrationRequired = false;
|
|
303
320
|
// Consume ONLY the deltas this overwrite published; anything that arrived
|
|
304
321
|
// during the awaits above remains pending for the next save.
|
|
305
322
|
for (const h of publishedAdded) this.added.delete(h);
|
|
@@ -360,15 +377,16 @@ export class ContentHashIndex {
|
|
|
360
377
|
const publishedRemoved = new Set<string>(this.removed);
|
|
361
378
|
const merged = new Set<string>(publishedAdded);
|
|
362
379
|
try {
|
|
363
|
-
const raw = await readMaybeEncryptedFile(
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
380
|
+
const raw = await readMaybeEncryptedFile(this.filePath, this.secureStoreKeyProvider(), this.memoryDir);
|
|
381
|
+
const parsed = parseHashIndex(raw);
|
|
382
|
+
if (parsed.versioned) {
|
|
383
|
+
for (const hash of parsed.hashes) {
|
|
384
|
+
// Keep prior/concurrent on-disk hashes EXCEPT the ones we removed.
|
|
385
|
+
if (!publishedRemoved.has(hash)) merged.add(hash);
|
|
386
|
+
}
|
|
387
|
+
} else {
|
|
388
|
+
// Do not carry hashes from the pre-NFC format into a v2 write.
|
|
389
|
+
log.warn(`content-hash index: discarded legacy unversioned entries from ${this.filePath}`);
|
|
372
390
|
}
|
|
373
391
|
} catch (err) {
|
|
374
392
|
if (err instanceof SecureStoreLockedError) throw err;
|
|
@@ -378,11 +396,12 @@ export class ContentHashIndex {
|
|
|
378
396
|
await mkdir(path.dirname(this.filePath), { recursive: true });
|
|
379
397
|
await writeMaybeEncryptedFile(
|
|
380
398
|
this.filePath,
|
|
381
|
-
|
|
399
|
+
serializeHashIndex(merged),
|
|
382
400
|
this.secureStoreWriteKeyProvider(),
|
|
383
401
|
{},
|
|
384
402
|
this.memoryDir
|
|
385
403
|
);
|
|
404
|
+
this.formatMigrationRequired = false;
|
|
386
405
|
// Consume ONLY the deltas we just published; deltas that arrived during
|
|
387
406
|
// the awaits above remain pending.
|
|
388
407
|
for (const h of publishedAdded) this.added.delete(h);
|
|
@@ -415,7 +434,7 @@ export class ContentHashIndex {
|
|
|
415
434
|
// the retry keeps re-attempting the locked publish until the addition
|
|
416
435
|
// lands on disk.
|
|
417
436
|
log.warn(
|
|
418
|
-
`content-hash index: lock not acquired for ${this.filePath}; deferring reconcile save (dirty retained, scheduling durable retry)
|
|
437
|
+
`content-hash index: lock not acquired for ${this.filePath}; deferring reconcile save (dirty retained, scheduling durable retry)`
|
|
419
438
|
);
|
|
420
439
|
this.scheduleReconcileRetry();
|
|
421
440
|
return;
|
|
@@ -431,7 +450,7 @@ export class ContentHashIndex {
|
|
|
431
450
|
// Published cleanly: any pending deferred retry is now satisfied.
|
|
432
451
|
this.cancelReconcileRetry();
|
|
433
452
|
}
|
|
434
|
-
}
|
|
453
|
+
}
|
|
435
454
|
);
|
|
436
455
|
log.debug(`content-hash index: reconcile-saved ${this.hashes.size} hashes`);
|
|
437
456
|
}
|
|
@@ -479,7 +498,7 @@ export class ContentHashIndex {
|
|
|
479
498
|
// so overwriting now could clobber its addition. Leave the index as-is
|
|
480
499
|
// and let the caller retry the authoritative rebuild on next use.
|
|
481
500
|
log.warn(
|
|
482
|
-
`content-hash index: lock not acquired for ${this.filePath}; deferring authoritative rebuild (index left non-authoritative, retried on next use)
|
|
501
|
+
`content-hash index: lock not acquired for ${this.filePath}; deferring authoritative rebuild (index left non-authoritative, retried on next use)`
|
|
483
502
|
);
|
|
484
503
|
return;
|
|
485
504
|
}
|
|
@@ -488,7 +507,7 @@ export class ContentHashIndex {
|
|
|
488
507
|
// is a full overwrite that supersedes any pending add/remove deltas.
|
|
489
508
|
await this.save();
|
|
490
509
|
published = true;
|
|
491
|
-
}
|
|
510
|
+
}
|
|
492
511
|
);
|
|
493
512
|
return published;
|
|
494
513
|
}
|
|
@@ -572,7 +591,7 @@ export class ContentHashIndex {
|
|
|
572
591
|
if (this.reconcileRetryAttempts >= maxAttempts) {
|
|
573
592
|
log.warn(
|
|
574
593
|
`content-hash index: exhausted ${maxAttempts} deferred reconcile-save retries for ${this.filePath}; ` +
|
|
575
|
-
`${this.added.size} addition(s) remain dirty in-memory (a process restart rebuilds the index from the durable corpus)
|
|
594
|
+
`${this.added.size} addition(s) remain dirty in-memory (a process restart rebuilds the index from the durable corpus)`
|
|
576
595
|
);
|
|
577
596
|
this.reconcileRetryAttempts = 0;
|
|
578
597
|
this.settleReconcileRetry();
|
|
@@ -681,10 +700,7 @@ export class ContentHashIndex {
|
|
|
681
700
|
* one hash per line and `has(content)` keeps its existing raw-content contract.
|
|
682
701
|
* Reconciliation supplies short-lived manifest rows where hash and path coexist.
|
|
683
702
|
*/
|
|
684
|
-
static resolvePathByHash(
|
|
685
|
-
hash: string,
|
|
686
|
-
entries: Iterable<ContentHashPathEntry>,
|
|
687
|
-
): string | undefined {
|
|
703
|
+
static resolvePathByHash(hash: string, entries: Iterable<ContentHashPathEntry>): string | undefined {
|
|
688
704
|
const canonicalHash = hash.toLowerCase();
|
|
689
705
|
if (!/^[a-f0-9]{64}$/.test(canonicalHash)) return undefined;
|
|
690
706
|
let canonicalPath: string | undefined;
|
|
@@ -60,7 +60,7 @@ export class EntityRefRepair {
|
|
|
60
60
|
*/
|
|
61
61
|
async gate(
|
|
62
62
|
fm: MemoryFrontmatter,
|
|
63
|
-
|
|
63
|
+
canonicalBody: string,
|
|
64
64
|
structuredAttributes?: Record<string, string>,
|
|
65
65
|
): Promise<TombstoneMatch | null> {
|
|
66
66
|
if (!this.deps.tombstonesEnabled()) return null;
|
|
@@ -73,7 +73,7 @@ export class EntityRefRepair {
|
|
|
73
73
|
? supersessionKeysForFact({ entityRef: fm.entityRef, structuredAttributes })
|
|
74
74
|
: [];
|
|
75
75
|
const match = applyTombstoneResurrectionGate(store, fm, {
|
|
76
|
-
normalizedText: ContentHashIndex.normalizeContent(
|
|
76
|
+
normalizedText: ContentHashIndex.normalizeContent(canonicalBody),
|
|
77
77
|
supersessionKeys,
|
|
78
78
|
namespace: this.deps.tombstonesNamespace(),
|
|
79
79
|
});
|
|
@@ -109,7 +109,7 @@ export class EntityRefRepair {
|
|
|
109
109
|
frontmatter: fm,
|
|
110
110
|
rewrite: async () => {
|
|
111
111
|
if (opts.regateFact && fm.category === "fact") {
|
|
112
|
-
await this.gate(fm, body,
|
|
112
|
+
await this.gate(fm, body, fm.structuredAttributes);
|
|
113
113
|
}
|
|
114
114
|
// Blocked-capture surface: a repair rewrite of a tombstone-blocked
|
|
115
115
|
// record must keep TombstoneBlockedCaptureIndex consistent.
|
|
@@ -378,6 +378,15 @@ export class MemoryReadStore {
|
|
|
378
378
|
);
|
|
379
379
|
}
|
|
380
380
|
|
|
381
|
+
async collectTombstoneMigrationPaths(): Promise<string[]> {
|
|
382
|
+
const tiers = await Promise.all([
|
|
383
|
+
this.collectActiveMemoryPaths(),
|
|
384
|
+
this.collectColdMemoryPaths(),
|
|
385
|
+
this.collectContainedMarkdownPaths([path.join(this.deps.baseDir, "archive")]),
|
|
386
|
+
]);
|
|
387
|
+
return tiers.flat();
|
|
388
|
+
}
|
|
389
|
+
|
|
381
390
|
async readMemoriesWindow(options: {
|
|
382
391
|
maxMemories?: number;
|
|
383
392
|
batchSize?: number;
|
|
@@ -7,15 +7,15 @@ import { log } from "../logger.js";
|
|
|
7
7
|
import type { MemoryFile, MemoryFrontmatter } from "../types.js";
|
|
8
8
|
import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
|
|
9
9
|
import { isErrnoCode } from "../utils/errno.js";
|
|
10
|
-
import { withHeldFileLock } from "../utils/serialize-mutations.js";
|
|
10
|
+
import type { withHeldFileLock } from "../utils/serialize-mutations.js";
|
|
11
11
|
import {
|
|
12
12
|
CONTENT_HASH_INDEX_RETRY_MAX_DELAY_MS,
|
|
13
13
|
ContentHashIndex,
|
|
14
14
|
type ContentHashIndexLockOptions,
|
|
15
15
|
} from "./content-hash-index.js";
|
|
16
16
|
import {
|
|
17
|
-
buildExplicitCaptureDedupKey,
|
|
18
17
|
TombstoneBlockedCaptureWriteLock,
|
|
18
|
+
buildExplicitCaptureDedupKey,
|
|
19
19
|
} from "./tombstone-blocked-capture-mutation.js";
|
|
20
20
|
import type { OfflineSyncMemoryParser } from "./tombstone-blocked-capture-sync.js";
|
|
21
21
|
export {
|
|
@@ -38,7 +38,6 @@ const MARKER_PROCESS_START_TOLERANCE_MS = 2_000;
|
|
|
38
38
|
const DOTNET_UNIX_EPOCH_TICKS = 621_355_968_000_000_000;
|
|
39
39
|
const MARKER_PROCESS_STARTED_AT_MS = Date.now() - process.uptime() * 1000;
|
|
40
40
|
|
|
41
|
-
|
|
42
41
|
type RebuildMarker = {
|
|
43
42
|
path: string;
|
|
44
43
|
committed: boolean;
|
|
@@ -49,7 +48,6 @@ type RebuildMarker = {
|
|
|
49
48
|
malformed?: boolean;
|
|
50
49
|
};
|
|
51
50
|
|
|
52
|
-
|
|
53
51
|
export type TombstoneBlockedCaptureIndexOptions = {
|
|
54
52
|
readonly stateDir: string;
|
|
55
53
|
readonly memoryDir: string;
|
|
@@ -326,7 +324,7 @@ export class TombstoneBlockedCaptureIndex {
|
|
|
326
324
|
await index.load();
|
|
327
325
|
const rebuildMarkers = await this.getRebuildMarkers();
|
|
328
326
|
const clearableRebuildMarkers = await this.getClearableRebuildMarkerPaths(rebuildMarkers);
|
|
329
|
-
let persisted =
|
|
327
|
+
let persisted = !index.requiresFormatMigration;
|
|
330
328
|
try {
|
|
331
329
|
await stat(this.indexPath());
|
|
332
330
|
if (rebuildMarkers.length > 0) persisted = false;
|
|
@@ -505,10 +503,7 @@ export class TombstoneBlockedCaptureIndex {
|
|
|
505
503
|
if (rebuildMarker) await this.markRebuildCommitted(rebuildMarker);
|
|
506
504
|
const rebuilt = await this.rebuild(await this.getIndex());
|
|
507
505
|
if (rebuilt) {
|
|
508
|
-
await this.clearRebuildRequired([
|
|
509
|
-
...clearableExistingMarkers,
|
|
510
|
-
marker,
|
|
511
|
-
]);
|
|
506
|
+
await this.clearRebuildRequired([...clearableExistingMarkers, marker]);
|
|
512
507
|
}
|
|
513
508
|
await this.setAuthoritative(rebuilt);
|
|
514
509
|
}
|
|
@@ -128,6 +128,19 @@ test("content-hash: normalizeContent produces stable text for equivalent inputs"
|
|
|
128
128
|
assert.equal(a, b, "normalizeContent must collapse whitespace consistently");
|
|
129
129
|
});
|
|
130
130
|
|
|
131
|
+
test("content-hash: distinct Japanese facts keep distinct hashes", () => {
|
|
132
|
+
assert.notEqual(
|
|
133
|
+
ContentHashIndex.computeHash("利用者は紅茶を好む。"),
|
|
134
|
+
ContentHashIndex.computeHash("利用者は珈琲を好む。"),
|
|
135
|
+
);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("content-hash: NFC and NFD forms share one hash", () => {
|
|
139
|
+
const nfc = "Café au lait";
|
|
140
|
+
const nfd = "Cafe\u0301 au lait";
|
|
141
|
+
assert.equal(ContentHashIndex.computeHash(nfc), ContentHashIndex.computeHash(nfd));
|
|
142
|
+
});
|
|
143
|
+
|
|
131
144
|
test("content-hash: the frontmatter contentHash matches computeHash of the raw content", async () => {
|
|
132
145
|
const { storage, cleanup } = await makeStorage();
|
|
133
146
|
try {
|
|
@@ -900,7 +900,7 @@ test("#2016: rebuildUnderLock refuses to publish while a peer holds the lock (no
|
|
|
900
900
|
await mkdir(stateDir, { recursive: true });
|
|
901
901
|
// A peer already committed a hash to disk.
|
|
902
902
|
const peerHash = ContentHashIndex.computeHash("peer-committed-fact");
|
|
903
|
-
await writeFile(path.join(stateDir, "fact-hashes.txt"),
|
|
903
|
+
await writeFile(path.join(stateDir, "fact-hashes.txt"), `# remnic-content-hash-index:v2\n${peerHash}\n`);
|
|
904
904
|
// The peer holds the advisory lock (fresh, non-stale, not stale-breakable).
|
|
905
905
|
const lockPath = path.join(stateDir, "fact-hashes.txt.lock");
|
|
906
906
|
await writeFile(lockPath, `99999 peer-owner ${new Date().toISOString()}\n`);
|