@steve02081504/fount-p2p 0.0.43 → 0.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -3
- package/dag/storage.mjs +157 -79
- package/discovery/nostr/census.mjs +5 -5
- package/discovery/nostr/census_monitor.mjs +2 -2
- package/discovery/nostr/constants.mjs +3 -0
- package/discovery/nostr/index.mjs +1 -1
- package/docs/nostr_relay_discovery.md +2 -0
- package/docs/runtime.md +18 -0
- package/docs/signaling.md +1 -0
- package/federation/part_query/runtime.mjs +4 -4
- package/index.mjs +2 -0
- package/mailbox/store.mjs +3 -22
- package/node/identity.mjs +6 -0
- package/node/instance.mjs +37 -5
- package/package.json +2 -2
- package/pages/index.mjs +4 -0
- package/timeline/prune.mjs +27 -23
- package/timeline/retention.mjs +38 -34
- package/utils/async_mutex.mjs +11 -4
- package/utils/shuffle.mjs +1 -1
- package/wire/adapter.mjs +1 -1
package/AGENTS.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
## Package layers (`@steve02081504/fount-p2p`)
|
|
4
4
|
|
|
5
5
|
| Layer | Directory | Role |
|
|
6
|
-
|
|
6
|
+
| --- | --- | --- |
|
|
7
7
|
| L0 | `core/` | IDs, logical entity, canonical JSON, bytes |
|
|
8
8
|
| L1 | `crypto/`, `wire/`, `schemas/` | Crypto, wire ingress, canonical validation |
|
|
9
9
|
| L2 | `node/` | `initNode`, identity, entity store, denylist, reputation, storage plugins |
|
|
@@ -19,7 +19,7 @@ Detail docs: [transports](docs/transports.md) · [mesh](docs/mesh.md) · [signal
|
|
|
19
19
|
### Runtime: isomorphic vs Node
|
|
20
20
|
|
|
21
21
|
| Surface | Runtime | Notes |
|
|
22
|
-
|
|
22
|
+
| --- | --- | --- |
|
|
23
23
|
| `core/*` | Node + browser | No `node:*` builtins |
|
|
24
24
|
| `crypto/crypto.mjs` | Node + browser | `@noble/hashes` + `@noble/curves` only — never `node:crypto` |
|
|
25
25
|
| `crypto/key.mjs` / `crypto/channel.mjs`, disk I/O, LAN/BT, `ws`, CLI / `startNode` | Node (+ Deno bridge) | Do not load the whole package via esm.sh in the browser |
|
|
@@ -31,6 +31,7 @@ Deno / native / BT: [runtime.md](docs/runtime.md).
|
|
|
31
31
|
## Conventions
|
|
32
32
|
|
|
33
33
|
- Prefer shared helpers under `utils/`, `core/`, `wire/subscribe`, `wire/adapter` — do not reimplement LRU/TTL/inflight/atomic-fs/shuffle.
|
|
34
|
+
- **JSONL read-modify-write:** do it inside `withAsyncMutex(jsonlMutexKey(path))`. `rewriteJsonlKeeping` / `appendJsonlSynced` / `writeJsonlSynced` take that per-file mutex themselves and are **not reentrant** — never call them while already holding the same key (use lock-free `writeJsonl` / `writeJsonlLines` instead). Rewrites preserve the original raw line (a `sanitize` passed to `rewriteJsonlKeeping` / `readJsonlEntries` only affects the value handed to the predicate) and skip the rename when nothing is dropped.
|
|
34
35
|
- **No pure-forward aliases:** do not add `fooText`/`fooAlias` that only `return foo(sameArgs)` when the callee already accepts those types (e.g. never wrap `sha256Hex` as `sha256TextHex`). Domain names must add logic or type narrowing, not just rename.
|
|
35
36
|
- **Heterogeneous backends:** normalize at the load boundary (e.g. `link/rtc/ice_local_hostname.mjs` wraps W3C RTC backends); call sites speak one contract.
|
|
36
37
|
- **File naming:** parent directory is scope — short child names. Tunables default `<dir>/tunables.json` (exception: `schemas/part_query.tunables.json`). Subpath `package.json` exports mirror filenames.
|
|
@@ -66,6 +67,7 @@ Deno / native / BT: [runtime.md](docs/runtime.md).
|
|
|
66
67
|
### Node / network
|
|
67
68
|
|
|
68
69
|
- **Node data:** `initNode({ nodeDir, entityStore? })` — `node.json`, `network.json`, `denylist.json`, `reputation.json`, `mailbox/`, `chunks/`. Default EntityStore: `{nodeDir}/entities/`. No `FOUNT_*` env knobs — subprocess IPC uses argv.
|
|
70
|
+
- **Storage vs runtime:** local identity (`ensureNodeSeed` / `getNodeHash` / `resolveLocalEntityHashFromRecoveryPubKeyHex`, all `node/storage` JSON helpers) must not require `initNode`. Configure the dir with `configureNodeStorage({ nodeDir })` (idempotent; cannot switch while running — `closeNode()` first) and only call `initNode` when the network should run. Hosts configure storage unconditionally so entities have a local identity with P2P disabled. [runtime.md](docs/runtime.md)
|
|
69
71
|
- **fount network:** shells use `startNode` / `ensureLinkToNode` / `sendToNodeLink` / rooms — never import `link/` internals or pick a transport. Providers: `registerLinkProvider` from `./link` or facade.
|
|
70
72
|
- **Link `level` vs discovery `priority`:** descending `level` picks data transport (`nostr` = −∞ last resort); ascending `priority` orders handshake/presence media only. [transports.md](docs/transports.md)
|
|
71
73
|
- **Mesh first / no versioning:** ≥N links (K acquaintances + N−K explore); discovery API is `listVisibleNodeHashes` + `connectToNode` only; no topic on the fount-network surface; no version/compat fields. [mesh.md](docs/mesh.md)
|
|
@@ -75,7 +77,7 @@ Deno / native / BT: [runtime.md](docs/runtime.md).
|
|
|
75
77
|
## Tunables JSON
|
|
76
78
|
|
|
77
79
|
| File | Directory |
|
|
78
|
-
|
|
80
|
+
| --- | --- |
|
|
79
81
|
| `tunables.json` | `transport/`, `infra/`, `reputation/`, `trust_graph/`, `mailbox/`, `governance/`, `dag/` |
|
|
80
82
|
| `part_query.tunables.json` | `schemas/` |
|
|
81
83
|
|
package/dag/storage.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer'
|
|
2
2
|
import { createReadStream, createWriteStream } from 'node:fs'
|
|
3
|
-
import { appendFile, mkdir, open,
|
|
4
|
-
import { dirname } from 'node:path'
|
|
3
|
+
import { appendFile, mkdir, open, unlink, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { dirname, resolve } from 'node:path'
|
|
5
5
|
import { createInterface } from 'node:readline'
|
|
6
6
|
import { Readable } from 'node:stream'
|
|
7
7
|
import { pipeline } from 'node:stream/promises'
|
|
@@ -29,26 +29,52 @@ function parseJsonlLine(line, sanitize) {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
/**
|
|
32
|
-
*
|
|
32
|
+
* 逐行读取 JSONL 原始行(不解析、不净化)。文件缺失(ENOENT)视为空流,
|
|
33
|
+
* 兼容 cleanup 竞态:群目录被删后台仍在尾巴上读它;其他读错误抛出。
|
|
34
|
+
* @param {string} filePath 文件路径
|
|
35
|
+
* @returns {AsyncGenerator<string>} 原始行(readline 已去掉行尾换行)
|
|
36
|
+
*/
|
|
37
|
+
async function* readJsonlRawLines(filePath) {
|
|
38
|
+
const input = createReadStream(filePath, { encoding: 'utf8' })
|
|
39
|
+
// stream 内部异步 open 失败会触发 'error' 事件;提前订阅避免 unhandled error,
|
|
40
|
+
// 真实错误仍由下方 for-await 抛出,由本函数统一收口。
|
|
41
|
+
input.on('error', () => { })
|
|
42
|
+
const lines = createInterface({ input, crlfDelay: Infinity })
|
|
43
|
+
try {
|
|
44
|
+
for await (const line of lines) yield line
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error?.code !== 'ENOENT') throw error
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* 读取 JSONL 为 `{ row, raw }` 列表,保留原始行以便无损写回。
|
|
53
|
+
* 空行/解析失败/净化抛错的行跳过;文件缺失(ENOENT)返回空数组,其他读错误抛出。
|
|
54
|
+
* @param {string} filePath 文件路径
|
|
55
|
+
* @param {{ sanitize?: (row: object) => object }} [options] 行净化
|
|
56
|
+
* @returns {Promise<Array<{ row: object, raw: string }>>} 行条目列表
|
|
57
|
+
*/
|
|
58
|
+
export async function readJsonlEntries(filePath, options = {}) {
|
|
59
|
+
const sanitize = options.sanitize ?? (row => row)
|
|
60
|
+
/** @type {Array<{ row: object, raw: string }>} */
|
|
61
|
+
const entries = []
|
|
62
|
+
for await (const raw of readJsonlRawLines(filePath)) {
|
|
63
|
+
const row = parseJsonlLine(raw, sanitize)
|
|
64
|
+
if (row != null) entries.push({ row, raw })
|
|
65
|
+
}
|
|
66
|
+
return entries
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* 读取 JSONL 文件并解析为对象数组;缺失(ENOENT)返回空数组,其他读错误抛出。
|
|
33
71
|
* @param {string} filePath 文件系统路径
|
|
34
72
|
* @param {{ sanitize?: (row: object) => object }} [options] 可选净化函数
|
|
35
73
|
* @returns {Promise<object[]>} 各行解析后的对象列表
|
|
36
74
|
*/
|
|
37
75
|
export async function readJsonl(filePath, options = {}) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const sanitize = options.sanitize ?? (row => row)
|
|
41
|
-
/** @type {object[]} */
|
|
42
|
-
const rows = []
|
|
43
|
-
for (const line of text.split('\n')) {
|
|
44
|
-
const row = parseJsonlLine(line, sanitize)
|
|
45
|
-
if (row) rows.push(row)
|
|
46
|
-
}
|
|
47
|
-
return rows
|
|
48
|
-
}
|
|
49
|
-
catch {
|
|
50
|
-
return []
|
|
51
|
-
}
|
|
76
|
+
const entries = await readJsonlEntries(filePath, options)
|
|
77
|
+
return entries.map(entry => entry.row).filter(Boolean)
|
|
52
78
|
}
|
|
53
79
|
|
|
54
80
|
/**
|
|
@@ -60,69 +86,93 @@ export async function readJsonl(filePath, options = {}) {
|
|
|
60
86
|
*/
|
|
61
87
|
export async function* readJsonlStream(filePath, options = {}) {
|
|
62
88
|
const sanitize = options.sanitize ?? (row => row)
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
input.on('error', () => { })
|
|
67
|
-
const lines = createInterface({ input, crlfDelay: Infinity })
|
|
68
|
-
try {
|
|
69
|
-
for await (const line of lines) {
|
|
70
|
-
const row = parseJsonlLine(line, sanitize)
|
|
71
|
-
if (row) yield row
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
if (error?.code !== 'ENOENT') throw error
|
|
89
|
+
for await (const line of readJsonlRawLines(filePath)) {
|
|
90
|
+
const row = parseJsonlLine(line, sanitize)
|
|
91
|
+
if (row) yield row
|
|
76
92
|
}
|
|
77
93
|
}
|
|
78
94
|
|
|
79
95
|
/**
|
|
80
|
-
* 流式过滤重写 JSONL:保留 `keep(row)===true`
|
|
96
|
+
* 流式过滤重写 JSONL:保留 `keep(row)===true` 的行,写回的是原始行字节,
|
|
97
|
+
* `sanitize` 只影响传给 `keep` 的值。
|
|
98
|
+
*
|
|
99
|
+
* 在 `jsonlMutexKey(filePath)` 互斥锁内执行;调用方不得已持有同一文件的锁(不可重入)。
|
|
81
100
|
* @param {string} filePath 目标路径
|
|
82
|
-
* @param {(row: object) => boolean} keep
|
|
101
|
+
* @param {(row: object) => boolean | Promise<boolean>} keep 保留谓词(可异步)
|
|
83
102
|
* @param {{ sanitize?: (row: object) => object }} [options] 读行净化
|
|
84
103
|
* @returns {Promise<{ kept: number, dropped: number }>} 统计
|
|
85
104
|
*/
|
|
86
105
|
export async function rewriteJsonlKeeping(filePath, keep, options = {}) {
|
|
87
|
-
|
|
88
|
-
|
|
106
|
+
return withAsyncMutex(jsonlMutexKey(filePath), () => rewriteJsonlKeepingUnlocked(filePath, keep, options))
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* `rewriteJsonlKeeping` 的无锁实现,调用方须已持有该文件互斥锁。
|
|
111
|
+
* @param {string} filePath 目标路径
|
|
112
|
+
* @param {(row: object) => boolean | Promise<boolean>} keep 保留谓词(可异步)
|
|
113
|
+
* @param {{ sanitize?: (row: object) => object }} [options] 读行净化
|
|
114
|
+
* @returns {Promise<{ kept: number, dropped: number }>} 统计
|
|
115
|
+
*/
|
|
116
|
+
async function rewriteJsonlKeepingUnlocked(filePath, keep, options = {}) {
|
|
117
|
+
const sanitize = options.sanitize ?? (row => row)
|
|
89
118
|
const temporaryPath = atomicTemporaryPath(filePath)
|
|
90
|
-
|
|
119
|
+
let tempCreated = false
|
|
120
|
+
/** @type {string[]} */
|
|
91
121
|
const buffer = []
|
|
92
122
|
let kept = 0
|
|
93
123
|
let dropped = 0
|
|
94
|
-
/** @returns {Promise<void>} */
|
|
124
|
+
/** @returns {Promise<void>} 将缓冲区原始行写入临时文件 */
|
|
95
125
|
const flush = async () => {
|
|
96
126
|
if (!buffer.length) return
|
|
97
|
-
|
|
98
|
-
for (const row of buffer)
|
|
99
|
-
block += `${JSON.stringify(row)}\n`
|
|
127
|
+
const block = buffer.map(line => `${line}\n`).join('')
|
|
100
128
|
await appendFile(temporaryPath, block, 'utf8')
|
|
129
|
+
tempCreated = true
|
|
101
130
|
buffer.length = 0
|
|
102
131
|
}
|
|
103
132
|
try {
|
|
104
|
-
for await (const
|
|
105
|
-
if (
|
|
106
|
-
|
|
133
|
+
for await (const raw of readJsonlRawLines(filePath)) {
|
|
134
|
+
if (!raw.trim()) continue
|
|
135
|
+
const row = parseJsonlLine(raw, sanitize)
|
|
136
|
+
if (row == null) {
|
|
137
|
+
dropped++
|
|
138
|
+
continue
|
|
139
|
+
}
|
|
140
|
+
if (await keep(row)) {
|
|
141
|
+
buffer.push(raw)
|
|
107
142
|
kept++
|
|
108
|
-
if (buffer.length >= WRITE_JSONL_CHUNK_LINES)
|
|
109
|
-
await flush()
|
|
143
|
+
if (buffer.length >= WRITE_JSONL_CHUNK_LINES) await flush()
|
|
110
144
|
}
|
|
111
145
|
else dropped++
|
|
146
|
+
}
|
|
112
147
|
await flush()
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
148
|
+
if (dropped === 0) {
|
|
149
|
+
if (tempCreated) await unlink(temporaryPath).catch(() => { })
|
|
150
|
+
return { kept, dropped }
|
|
151
|
+
}
|
|
152
|
+
await mkdir(dirname(filePath), { recursive: true })
|
|
153
|
+
if (!tempCreated) {
|
|
154
|
+
await writeFile(temporaryPath, '', 'utf8')
|
|
155
|
+
tempCreated = true
|
|
156
|
+
}
|
|
157
|
+
const fileHandle = await open(temporaryPath, 'r+')
|
|
158
|
+
try {
|
|
159
|
+
await fileHandle.sync()
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
await fileHandle.close()
|
|
163
|
+
}
|
|
116
164
|
await finalizeAtomicRename(temporaryPath, filePath)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
165
|
+
return { kept, dropped }
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
if (tempCreated) await unlink(temporaryPath).catch(() => { })
|
|
169
|
+
throw error
|
|
170
|
+
}
|
|
122
171
|
}
|
|
123
172
|
|
|
124
173
|
/**
|
|
125
|
-
* 读取 JSONL
|
|
174
|
+
* 读取 JSONL 末条可解析事件的 `id`(DAG tip);从文件尾部最多读 1 MiB,
|
|
175
|
+
* 跳过撕裂/坏行后取最后一个非空且 `id` 非 null 的行;空文件或缺失为 null。
|
|
126
176
|
* @param {string} filePath 文件路径
|
|
127
177
|
* @returns {Promise<string | null>} tip event id
|
|
128
178
|
*/
|
|
@@ -132,14 +182,20 @@ export async function readJsonlTipId(filePath) {
|
|
|
132
182
|
try {
|
|
133
183
|
const { size } = await fh.stat()
|
|
134
184
|
if (!size) return null
|
|
135
|
-
const chunk = Math.min(size,
|
|
185
|
+
const chunk = Math.min(size, 1_048_576)
|
|
136
186
|
const buffer = Buffer.alloc(chunk)
|
|
137
187
|
await fh.read(buffer, 0, chunk, size - chunk)
|
|
138
|
-
const lines = buffer.toString('utf8').split('\n')
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
188
|
+
const lines = buffer.toString('utf8').split('\n')
|
|
189
|
+
for (let index = lines.length - 1; index >= 0; index--) {
|
|
190
|
+
const line = lines[index].trim()
|
|
191
|
+
if (!line) continue
|
|
192
|
+
try {
|
|
193
|
+
const row = JSON.parse(line)
|
|
194
|
+
if (row && typeof row === 'object' && row.id != null) return String(row.id)
|
|
195
|
+
}
|
|
196
|
+
catch { /* 撕裂/坏行:继续向前找 */ }
|
|
197
|
+
}
|
|
198
|
+
return null
|
|
143
199
|
}
|
|
144
200
|
finally {
|
|
145
201
|
await fh.close()
|
|
@@ -151,30 +207,50 @@ export async function readJsonlTipId(filePath) {
|
|
|
151
207
|
}
|
|
152
208
|
|
|
153
209
|
/**
|
|
154
|
-
*
|
|
210
|
+
* 流式写入原始 JSONL 行(临时文件 + fsync + rename),每行自动补 `\n`。
|
|
211
|
+
* 不取锁——调用方需自行持有 `jsonlMutexKey(filePath)`。
|
|
155
212
|
* @param {string} filePath 目标路径
|
|
156
|
-
* @param {
|
|
213
|
+
* @param {Iterable<string>} lines 原始行(不含换行)
|
|
157
214
|
* @returns {Promise<void>}
|
|
158
215
|
*/
|
|
159
|
-
export async function
|
|
216
|
+
export async function writeJsonlLines(filePath, lines) {
|
|
160
217
|
const dir = dirname(filePath)
|
|
161
218
|
await mkdir(dir, { recursive: true })
|
|
162
219
|
const temporaryPath = atomicTemporaryPath(filePath)
|
|
163
|
-
/** @returns {Generator<string>}
|
|
164
|
-
function*
|
|
165
|
-
for (const
|
|
166
|
-
yield `${
|
|
220
|
+
/** @returns {Generator<string>} 带换行的行 */
|
|
221
|
+
function* withEol() {
|
|
222
|
+
for (const line of lines)
|
|
223
|
+
yield `${line}\n`
|
|
224
|
+
}
|
|
225
|
+
await pipeline(Readable.from(withEol()), createWriteStream(temporaryPath, { encoding: 'utf8' }))
|
|
226
|
+
const fileHandle = await open(temporaryPath, 'r+')
|
|
227
|
+
try {
|
|
228
|
+
await fileHandle.sync()
|
|
229
|
+
}
|
|
230
|
+
finally {
|
|
231
|
+
await fileHandle.close()
|
|
167
232
|
}
|
|
168
|
-
await pipeline(Readable.from(lines()), createWriteStream(temporaryPath, { encoding: 'utf8' }))
|
|
169
233
|
await finalizeAtomicRename(temporaryPath, filePath)
|
|
170
234
|
}
|
|
171
235
|
|
|
236
|
+
/**
|
|
237
|
+
* 流式重写 JSONL(临时文件 + fsync + rename),避免大数组 join 的内存峰值。
|
|
238
|
+
* 不取锁——`mailbox/store.mjs` 等调用方在自己的 `jsonlMutexKey` 锁内调用;
|
|
239
|
+
* 需要锁时用 `writeJsonlSynced`。
|
|
240
|
+
* @param {string} filePath 目标路径
|
|
241
|
+
* @param {object[]} records 行对象列表
|
|
242
|
+
* @returns {Promise<void>}
|
|
243
|
+
*/
|
|
244
|
+
export async function writeJsonl(filePath, records) {
|
|
245
|
+
return writeJsonlLines(filePath, records.map(record => JSON.stringify(record)))
|
|
246
|
+
}
|
|
247
|
+
|
|
172
248
|
/**
|
|
173
249
|
* @param {string} filePath JSONL 路径
|
|
174
|
-
* @returns {string}
|
|
250
|
+
* @returns {string} 进程内互斥键(路径已规范化为绝对路径)
|
|
175
251
|
*/
|
|
176
252
|
export function jsonlMutexKey(filePath) {
|
|
177
|
-
return `jsonl:${filePath}`
|
|
253
|
+
return `jsonl:${resolve(filePath)}`
|
|
178
254
|
}
|
|
179
255
|
|
|
180
256
|
/**
|
|
@@ -188,21 +264,23 @@ export async function writeJsonlSynced(filePath, records) {
|
|
|
188
264
|
}
|
|
189
265
|
|
|
190
266
|
/**
|
|
191
|
-
* 追加一行 JSONL 并 `fsync
|
|
267
|
+
* 追加一行 JSONL 并 `fsync`;与 rewrite 共用同一 per-file 互斥锁。
|
|
192
268
|
* @param {string} filePath 目标路径
|
|
193
269
|
* @param {object} record 记录对象
|
|
194
270
|
* @returns {Promise<void>}
|
|
195
271
|
*/
|
|
196
272
|
export async function appendJsonlSynced(filePath, record) {
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
273
|
+
return withAsyncMutex(jsonlMutexKey(filePath), async () => {
|
|
274
|
+
await mkdir(dirname(filePath), { recursive: true })
|
|
275
|
+
const fh = await open(filePath, 'a')
|
|
276
|
+
try {
|
|
277
|
+
await fh.appendFile(`${JSON.stringify(record)}\n`, 'utf8')
|
|
278
|
+
await fh.sync()
|
|
279
|
+
}
|
|
280
|
+
finally {
|
|
281
|
+
await fh.close()
|
|
282
|
+
}
|
|
283
|
+
})
|
|
206
284
|
}
|
|
207
285
|
|
|
208
286
|
/**
|
|
@@ -21,17 +21,17 @@ import { ensureNodeSeed, getNodeHash } from '../../node/identity.mjs'
|
|
|
21
21
|
import { getP2PFeatures, isNodeInitialized } from '../../node/instance.mjs'
|
|
22
22
|
import { nodeDebug } from '../../node/log.mjs'
|
|
23
23
|
|
|
24
|
-
import {
|
|
25
|
-
CENSUS_TAG_FOUNT,
|
|
26
|
-
CENSUS_TAG_X,
|
|
27
|
-
NOSTR_CENSUS_KIND,
|
|
28
|
-
} from './constants.mjs'
|
|
29
24
|
import {
|
|
30
25
|
CENSUS_TARGET_EVENTS,
|
|
31
26
|
estimatePopulation,
|
|
32
27
|
nextInclusionProbability,
|
|
33
28
|
} from './census_math.mjs'
|
|
34
29
|
import { CENSUS_TTL_MS, buildCensusMessage, verifyCensusBytes } from './census_verify.mjs'
|
|
30
|
+
import {
|
|
31
|
+
CENSUS_TAG_FOUNT,
|
|
32
|
+
CENSUS_TAG_X,
|
|
33
|
+
NOSTR_CENSUS_KIND,
|
|
34
|
+
} from './constants.mjs'
|
|
35
35
|
import { resolveRelayConnectTarget } from './relays.mjs'
|
|
36
36
|
|
|
37
37
|
/** census 订阅标签数组(发布用)。 */
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { base64ToBytes } from '../../core/bytes_codec.mjs'
|
|
12
12
|
|
|
13
|
+
import { estimatePopulation } from './census_math.mjs'
|
|
14
|
+
import { CENSUS_TTL_MS, verifyCensusBytes } from './census_verify.mjs'
|
|
13
15
|
import {
|
|
14
16
|
CENSUS_TAG_FOUNT,
|
|
15
17
|
CENSUS_TAG_X,
|
|
@@ -17,8 +19,6 @@ import {
|
|
|
17
19
|
NIP66_BOOTSTRAP_RELAYS,
|
|
18
20
|
NOSTR_CENSUS_KIND,
|
|
19
21
|
} from './constants.mjs'
|
|
20
|
-
import { estimatePopulation } from './census_math.mjs'
|
|
21
|
-
import { CENSUS_TTL_MS, verifyCensusBytes } from './census_verify.mjs'
|
|
22
22
|
|
|
23
23
|
/** NIP-66 relay 注册 kind。 */
|
|
24
24
|
const NIP66_KIND = 30166
|
|
@@ -18,8 +18,8 @@ import {
|
|
|
18
18
|
} from '../internal/signal_crypto.mjs'
|
|
19
19
|
import { noteDiscoveryPeerClue } from '../peer_clue.mjs'
|
|
20
20
|
|
|
21
|
-
import { DEFAULT_RELAY_URLS } from './constants.mjs'
|
|
22
21
|
import { createNostrCensus } from './census.mjs'
|
|
22
|
+
import { DEFAULT_RELAY_URLS } from './constants.mjs'
|
|
23
23
|
import {
|
|
24
24
|
getListenRelays,
|
|
25
25
|
getPeerRoute,
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
Nostr is the fount-network fallback transport (`link` level −∞, discovery `priority` 100). This document covers the **relay pool**, NIP-66 discovery, health scoring, advert relay-field signing, and handshake routing. Top-level surface: [signaling.md](signaling.md) · [transports.md](transports.md) · [mesh.md](mesh.md).
|
|
4
4
|
|
|
5
5
|
Source layout (`discovery/nostr/`):
|
|
6
|
+
|
|
6
7
|
- `index.mjs` — provider (presence / signal / advert subscriptions)
|
|
7
8
|
- `relays.mjs` — pool, health, NIP-66 discovery, normalization, persistence
|
|
8
9
|
- `selection.mjs` — handshake routing, backoff, fanout
|
|
@@ -72,6 +73,7 @@ Lower is better. `recordProbeSuccess` / `recordProbeFailure` / `recordPublishRes
|
|
|
72
73
|
## Handshake routing (`selection.mjs`)
|
|
73
74
|
|
|
74
75
|
`handshakeTargets(nodeHash, attempt)`:
|
|
76
|
+
|
|
75
77
|
- **Round 0**: peer-claimed `listenRelays` top 4 by composite score (own health + peer rtt); else local `workingRelays` top 4; else pinned top 4.
|
|
76
78
|
- **Round ≥1**: backoff `min(2000 · 2^(attempt−1), 60000)`; base on `lastGoodNostrRelays` (expanded via `expandFromHistory` ≤ 16), or weighted-random sample of `workingRelays` (weight `1/score`); round-0 core always included; fanout capped at `MAX_ROUTING_FANOUT` (64).
|
|
77
79
|
- Retries ≤ `MAX_ROUTING_ATTEMPTS` (4).
|
package/docs/runtime.md
CHANGED
|
@@ -2,6 +2,24 @@
|
|
|
2
2
|
|
|
3
3
|
`ensureRuntime`, startup/shutdown budgets, and Bluetooth hardware probe. Day-to-day shell rules: [AGENTS.md](../AGENTS.md). Providers: [transports.md](transports.md). Mesh N/K: [mesh.md](mesh.md).
|
|
4
4
|
|
|
5
|
+
## Offline local identity (storage without a running node)
|
|
6
|
+
|
|
7
|
+
Three independent stages — do not couple them:
|
|
8
|
+
|
|
9
|
+
1. **Storage** — `configureNodeStorage({ nodeDir })` records where `node.json` / identity seed live. No runtime, no disk I/O, no discovery.
|
|
10
|
+
2. **Local identity** — `ensureNodeSeed` / `getNodeHash` / `resolveLocalEntityHashFromRecoveryPubKeyHex` (and `node/storage` JSON helpers) only need stage 1, so a local `entityHash` can be created while federation is off.
|
|
11
|
+
3. **Network** — `initNode({ nodeDir, entityStore })` builds the runtime and link graph. Same `nodeDir` as stage 1 reuses the persisted seed and hashes.
|
|
12
|
+
|
|
13
|
+
```javascript
|
|
14
|
+
import { configureNodeStorage } from '@steve02081504/fount-p2p/node/instance'
|
|
15
|
+
import { resolveLocalEntityHashFromRecoveryPubKeyHex } from '@steve02081504/fount-p2p/node/identity'
|
|
16
|
+
|
|
17
|
+
configureNodeStorage({ nodeDir: '/data/p2p/node' })
|
|
18
|
+
const entityHash = resolveLocalEntityHashFromRecoveryPubKeyHex(recoveryPubKeyHex) // works with P2P disabled
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
`configureNodeStorage` is idempotent for the same dir; switching dirs while a node is running is rejected (call `closeNode()` first). `closeNode()` clears the storage config, so re-configure after closing. Consumer hosts (fount, subfount) call `configureNodeStorage` unconditionally at startup and only call `initNode` when P2P is enabled.
|
|
22
|
+
|
|
5
23
|
## `ensureRuntime` contract
|
|
6
24
|
|
|
7
25
|
Returns after registering lan / nostr discovery providers (subject to the signaling `channels` whitelist), registering all enabled link providers (including `ble_gatt`), and scheduling background warm — does **not** await lan_tcp listen, Nostr relays, or BT availability.
|
package/docs/signaling.md
CHANGED
|
@@ -25,6 +25,7 @@ When set: send final offer/answer after ICE gathering, dedupe remote signals, qu
|
|
|
25
25
|
`channels` selects which discovery/link media are active — `nostr`, `lan`, `bt`, `webrtc`. Semantics per channel key: `false` **disables**; any other value (`undefined`, `true`, or an object) **enables** — an object is merged over that channel's default config. Channels not mentioned keep their default enabled state.
|
|
26
26
|
|
|
27
27
|
Channel → components:
|
|
28
|
+
|
|
28
29
|
- `nostr` (nostr discovery + nostr link) — config `relay` **replaces** the default public relay list (do not merge defaults back in).
|
|
29
30
|
- `lan` (lan discovery + lan_tcp link)
|
|
30
31
|
- `bt` (bt discovery + ble_gatt link)
|
|
@@ -227,7 +227,7 @@ function mergeRowsWithSources(entries, maxHits, rowKey) {
|
|
|
227
227
|
try { return JSON.stringify(row) }
|
|
228
228
|
catch { return `\0${rows.length}` }
|
|
229
229
|
})
|
|
230
|
-
for (const entry of entries)
|
|
230
|
+
for (const entry of entries)
|
|
231
231
|
for (const row of entry.rows || []) {
|
|
232
232
|
const key = keyOf(row)
|
|
233
233
|
if (!seen.has(key)) {
|
|
@@ -240,7 +240,7 @@ function mergeRowsWithSources(entries, maxHits, rowKey) {
|
|
|
240
240
|
}
|
|
241
241
|
if (entry.sourceNodeHash) sources.get(key).add(entry.sourceNodeHash)
|
|
242
242
|
}
|
|
243
|
-
|
|
243
|
+
|
|
244
244
|
return { rows, sources }
|
|
245
245
|
}
|
|
246
246
|
|
|
@@ -325,10 +325,10 @@ function filterRowsBySource(rows, sources, rowKey, isSourceBlocked) {
|
|
|
325
325
|
return rows.filter(row => {
|
|
326
326
|
let key
|
|
327
327
|
if (rowKey) key = rowKey(row)
|
|
328
|
-
else
|
|
328
|
+
else
|
|
329
329
|
try { key = JSON.stringify(row) }
|
|
330
330
|
catch { return true }
|
|
331
|
-
|
|
331
|
+
|
|
332
332
|
const set = sources.get(key)
|
|
333
333
|
if (!set || set.size === 0) return true
|
|
334
334
|
for (const source of set) if (!isSourceBlocked(source)) return true
|
package/index.mjs
CHANGED
|
@@ -13,6 +13,7 @@ import { registerLinkProvider } from './link/providers/index.mjs'
|
|
|
13
13
|
import { ensureNodeDefaults, getNodeHash } from './node/identity.mjs'
|
|
14
14
|
import {
|
|
15
15
|
closeNode,
|
|
16
|
+
configureNodeStorage,
|
|
16
17
|
getNodeDir,
|
|
17
18
|
getP2PFeatures,
|
|
18
19
|
initNode,
|
|
@@ -65,6 +66,7 @@ export {
|
|
|
65
66
|
attachNodeScopeDefaultFeatures,
|
|
66
67
|
closeNode,
|
|
67
68
|
configureLinkRegistry,
|
|
69
|
+
configureNodeStorage,
|
|
68
70
|
createGroupLinkSet,
|
|
69
71
|
createScopedLinkRoom,
|
|
70
72
|
ensureChannelAvailable,
|
package/mailbox/store.mjs
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import { mkdir, readFile } from 'node:fs/promises'
|
|
2
|
-
import { dirname } from 'node:path'
|
|
3
|
-
|
|
4
1
|
import { isHex64 } from '../core/hexIds.mjs'
|
|
5
|
-
import { jsonlMutexKey, writeJsonl } from '../dag/storage.mjs'
|
|
2
|
+
import { jsonlMutexKey, readJsonl, writeJsonl } from '../dag/storage.mjs'
|
|
6
3
|
import { mailboxStorePath } from '../node/user_paths.mjs'
|
|
7
4
|
import { withAsyncMutex } from '../utils/async_mutex.mjs'
|
|
8
5
|
|
|
@@ -103,21 +100,7 @@ export function relayHopAfterWireIngress(wireHop, existingStoredHop) {
|
|
|
103
100
|
* @returns {Promise<MailboxRecord[]>} 全部有效记录
|
|
104
101
|
*/
|
|
105
102
|
async function readAll() {
|
|
106
|
-
|
|
107
|
-
const text = await readFile(mailboxStorePath(), 'utf8')
|
|
108
|
-
/** @type {MailboxRecord[]} */
|
|
109
|
-
const rows = []
|
|
110
|
-
for (const line of text.split('\n')) {
|
|
111
|
-
const trimmed = line.trim()
|
|
112
|
-
if (!trimmed) continue
|
|
113
|
-
try {
|
|
114
|
-
rows.push(JSON.parse(trimmed))
|
|
115
|
-
}
|
|
116
|
-
catch { /* skip corrupt line */ }
|
|
117
|
-
}
|
|
118
|
-
return rows
|
|
119
|
-
}
|
|
120
|
-
catch { return [] }
|
|
103
|
+
return /** @type {MailboxRecord[]} */ await readJsonl(mailboxStorePath())
|
|
121
104
|
}
|
|
122
105
|
|
|
123
106
|
/**
|
|
@@ -125,13 +108,11 @@ async function readAll() {
|
|
|
125
108
|
* @returns {Promise<void>}
|
|
126
109
|
*/
|
|
127
110
|
async function writeAll(rows) {
|
|
128
|
-
const filePath = mailboxStorePath()
|
|
129
|
-
await mkdir(dirname(filePath), { recursive: true })
|
|
130
111
|
const now = Date.now()
|
|
131
112
|
const kept = pruneMailboxGlobalFair(
|
|
132
113
|
pruneMailboxBuckets(sortMailboxForRetention(rows.filter(record => record.expiresAt > now))),
|
|
133
114
|
)
|
|
134
|
-
await writeJsonl(
|
|
115
|
+
await writeJsonl(mailboxStorePath(), kept)
|
|
135
116
|
}
|
|
136
117
|
|
|
137
118
|
/**
|
package/node/identity.mjs
CHANGED
|
@@ -42,6 +42,8 @@ function saveNodeFile(patch) {
|
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
/**
|
|
45
|
+
* 确保本地节点 seed 已持久化并返回。
|
|
46
|
+
* 只依赖本地存储目录(`configureNodeStorage` 或 `initNode` 提供),不要求运行中的节点。
|
|
45
47
|
* @returns {string} 64 位十六进制 节点种子
|
|
46
48
|
*/
|
|
47
49
|
export function ensureNodeSeed() {
|
|
@@ -54,6 +56,8 @@ export function ensureNodeSeed() {
|
|
|
54
56
|
}
|
|
55
57
|
|
|
56
58
|
/**
|
|
59
|
+
* 由本地 seed 派生本节点 nodeHash。
|
|
60
|
+
* 只依赖本地存储目录,不要求运行中的节点。
|
|
57
61
|
* @returns {string} 本节点 64 hex nodeHash
|
|
58
62
|
*/
|
|
59
63
|
export function getNodeHash() {
|
|
@@ -111,6 +115,8 @@ export function entityHashFromKeys(nodeHash, recoveryPubKeyHex) {
|
|
|
111
115
|
}
|
|
112
116
|
|
|
113
117
|
/**
|
|
118
|
+
* 由本地 nodeHash 与 recovery 公钥派生本节点 entityHash。
|
|
119
|
+
* 只依赖本地存储目录,不要求运行中的节点(联邦未启动时也可创建本地身份)。
|
|
114
120
|
* @param {string} recoveryPubKeyHex 64 位十六进制 recovery 公钥
|
|
115
121
|
* @returns {string | null} 本节点 entityHash
|
|
116
122
|
*/
|
package/node/instance.mjs
CHANGED
|
@@ -22,6 +22,13 @@ import { defaultSignalingRuntimeConfig, resolveSignalingRuntimeConfig } from './
|
|
|
22
22
|
/** @type {NodeRuntime | null} */
|
|
23
23
|
let runtime = null
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* 本地节点数据目录。与 `runtime` 解耦:身份 seed 等纯本地持久化只依赖存储目录,
|
|
27
|
+
* 不要求 `initNode()`(联邦未启动时也要能派生本地身份)。
|
|
28
|
+
* @type {string | null}
|
|
29
|
+
*/
|
|
30
|
+
let nodeStorageDir = null
|
|
31
|
+
|
|
25
32
|
/** @type {Set<(event: string, payload?: unknown) => void>} */
|
|
26
33
|
const changeListeners = new Set()
|
|
27
34
|
|
|
@@ -35,6 +42,28 @@ export function getRtcPolyfillCacheEpoch() {
|
|
|
35
42
|
return rtcPolyfillCacheEpoch
|
|
36
43
|
}
|
|
37
44
|
|
|
45
|
+
/**
|
|
46
|
+
* 配置本地节点数据目录(`node.json` / 身份 seed 等纯本地持久化位置),
|
|
47
|
+
* 不初始化节点运行时、不创建 disk 资源、不启动发现或连接。
|
|
48
|
+
*
|
|
49
|
+
* 配置后 `getNodeDir` / `ensureNodeSeed` / `getNodeHash` /
|
|
50
|
+
* `resolveLocalEntityHashFromRecoveryPubKeyHex` 可在无运行节点时使用;
|
|
51
|
+
* 之后以同一目录调用 `initNode` 会复用已持久化的 seed 与身份。
|
|
52
|
+
*
|
|
53
|
+
* 同目录重复配置为幂等;运行中的节点不允许切换到不同目录,须先 `closeNode()`。
|
|
54
|
+
* @param {{ nodeDir: string }} options - 本地节点数据目录
|
|
55
|
+
* @returns {string} 规范化为绝对路径后的目录
|
|
56
|
+
*/
|
|
57
|
+
export function configureNodeStorage(options) {
|
|
58
|
+
const rawDir = options?.nodeDir
|
|
59
|
+
if (typeof rawDir !== 'string' || !rawDir) throw new Error('p2p: configureNodeStorage requires nodeDir')
|
|
60
|
+
const nodeDir = path.resolve(rawDir)
|
|
61
|
+
if (runtime && runtime.nodeDir !== nodeDir)
|
|
62
|
+
throw new Error('p2p: running node uses another nodeDir — call closeNode() before reconfiguring storage')
|
|
63
|
+
nodeStorageDir = nodeDir
|
|
64
|
+
return nodeDir
|
|
65
|
+
}
|
|
66
|
+
|
|
38
67
|
/**
|
|
39
68
|
* @param {{ nodeDir: string, entityStore?: import('./entity_store.mjs').EntityStore }} options - 节点目录与可选 entity store
|
|
40
69
|
* @returns {NodeRuntime} 初始化后的运行时
|
|
@@ -43,7 +72,7 @@ export function initNode(options) {
|
|
|
43
72
|
if (runtime) throw new Error('p2p: initNode already called — use setNodeLogger / setSignalingRuntimeConfig or closeNode')
|
|
44
73
|
if (options?.logger !== undefined || options?.signaling !== undefined)
|
|
45
74
|
throw new Error('p2p: initNode only accepts nodeDir/entityStore — use setNodeLogger / setSignalingRuntimeConfig')
|
|
46
|
-
const nodeDir =
|
|
75
|
+
const nodeDir = options?.nodeDir == null ? nodeStorageDir : configureNodeStorage({ nodeDir: options.nodeDir })
|
|
47
76
|
if (!nodeDir) throw new Error('p2p: initNode requires nodeDir')
|
|
48
77
|
const entityStore = options.entityStore ?? createFsEntityStore(path.join(nodeDir, 'entities'))
|
|
49
78
|
runtime = {
|
|
@@ -123,10 +152,12 @@ export function getP2PFeatures() {
|
|
|
123
152
|
}
|
|
124
153
|
|
|
125
154
|
/**
|
|
126
|
-
* @returns {string}
|
|
155
|
+
* @returns {string} 节点数据目录绝对路径;未配置且节点未初始化时抛错
|
|
127
156
|
*/
|
|
128
157
|
export function getNodeDir() {
|
|
129
|
-
|
|
158
|
+
if (runtime) return runtime.nodeDir
|
|
159
|
+
if (nodeStorageDir) return nodeStorageDir
|
|
160
|
+
throw new Error('p2p: node storage not configured — call configureNodeStorage() or initNode() first')
|
|
130
161
|
}
|
|
131
162
|
|
|
132
163
|
/**
|
|
@@ -164,12 +195,13 @@ export function onNodeChange(listener) {
|
|
|
164
195
|
}
|
|
165
196
|
|
|
166
197
|
/**
|
|
167
|
-
* 关闭节点:释放全部文件句柄(chunk
|
|
168
|
-
* 之后可用 initNode 重新引导。
|
|
198
|
+
* 关闭节点:释放全部文件句柄(chunk 读/写流等)并清空运行时、存储目录配置与监听器。
|
|
199
|
+
* 之后可用 initNode / configureNodeStorage 重新引导。
|
|
169
200
|
* @returns {Promise<void>}
|
|
170
201
|
*/
|
|
171
202
|
export async function closeNode() {
|
|
172
203
|
runtime = null
|
|
204
|
+
nodeStorageDir = null
|
|
173
205
|
changeListeners.clear()
|
|
174
206
|
rtcPolyfillCacheEpoch++
|
|
175
207
|
await closeAllFileStreams()
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@steve02081504/fount-p2p",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.46",
|
|
4
4
|
"description": "fount federation P2P layer — link, trust graph, mailbox, DAG, EVFS.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"network",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"repository": {
|
|
16
16
|
"type": "git",
|
|
17
17
|
"url": "git+https://github.com/steve02081504/fount-p2p.git",
|
|
18
|
-
"directory": "
|
|
18
|
+
"directory": "js"
|
|
19
19
|
},
|
|
20
20
|
"engines": {
|
|
21
21
|
"node": ">=20"
|
package/pages/index.mjs
CHANGED
|
@@ -32,6 +32,10 @@ function startLive() {
|
|
|
32
32
|
try {
|
|
33
33
|
monitor = createPopulationMonitor({
|
|
34
34
|
relays: relayUrl ? [relayUrl] : undefined,
|
|
35
|
+
/**
|
|
36
|
+
* @param {{ estimate: number, sampleSize: number, eventsInWindow: number, relayUrl: string, relays: number }} snapshot 人口快照
|
|
37
|
+
* @returns {void}
|
|
38
|
+
*/
|
|
35
39
|
onUpdate: snapshot => {
|
|
36
40
|
render(snapshot)
|
|
37
41
|
statusEl.textContent = `监听 ${snapshot.relayUrl}(census,共 ${snapshot.relays} 个 relay)`
|
package/timeline/prune.mjs
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { mkdir } from 'node:fs/promises'
|
|
2
|
-
import { dirname } from 'node:path'
|
|
3
|
-
|
|
4
1
|
import { topologicalCanonicalOrder } from '../dag/index.mjs'
|
|
5
|
-
import {
|
|
2
|
+
import { jsonlMutexKey, readJsonlEntries, writeJsonlLines } from '../dag/storage.mjs'
|
|
6
3
|
import { invalidateTopologicalOrderMemo } from '../federation/topo_order_memo.mjs'
|
|
7
4
|
import { descendantClosureFromTip } from '../governance/branch.mjs'
|
|
5
|
+
import { withAsyncMutex } from '../utils/async_mutex.mjs'
|
|
8
6
|
|
|
9
7
|
/**
|
|
10
8
|
* 将 events.jsonl 裁剪为 checkpoint 起后代闭包(连通子图,非拓扑下标切片)。
|
|
9
|
+
*
|
|
10
|
+
* 整个读-算-写在 `jsonlMutexKey(eventsFilePath)` 锁内完成,避免与并发 append 竞态;
|
|
11
|
+
* 写回的是保留行的原始字节(非 sanitize 后的重序列化),保持无损失。
|
|
11
12
|
* @param {string} eventsFilePath events.jsonl 路径
|
|
12
13
|
* @param {object | null} checkpoint 含 checkpoint_event_id 的快照
|
|
13
14
|
* @param {(row: object) => object} [sanitize] 行规范化
|
|
@@ -16,24 +17,27 @@ import { descendantClosureFromTip } from '../governance/branch.mjs'
|
|
|
16
17
|
export async function pruneEventsJsonlAfterCheckpoint(eventsFilePath, checkpoint, sanitize = row => row) {
|
|
17
18
|
const tipId = checkpoint?.checkpoint_event_id
|
|
18
19
|
if (!tipId) return { pruned: false, kept: 0, dropped: 0 }
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
return withAsyncMutex(jsonlMutexKey(eventsFilePath), async () => {
|
|
21
|
+
const entries = await readJsonlEntries(eventsFilePath, { sanitize })
|
|
22
|
+
if (!entries.length) return { pruned: false, kept: 0, dropped: 0 }
|
|
23
|
+
const byId = new Map(entries.map(entry => [entry.row.id, entry.row]))
|
|
24
|
+
const rawById = new Map(entries.map(entry => [entry.row.id, entry.raw]))
|
|
25
|
+
if (!byId.has(tipId)) return { pruned: false, kept: entries.length, dropped: 0 }
|
|
23
26
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
27
|
+
const keepIds = descendantClosureFromTip(tipId, byId)
|
|
28
|
+
const order = topologicalCanonicalOrder(entries.map(entry => ({
|
|
29
|
+
id: entry.row.id,
|
|
30
|
+
prev_event_ids: entry.row.prev_event_ids,
|
|
31
|
+
hlc: entry.row.hlc,
|
|
32
|
+
node_id: entry.row.node_id,
|
|
33
|
+
sender: entry.row.sender,
|
|
34
|
+
})))
|
|
35
|
+
const keptIds = order.filter(id => keepIds.has(id))
|
|
36
|
+
const dropped = entries.length - keptIds.length
|
|
37
|
+
if (dropped <= 0) return { pruned: false, kept: keptIds.length, dropped: 0 }
|
|
38
|
+
const keptRaws = keptIds.map(id => rawById.get(id)).filter(raw => raw !== undefined)
|
|
39
|
+
await writeJsonlLines(eventsFilePath, keptRaws)
|
|
40
|
+
invalidateTopologicalOrderMemo(eventsFilePath)
|
|
41
|
+
return { pruned: true, kept: keptIds.length, dropped }
|
|
42
|
+
})
|
|
39
43
|
}
|
package/timeline/retention.mjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { mkdir } from 'node:fs/promises'
|
|
2
|
-
import { dirname } from 'node:path'
|
|
3
|
-
|
|
4
1
|
import { topologicalCanonicalOrder } from '../dag/index.mjs'
|
|
5
|
-
import {
|
|
2
|
+
import { jsonlMutexKey, readJsonlEntries, writeJsonlLines } from '../dag/storage.mjs'
|
|
6
3
|
import { invalidateTopologicalOrderMemo } from '../federation/topo_order_memo.mjs'
|
|
7
4
|
import { computeDagTipIdsFromEvents, selectConsensusBranchTip } from '../governance/branch.mjs'
|
|
8
5
|
import { computeRetentionKeepIds } from '../node/retention_policy.mjs'
|
|
6
|
+
import { withAsyncMutex } from '../utils/async_mutex.mjs'
|
|
9
7
|
|
|
10
8
|
/**
|
|
9
|
+
* 整个读-算-写在 `jsonlMutexKey(eventsFilePath)` 锁内完成,避免与并发 append 竞态;
|
|
10
|
+
* 写回的是保留行的原始字节(非 sanitize 后的重序列化),保持无损失。
|
|
11
11
|
* @param {string} eventsFilePath events.jsonl 路径
|
|
12
12
|
* @param {object | null} checkpointHint 检查点提示
|
|
13
13
|
* @param {{ maxDepth: number, maxMs: number, anchorTypes: Set<string> }} policy 保留策略
|
|
@@ -20,35 +20,39 @@ export async function enforceTimelineEventRetention(
|
|
|
20
20
|
policy,
|
|
21
21
|
sanitize = row => row,
|
|
22
22
|
) {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
23
|
+
return withAsyncMutex(jsonlMutexKey(eventsFilePath), async () => {
|
|
24
|
+
const entries = await readJsonlEntries(eventsFilePath, { sanitize })
|
|
25
|
+
if (!entries.length) return { pruned: false, kept: 0, dropped: 0 }
|
|
26
|
+
const maxDepth = Math.max(256, Number(policy.maxDepth) || 200_000)
|
|
27
|
+
const maxMs = Math.max(3_600_000, Number(policy.maxMs) || 365 * 24 * 3600 * 1000)
|
|
28
|
+
const cutoffWall = Date.now() - maxMs
|
|
29
|
+
const events = entries.map(entry => entry.row)
|
|
30
|
+
const rawById = new Map(entries.map(entry => [entry.row.id, entry.raw]))
|
|
31
|
+
const byId = new Map(events.map(e => [e.id, e]))
|
|
32
|
+
const order = topologicalCanonicalOrder(events.map(e => ({
|
|
33
|
+
id: e.id,
|
|
34
|
+
prev_event_ids: e.prev_event_ids,
|
|
35
|
+
hlc: e.hlc,
|
|
36
|
+
node_id: e.node_id,
|
|
37
|
+
sender: e.sender,
|
|
38
|
+
})))
|
|
39
|
+
const tips = computeDagTipIdsFromEvents(events)
|
|
40
|
+
const branchTipId = selectConsensusBranchTip(tips, byId)
|
|
41
|
+
const checkpointTipId = checkpointHint?.checkpoint_event_id || null
|
|
42
|
+
const keepIds = computeRetentionKeepIds(order, byId, {
|
|
43
|
+
maxDepth,
|
|
44
|
+
cutoffWall,
|
|
45
|
+
anchorTypes: policy.anchorTypes,
|
|
46
|
+
checkpointTipId,
|
|
47
|
+
branchTipId,
|
|
48
|
+
})
|
|
49
|
+
if (keepIds.size >= events.length) return { pruned: false, kept: events.length, dropped: 0 }
|
|
50
|
+
const keptIds = order.filter(id => keepIds.has(id))
|
|
51
|
+
const dropped = events.length - keptIds.length
|
|
52
|
+
if (dropped <= 0) return { pruned: false, kept: keptIds.length, dropped: 0 }
|
|
53
|
+
const keptRaws = keptIds.map(id => rawById.get(id)).filter(raw => raw !== undefined)
|
|
54
|
+
await writeJsonlLines(eventsFilePath, keptRaws)
|
|
55
|
+
invalidateTopologicalOrderMemo(eventsFilePath)
|
|
56
|
+
return { pruned: true, kept: keptIds.length, dropped }
|
|
45
57
|
})
|
|
46
|
-
if (keepIds.size >= events.length) return { pruned: false, kept: events.length, dropped: 0 }
|
|
47
|
-
const kept = order.map(id => byId.get(id)).filter(ev => ev && keepIds.has(ev.id))
|
|
48
|
-
const dropped = events.length - kept.length
|
|
49
|
-
if (dropped <= 0) return { pruned: false, kept: kept.length, dropped: 0 }
|
|
50
|
-
await mkdir(dirname(eventsFilePath), { recursive: true })
|
|
51
|
-
await writeJsonlSynced(eventsFilePath, kept)
|
|
52
|
-
invalidateTopologicalOrderMemo(eventsFilePath)
|
|
53
|
-
return { pruned: true, kept: kept.length, dropped }
|
|
54
58
|
}
|
package/utils/async_mutex.mjs
CHANGED
|
@@ -25,13 +25,21 @@ function mutexState(lockKey) {
|
|
|
25
25
|
*/
|
|
26
26
|
function releaseMutex(lockKey, state) {
|
|
27
27
|
const next = state.queue.shift()
|
|
28
|
-
if (next)
|
|
28
|
+
if (next) queueMicrotask(next)
|
|
29
29
|
else {
|
|
30
30
|
state.locked = false
|
|
31
31
|
mutexes.delete(lockKey)
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* 当前存活的互斥键数量(仅供测试观察锁表是否回落到基线)。
|
|
37
|
+
* @returns {number} `mutexes` 中的键数量
|
|
38
|
+
*/
|
|
39
|
+
export function activeMutexCount() {
|
|
40
|
+
return mutexes.size
|
|
41
|
+
}
|
|
42
|
+
|
|
35
43
|
/**
|
|
36
44
|
* @param {string} key 锁键
|
|
37
45
|
* @param {() => Promise<T>} criticalSection 临界区
|
|
@@ -39,15 +47,14 @@ function releaseMutex(lockKey, state) {
|
|
|
39
47
|
* @template T
|
|
40
48
|
*/
|
|
41
49
|
export async function withAsyncMutex(key, criticalSection) {
|
|
42
|
-
const
|
|
43
|
-
const state = mutexState(lockKey)
|
|
50
|
+
const state = mutexState(key)
|
|
44
51
|
return new Promise((resolve, reject) => {
|
|
45
52
|
/** 执行临界区并最终释放锁 */
|
|
46
53
|
const run = () => {
|
|
47
54
|
Promise.resolve()
|
|
48
55
|
.then(criticalSection)
|
|
49
56
|
.then(resolve, reject)
|
|
50
|
-
.finally(() => releaseMutex(
|
|
57
|
+
.finally(() => releaseMutex(key, state))
|
|
51
58
|
}
|
|
52
59
|
if (state.locked) state.queue.push(run)
|
|
53
60
|
else {
|
package/utils/shuffle.mjs
CHANGED
package/wire/adapter.mjs
CHANGED