@steve02081504/fount-p2p 0.0.44 → 0.0.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/AGENTS.md CHANGED
@@ -3,7 +3,7 @@
3
3
  ## Package layers (`@steve02081504/fount-p2p`)
4
4
 
5
5
  | Layer | Directory | Role |
6
- |---|---|---|
6
+ | --- | --- | --- |
7
7
  | L0 | `core/` | IDs, logical entity, canonical JSON, bytes |
8
8
  | L1 | `crypto/`, `wire/`, `schemas/` | Crypto, wire ingress, canonical validation |
9
9
  | L2 | `node/` | `initNode`, identity, entity store, denylist, reputation, storage plugins |
@@ -19,7 +19,7 @@ Detail docs: [transports](docs/transports.md) · [mesh](docs/mesh.md) · [signal
19
19
  ### Runtime: isomorphic vs Node
20
20
 
21
21
  | Surface | Runtime | Notes |
22
- |---|---|---|
22
+ | --- | --- | --- |
23
23
  | `core/*` | Node + browser | No `node:*` builtins |
24
24
  | `crypto/crypto.mjs` | Node + browser | `@noble/hashes` + `@noble/curves` only — never `node:crypto` |
25
25
  | `crypto/key.mjs` / `crypto/channel.mjs`, disk I/O, LAN/BT, `ws`, CLI / `startNode` | Node (+ Deno bridge) | Do not load the whole package via esm.sh in the browser |
@@ -31,6 +31,7 @@ Deno / native / BT: [runtime.md](docs/runtime.md).
31
31
  ## Conventions
32
32
 
33
33
  - Prefer shared helpers under `utils/`, `core/`, `wire/subscribe`, `wire/adapter` — do not reimplement LRU/TTL/inflight/atomic-fs/shuffle.
34
+ - **JSONL read-modify-write:** do it inside `withAsyncMutex(jsonlMutexKey(path))`. `rewriteJsonlKeeping` / `appendJsonlSynced` / `writeJsonlSynced` take that per-file mutex themselves and are **not reentrant** — never call them while already holding the same key (use lock-free `writeJsonl` / `writeJsonlLines` instead). Rewrites preserve the original raw line (a `sanitize` passed to `rewriteJsonlKeeping` / `readJsonlEntries` only affects the value handed to the predicate) and skip the rename when nothing is dropped.
34
35
  - **No pure-forward aliases:** do not add `fooText`/`fooAlias` that only `return foo(sameArgs)` when the callee already accepts those types (e.g. never wrap `sha256Hex` as `sha256TextHex`). Domain names must add logic or type narrowing, not just rename.
35
36
  - **Heterogeneous backends:** normalize at the load boundary (e.g. `link/rtc/ice_local_hostname.mjs` wraps W3C RTC backends); call sites speak one contract.
36
37
  - **File naming:** parent directory is scope — short child names. Tunables default `<dir>/tunables.json` (exception: `schemas/part_query.tunables.json`). Subpath `package.json` exports mirror filenames.
@@ -76,7 +77,7 @@ Deno / native / BT: [runtime.md](docs/runtime.md).
76
77
  ## Tunables JSON
77
78
 
78
79
  | File | Directory |
79
- |---|---|
80
+ | --- | --- |
80
81
  | `tunables.json` | `transport/`, `infra/`, `reputation/`, `trust_graph/`, `mailbox/`, `governance/`, `dag/` |
81
82
  | `part_query.tunables.json` | `schemas/` |
82
83
 
package/dag/storage.mjs CHANGED
@@ -1,7 +1,7 @@
1
1
  import { Buffer } from 'node:buffer'
2
2
  import { createReadStream, createWriteStream } from 'node:fs'
3
- import { appendFile, mkdir, open, readFile, writeFile } from 'node:fs/promises'
4
- import { dirname } from 'node:path'
3
+ import { appendFile, mkdir, open, unlink, writeFile } from 'node:fs/promises'
4
+ import { dirname, resolve } from 'node:path'
5
5
  import { createInterface } from 'node:readline'
6
6
  import { Readable } from 'node:stream'
7
7
  import { pipeline } from 'node:stream/promises'
@@ -29,26 +29,52 @@ function parseJsonlLine(line, sanitize) {
29
29
  }
30
30
 
31
31
  /**
32
- * 读取 JSONL 文件并解析为对象数组;缺失或读失败时返回空数组。
32
+ * 逐行读取 JSONL 原始行(不解析、不净化)。文件缺失(ENOENT)视为空流,
33
+ * 兼容 cleanup 竞态:群目录被删后台仍在尾巴上读它;其他读错误抛出。
34
+ * @param {string} filePath 文件路径
35
+ * @returns {AsyncGenerator<string>} 原始行(readline 已去掉行尾换行)
36
+ */
37
+ async function* readJsonlRawLines(filePath) {
38
+ const input = createReadStream(filePath, { encoding: 'utf8' })
39
+ // stream 内部异步 open 失败会触发 'error' 事件;提前订阅避免 unhandled error,
40
+ // 真实错误仍由下方 for-await 抛出,由本函数统一收口。
41
+ input.on('error', () => { })
42
+ const lines = createInterface({ input, crlfDelay: Infinity })
43
+ try {
44
+ for await (const line of lines) yield line
45
+ }
46
+ catch (error) {
47
+ if (error?.code !== 'ENOENT') throw error
48
+ }
49
+ }
50
+
51
+ /**
52
+ * 读取 JSONL 为 `{ row, raw }` 列表,保留原始行以便无损写回。
53
+ * 空行/解析失败/净化抛错的行跳过;文件缺失(ENOENT)返回空数组,其他读错误抛出。
54
+ * @param {string} filePath 文件路径
55
+ * @param {{ sanitize?: (row: object) => object }} [options] 行净化
56
+ * @returns {Promise<Array<{ row: object, raw: string }>>} 行条目列表
57
+ */
58
+ export async function readJsonlEntries(filePath, options = {}) {
59
+ const sanitize = options.sanitize ?? (row => row)
60
+ /** @type {Array<{ row: object, raw: string }>} */
61
+ const entries = []
62
+ for await (const raw of readJsonlRawLines(filePath)) {
63
+ const row = parseJsonlLine(raw, sanitize)
64
+ if (row != null) entries.push({ row, raw })
65
+ }
66
+ return entries
67
+ }
68
+
69
+ /**
70
+ * 读取 JSONL 文件并解析为对象数组;缺失(ENOENT)返回空数组,其他读错误抛出。
33
71
  * @param {string} filePath 文件系统路径
34
72
  * @param {{ sanitize?: (row: object) => object }} [options] 可选净化函数
35
73
  * @returns {Promise<object[]>} 各行解析后的对象列表
36
74
  */
37
75
  export async function readJsonl(filePath, options = {}) {
38
- try {
39
- const text = await readFile(filePath, 'utf8')
40
- const sanitize = options.sanitize ?? (row => row)
41
- /** @type {object[]} */
42
- const rows = []
43
- for (const line of text.split('\n')) {
44
- const row = parseJsonlLine(line, sanitize)
45
- if (row) rows.push(row)
46
- }
47
- return rows
48
- }
49
- catch {
50
- return []
51
- }
76
+ const entries = await readJsonlEntries(filePath, options)
77
+ return entries.map(entry => entry.row).filter(Boolean)
52
78
  }
53
79
 
54
80
  /**
@@ -60,69 +86,93 @@ export async function readJsonl(filePath, options = {}) {
60
86
  */
61
87
  export async function* readJsonlStream(filePath, options = {}) {
62
88
  const sanitize = options.sanitize ?? (row => row)
63
- const input = createReadStream(filePath, { encoding: 'utf8' })
64
- // stream 内部异步 open 失败会触发 'error' 事件;提前订阅避免 unhandled error,
65
- // 真实错误仍由下方 for-await 抛出,被外层 try/catch 收口。
66
- input.on('error', () => { })
67
- const lines = createInterface({ input, crlfDelay: Infinity })
68
- try {
69
- for await (const line of lines) {
70
- const row = parseJsonlLine(line, sanitize)
71
- if (row) yield row
72
- }
73
- }
74
- catch (error) {
75
- if (error?.code !== 'ENOENT') throw error
89
+ for await (const line of readJsonlRawLines(filePath)) {
90
+ const row = parseJsonlLine(line, sanitize)
91
+ if (row) yield row
76
92
  }
77
93
  }
78
94
 
79
95
  /**
80
- * 流式过滤重写 JSONL:保留 `keep(row)===true` 的行。
96
+ * 流式过滤重写 JSONL:保留 `keep(row)===true` 的行,写回的是原始行字节,
97
+ * `sanitize` 只影响传给 `keep` 的值。
98
+ *
99
+ * 在 `jsonlMutexKey(filePath)` 互斥锁内执行;调用方不得已持有同一文件的锁(不可重入)。
81
100
  * @param {string} filePath 目标路径
82
- * @param {(row: object) => boolean} keep 保留谓词
101
+ * @param {(row: object) => boolean | Promise<boolean>} keep 保留谓词(可异步)
83
102
  * @param {{ sanitize?: (row: object) => object }} [options] 读行净化
84
103
  * @returns {Promise<{ kept: number, dropped: number }>} 统计
85
104
  */
86
105
  export async function rewriteJsonlKeeping(filePath, keep, options = {}) {
87
- const dir = dirname(filePath)
88
- await mkdir(dir, { recursive: true })
106
+ return withAsyncMutex(jsonlMutexKey(filePath), () => rewriteJsonlKeepingUnlocked(filePath, keep, options))
107
+ }
108
+
109
+ /**
110
+ * `rewriteJsonlKeeping` 的无锁实现,调用方须已持有该文件互斥锁。
111
+ * @param {string} filePath 目标路径
112
+ * @param {(row: object) => boolean | Promise<boolean>} keep 保留谓词(可异步)
113
+ * @param {{ sanitize?: (row: object) => object }} [options] 读行净化
114
+ * @returns {Promise<{ kept: number, dropped: number }>} 统计
115
+ */
116
+ async function rewriteJsonlKeepingUnlocked(filePath, keep, options = {}) {
117
+ const sanitize = options.sanitize ?? (row => row)
89
118
  const temporaryPath = atomicTemporaryPath(filePath)
90
- /** @type {object[]} */
119
+ let tempCreated = false
120
+ /** @type {string[]} */
91
121
  const buffer = []
92
122
  let kept = 0
93
123
  let dropped = 0
94
- /** @returns {Promise<void>} */
124
+ /** @returns {Promise<void>} 将缓冲区原始行写入临时文件 */
95
125
  const flush = async () => {
96
126
  if (!buffer.length) return
97
- let block = ''
98
- for (const row of buffer)
99
- block += `${JSON.stringify(row)}\n`
127
+ const block = buffer.map(line => `${line}\n`).join('')
100
128
  await appendFile(temporaryPath, block, 'utf8')
129
+ tempCreated = true
101
130
  buffer.length = 0
102
131
  }
103
132
  try {
104
- for await (const row of readJsonlStream(filePath, options))
105
- if (keep(row)) {
106
- buffer.push(row)
133
+ for await (const raw of readJsonlRawLines(filePath)) {
134
+ if (!raw.trim()) continue
135
+ const row = parseJsonlLine(raw, sanitize)
136
+ if (row == null) {
137
+ dropped++
138
+ continue
139
+ }
140
+ if (await keep(row)) {
141
+ buffer.push(raw)
107
142
  kept++
108
- if (buffer.length >= WRITE_JSONL_CHUNK_LINES)
109
- await flush()
143
+ if (buffer.length >= WRITE_JSONL_CHUNK_LINES) await flush()
110
144
  }
111
145
  else dropped++
146
+ }
112
147
  await flush()
113
- }
114
- catch { /* source missing */ }
115
- if (kept > 0 || dropped > 0)
148
+ if (dropped === 0) {
149
+ if (tempCreated) await unlink(temporaryPath).catch(() => { })
150
+ return { kept, dropped }
151
+ }
152
+ await mkdir(dirname(filePath), { recursive: true })
153
+ if (!tempCreated) {
154
+ await writeFile(temporaryPath, '', 'utf8')
155
+ tempCreated = true
156
+ }
157
+ const fileHandle = await open(temporaryPath, 'r+')
158
+ try {
159
+ await fileHandle.sync()
160
+ }
161
+ finally {
162
+ await fileHandle.close()
163
+ }
116
164
  await finalizeAtomicRename(temporaryPath, filePath)
117
- else
118
- try { await writeFile(filePath, '', 'utf8') }
119
- catch { /* ok */ }
120
-
121
- return { kept, dropped }
165
+ return { kept, dropped }
166
+ }
167
+ catch (error) {
168
+ if (tempCreated) await unlink(temporaryPath).catch(() => { })
169
+ throw error
170
+ }
122
171
  }
123
172
 
124
173
  /**
125
- * 读取 JSONL 末行事件的 `id`(DAG tip);空文件为 null。
174
+ * 读取 JSONL 末条可解析事件的 `id`(DAG tip);从文件尾部最多读 1 MiB,
175
+ * 跳过撕裂/坏行后取最后一个非空且 `id` 非 null 的行;空文件或缺失为 null。
126
176
  * @param {string} filePath 文件路径
127
177
  * @returns {Promise<string | null>} tip event id
128
178
  */
@@ -132,14 +182,20 @@ export async function readJsonlTipId(filePath) {
132
182
  try {
133
183
  const { size } = await fh.stat()
134
184
  if (!size) return null
135
- const chunk = Math.min(size, 65_536)
185
+ const chunk = Math.min(size, 1_048_576)
136
186
  const buffer = Buffer.alloc(chunk)
137
187
  await fh.read(buffer, 0, chunk, size - chunk)
138
- const lines = buffer.toString('utf8').split('\n').filter(Boolean)
139
- const last = lines[lines.length - 1]
140
- if (!last) return null
141
- const row = JSON.parse(last)
142
- return row?.id != null ? String(row.id) : null
188
+ const lines = buffer.toString('utf8').split('\n')
189
+ for (let index = lines.length - 1; index >= 0; index--) {
190
+ const line = lines[index].trim()
191
+ if (!line) continue
192
+ try {
193
+ const row = JSON.parse(line)
194
+ if (row && typeof row === 'object' && row.id != null) return String(row.id)
195
+ }
196
+ catch { /* 撕裂/坏行:继续向前找 */ }
197
+ }
198
+ return null
143
199
  }
144
200
  finally {
145
201
  await fh.close()
@@ -151,30 +207,50 @@ export async function readJsonlTipId(filePath) {
151
207
  }
152
208
 
153
209
  /**
154
- * 流式重写 JSONL(临时文件 + rename),避免大数组 join 的内存峰值。
210
+ * 流式写入原始 JSONL 行(临时文件 + fsync + rename),每行自动补 `\n`。
211
+ * 不取锁——调用方需自行持有 `jsonlMutexKey(filePath)`。
155
212
  * @param {string} filePath 目标路径
156
- * @param {object[]} records 行对象列表
213
+ * @param {Iterable<string>} lines 原始行(不含换行)
157
214
  * @returns {Promise<void>}
158
215
  */
159
- export async function writeJsonl(filePath, records) {
216
+ export async function writeJsonlLines(filePath, lines) {
160
217
  const dir = dirname(filePath)
161
218
  await mkdir(dir, { recursive: true })
162
219
  const temporaryPath = atomicTemporaryPath(filePath)
163
- /** @returns {Generator<string>} JSONL 行 */
164
- function* lines() {
165
- for (const rec of records)
166
- yield `${JSON.stringify(rec)}\n`
220
+ /** @returns {Generator<string>} 带换行的行 */
221
+ function* withEol() {
222
+ for (const line of lines)
223
+ yield `${line}\n`
224
+ }
225
+ await pipeline(Readable.from(withEol()), createWriteStream(temporaryPath, { encoding: 'utf8' }))
226
+ const fileHandle = await open(temporaryPath, 'r+')
227
+ try {
228
+ await fileHandle.sync()
229
+ }
230
+ finally {
231
+ await fileHandle.close()
167
232
  }
168
- await pipeline(Readable.from(lines()), createWriteStream(temporaryPath, { encoding: 'utf8' }))
169
233
  await finalizeAtomicRename(temporaryPath, filePath)
170
234
  }
171
235
 
236
+ /**
237
+ * 流式重写 JSONL(临时文件 + fsync + rename),避免大数组 join 的内存峰值。
238
+ * 不取锁——`mailbox/store.mjs` 等调用方在自己的 `jsonlMutexKey` 锁内调用;
239
+ * 需要锁时用 `writeJsonlSynced`。
240
+ * @param {string} filePath 目标路径
241
+ * @param {object[]} records 行对象列表
242
+ * @returns {Promise<void>}
243
+ */
244
+ export async function writeJsonl(filePath, records) {
245
+ return writeJsonlLines(filePath, records.map(record => JSON.stringify(record)))
246
+ }
247
+
172
248
  /**
173
249
  * @param {string} filePath JSONL 路径
174
- * @returns {string} 进程内互斥键
250
+ * @returns {string} 进程内互斥键(路径已规范化为绝对路径)
175
251
  */
176
252
  export function jsonlMutexKey(filePath) {
177
- return `jsonl:${filePath}`
253
+ return `jsonl:${resolve(filePath)}`
178
254
  }
179
255
 
180
256
  /**
@@ -188,21 +264,23 @@ export async function writeJsonlSynced(filePath, records) {
188
264
  }
189
265
 
190
266
  /**
191
- * 追加一行 JSONL 并 `fsync`。
267
+ * 追加一行 JSONL 并 `fsync`;与 rewrite 共用同一 per-file 互斥锁。
192
268
  * @param {string} filePath 目标路径
193
269
  * @param {object} record 记录对象
194
270
  * @returns {Promise<void>}
195
271
  */
196
272
  export async function appendJsonlSynced(filePath, record) {
197
- await mkdir(dirname(filePath), { recursive: true })
198
- const fh = await open(filePath, 'a')
199
- try {
200
- await fh.appendFile(`${JSON.stringify(record)}\n`, 'utf8')
201
- await fh.sync()
202
- }
203
- finally {
204
- await fh.close()
205
- }
273
+ return withAsyncMutex(jsonlMutexKey(filePath), async () => {
274
+ await mkdir(dirname(filePath), { recursive: true })
275
+ const fh = await open(filePath, 'a')
276
+ try {
277
+ await fh.appendFile(`${JSON.stringify(record)}\n`, 'utf8')
278
+ await fh.sync()
279
+ }
280
+ finally {
281
+ await fh.close()
282
+ }
283
+ })
206
284
  }
207
285
 
208
286
  /**
@@ -21,17 +21,17 @@ import { ensureNodeSeed, getNodeHash } from '../../node/identity.mjs'
21
21
  import { getP2PFeatures, isNodeInitialized } from '../../node/instance.mjs'
22
22
  import { nodeDebug } from '../../node/log.mjs'
23
23
 
24
- import {
25
- CENSUS_TAG_FOUNT,
26
- CENSUS_TAG_X,
27
- NOSTR_CENSUS_KIND,
28
- } from './constants.mjs'
29
24
  import {
30
25
  CENSUS_TARGET_EVENTS,
31
26
  estimatePopulation,
32
27
  nextInclusionProbability,
33
28
  } from './census_math.mjs'
34
29
  import { CENSUS_TTL_MS, buildCensusMessage, verifyCensusBytes } from './census_verify.mjs'
30
+ import {
31
+ CENSUS_TAG_FOUNT,
32
+ CENSUS_TAG_X,
33
+ NOSTR_CENSUS_KIND,
34
+ } from './constants.mjs'
35
35
  import { resolveRelayConnectTarget } from './relays.mjs'
36
36
 
37
37
  /** census 订阅标签数组(发布用)。 */
@@ -10,6 +10,8 @@
10
10
  */
11
11
  import { base64ToBytes } from '../../core/bytes_codec.mjs'
12
12
 
13
+ import { estimatePopulation } from './census_math.mjs'
14
+ import { CENSUS_TTL_MS, verifyCensusBytes } from './census_verify.mjs'
13
15
  import {
14
16
  CENSUS_TAG_FOUNT,
15
17
  CENSUS_TAG_X,
@@ -17,8 +19,6 @@ import {
17
19
  NIP66_BOOTSTRAP_RELAYS,
18
20
  NOSTR_CENSUS_KIND,
19
21
  } from './constants.mjs'
20
- import { estimatePopulation } from './census_math.mjs'
21
- import { CENSUS_TTL_MS, verifyCensusBytes } from './census_verify.mjs'
22
22
 
23
23
  /** NIP-66 relay 注册 kind。 */
24
24
  const NIP66_KIND = 30166
@@ -57,4 +57,7 @@ export const NOSTR_CENSUS_KIND = 30789
57
57
 
58
58
  /** census 订阅/发布标签:`t=fount` + `x=census`(subscribeNostrKind 以 rendezvousKey/tagX 匹配)。 */
59
59
  export const CENSUS_TAG_FOUNT = 'fount'
60
+ /**
61
+ *
62
+ */
60
63
  export const CENSUS_TAG_X = 'census'
@@ -18,8 +18,8 @@ import {
18
18
  } from '../internal/signal_crypto.mjs'
19
19
  import { noteDiscoveryPeerClue } from '../peer_clue.mjs'
20
20
 
21
- import { DEFAULT_RELAY_URLS } from './constants.mjs'
22
21
  import { createNostrCensus } from './census.mjs'
22
+ import { DEFAULT_RELAY_URLS } from './constants.mjs'
23
23
  import {
24
24
  getListenRelays,
25
25
  getPeerRoute,
@@ -3,6 +3,7 @@
3
3
  Nostr is the fount-network fallback transport (`link` level −∞, discovery `priority` 100). This document covers the **relay pool**, NIP-66 discovery, health scoring, advert relay-field signing, and handshake routing. Top-level surface: [signaling.md](signaling.md) · [transports.md](transports.md) · [mesh.md](mesh.md).
4
4
 
5
5
  Source layout (`discovery/nostr/`):
6
+
6
7
  - `index.mjs` — provider (presence / signal / advert subscriptions)
7
8
  - `relays.mjs` — pool, health, NIP-66 discovery, normalization, persistence
8
9
  - `selection.mjs` — handshake routing, backoff, fanout
@@ -72,6 +73,7 @@ Lower is better. `recordProbeSuccess` / `recordProbeFailure` / `recordPublishRes
72
73
  ## Handshake routing (`selection.mjs`)
73
74
 
74
75
  `handshakeTargets(nodeHash, attempt)`:
76
+
75
77
  - **Round 0**: peer-claimed `listenRelays` top 4 by composite score (own health + peer rtt); else local `workingRelays` top 4; else pinned top 4.
76
78
  - **Round ≥1**: backoff `min(2000 · 2^(attempt−1), 60000)`; base on `lastGoodNostrRelays` (expanded via `expandFromHistory` ≤ 16), or weighted-random sample of `workingRelays` (weight `1/score`); round-0 core always included; fanout capped at `MAX_ROUTING_FANOUT` (64).
77
79
  - Retries ≤ `MAX_ROUTING_ATTEMPTS` (4).
package/docs/signaling.md CHANGED
@@ -25,6 +25,7 @@ When set: send final offer/answer after ICE gathering, dedupe remote signals, qu
25
25
  `channels` selects which discovery/link media are active — `nostr`, `lan`, `bt`, `webrtc`. Semantics per channel key: `false` **disables**; any other value (`undefined`, `true`, or an object) **enables** — an object is merged over that channel's default config. Channels not mentioned keep their default enabled state.
26
26
 
27
27
  Channel → components:
28
+
28
29
  - `nostr` (nostr discovery + nostr link) — config `relay` **replaces** the default public relay list (do not merge defaults back in).
29
30
  - `lan` (lan discovery + lan_tcp link)
30
31
  - `bt` (bt discovery + ble_gatt link)
@@ -227,7 +227,7 @@ function mergeRowsWithSources(entries, maxHits, rowKey) {
227
227
  try { return JSON.stringify(row) }
228
228
  catch { return `\0${rows.length}` }
229
229
  })
230
- for (const entry of entries) {
230
+ for (const entry of entries)
231
231
  for (const row of entry.rows || []) {
232
232
  const key = keyOf(row)
233
233
  if (!seen.has(key)) {
@@ -240,7 +240,7 @@ function mergeRowsWithSources(entries, maxHits, rowKey) {
240
240
  }
241
241
  if (entry.sourceNodeHash) sources.get(key).add(entry.sourceNodeHash)
242
242
  }
243
- }
243
+
244
244
  return { rows, sources }
245
245
  }
246
246
 
@@ -325,10 +325,10 @@ function filterRowsBySource(rows, sources, rowKey, isSourceBlocked) {
325
325
  return rows.filter(row => {
326
326
  let key
327
327
  if (rowKey) key = rowKey(row)
328
- else {
328
+ else
329
329
  try { key = JSON.stringify(row) }
330
330
  catch { return true }
331
- }
331
+
332
332
  const set = sources.get(key)
333
333
  if (!set || set.size === 0) return true
334
334
  for (const source of set) if (!isSourceBlocked(source)) return true
package/mailbox/store.mjs CHANGED
@@ -1,8 +1,5 @@
1
- import { mkdir, readFile } from 'node:fs/promises'
2
- import { dirname } from 'node:path'
3
-
4
1
  import { isHex64 } from '../core/hexIds.mjs'
5
- import { jsonlMutexKey, writeJsonl } from '../dag/storage.mjs'
2
+ import { jsonlMutexKey, readJsonl, writeJsonl } from '../dag/storage.mjs'
6
3
  import { mailboxStorePath } from '../node/user_paths.mjs'
7
4
  import { withAsyncMutex } from '../utils/async_mutex.mjs'
8
5
 
@@ -103,21 +100,7 @@ export function relayHopAfterWireIngress(wireHop, existingStoredHop) {
103
100
  * @returns {Promise<MailboxRecord[]>} 全部有效记录
104
101
  */
105
102
  async function readAll() {
106
- try {
107
- const text = await readFile(mailboxStorePath(), 'utf8')
108
- /** @type {MailboxRecord[]} */
109
- const rows = []
110
- for (const line of text.split('\n')) {
111
- const trimmed = line.trim()
112
- if (!trimmed) continue
113
- try {
114
- rows.push(JSON.parse(trimmed))
115
- }
116
- catch { /* skip corrupt line */ }
117
- }
118
- return rows
119
- }
120
- catch { return [] }
103
+ return /** @type {MailboxRecord[]} */ await readJsonl(mailboxStorePath())
121
104
  }
122
105
 
123
106
  /**
@@ -125,13 +108,11 @@ async function readAll() {
125
108
  * @returns {Promise<void>}
126
109
  */
127
110
  async function writeAll(rows) {
128
- const filePath = mailboxStorePath()
129
- await mkdir(dirname(filePath), { recursive: true })
130
111
  const now = Date.now()
131
112
  const kept = pruneMailboxGlobalFair(
132
113
  pruneMailboxBuckets(sortMailboxForRetention(rows.filter(record => record.expiresAt > now))),
133
114
  )
134
- await writeJsonl(filePath, kept)
115
+ await writeJsonl(mailboxStorePath(), kept)
135
116
  }
136
117
 
137
118
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@steve02081504/fount-p2p",
3
- "version": "0.0.44",
3
+ "version": "0.0.46",
4
4
  "description": "fount federation P2P layer — link, trust graph, mailbox, DAG, EVFS.",
5
5
  "keywords": [
6
6
  "network",
package/pages/index.mjs CHANGED
@@ -32,6 +32,10 @@ function startLive() {
32
32
  try {
33
33
  monitor = createPopulationMonitor({
34
34
  relays: relayUrl ? [relayUrl] : undefined,
35
+ /**
36
+ * @param {{ estimate: number, sampleSize: number, eventsInWindow: number, relayUrl: string, relays: number }} snapshot 人口快照
37
+ * @returns {void}
38
+ */
35
39
  onUpdate: snapshot => {
36
40
  render(snapshot)
37
41
  statusEl.textContent = `监听 ${snapshot.relayUrl}(census,共 ${snapshot.relays} 个 relay)`
@@ -1,13 +1,14 @@
1
- import { mkdir } from 'node:fs/promises'
2
- import { dirname } from 'node:path'
3
-
4
1
  import { topologicalCanonicalOrder } from '../dag/index.mjs'
5
- import { readJsonl, writeJsonlSynced } from '../dag/storage.mjs'
2
+ import { jsonlMutexKey, readJsonlEntries, writeJsonlLines } from '../dag/storage.mjs'
6
3
  import { invalidateTopologicalOrderMemo } from '../federation/topo_order_memo.mjs'
7
4
  import { descendantClosureFromTip } from '../governance/branch.mjs'
5
+ import { withAsyncMutex } from '../utils/async_mutex.mjs'
8
6
 
9
7
  /**
10
8
  * 将 events.jsonl 裁剪为 checkpoint 起后代闭包(连通子图,非拓扑下标切片)。
9
+ *
10
+ * 整个读-算-写在 `jsonlMutexKey(eventsFilePath)` 锁内完成,避免与并发 append 竞态;
11
+ * 写回的是保留行的原始字节(非 sanitize 后的重序列化),保持无损失。
11
12
  * @param {string} eventsFilePath events.jsonl 路径
12
13
  * @param {object | null} checkpoint 含 checkpoint_event_id 的快照
13
14
  * @param {(row: object) => object} [sanitize] 行规范化
@@ -16,24 +17,27 @@ import { descendantClosureFromTip } from '../governance/branch.mjs'
16
17
  export async function pruneEventsJsonlAfterCheckpoint(eventsFilePath, checkpoint, sanitize = row => row) {
17
18
  const tipId = checkpoint?.checkpoint_event_id
18
19
  if (!tipId) return { pruned: false, kept: 0, dropped: 0 }
19
- const events = await readJsonl(eventsFilePath, { sanitize })
20
- if (!events.length) return { pruned: false, kept: 0, dropped: 0 }
21
- const byId = new Map(events.map(dagEvent => [dagEvent.id, dagEvent]))
22
- if (!byId.has(tipId)) return { pruned: false, kept: events.length, dropped: 0 }
20
+ return withAsyncMutex(jsonlMutexKey(eventsFilePath), async () => {
21
+ const entries = await readJsonlEntries(eventsFilePath, { sanitize })
22
+ if (!entries.length) return { pruned: false, kept: 0, dropped: 0 }
23
+ const byId = new Map(entries.map(entry => [entry.row.id, entry.row]))
24
+ const rawById = new Map(entries.map(entry => [entry.row.id, entry.raw]))
25
+ if (!byId.has(tipId)) return { pruned: false, kept: entries.length, dropped: 0 }
23
26
 
24
- const keepIds = descendantClosureFromTip(tipId, byId)
25
- const order = topologicalCanonicalOrder(events.map(dagEvent => ({
26
- id: dagEvent.id,
27
- prev_event_ids: dagEvent.prev_event_ids,
28
- hlc: dagEvent.hlc,
29
- node_id: dagEvent.node_id,
30
- sender: dagEvent.sender,
31
- })))
32
- const kept = order.map(id => byId.get(id)).filter(ev => ev && keepIds.has(ev.id))
33
- const dropped = events.length - kept.length
34
- if (dropped <= 0) return { pruned: false, kept: kept.length, dropped: 0 }
35
- await mkdir(dirname(eventsFilePath), { recursive: true })
36
- await writeJsonlSynced(eventsFilePath, kept)
37
- invalidateTopologicalOrderMemo(eventsFilePath)
38
- return { pruned: true, kept: kept.length, dropped }
27
+ const keepIds = descendantClosureFromTip(tipId, byId)
28
+ const order = topologicalCanonicalOrder(entries.map(entry => ({
29
+ id: entry.row.id,
30
+ prev_event_ids: entry.row.prev_event_ids,
31
+ hlc: entry.row.hlc,
32
+ node_id: entry.row.node_id,
33
+ sender: entry.row.sender,
34
+ })))
35
+ const keptIds = order.filter(id => keepIds.has(id))
36
+ const dropped = entries.length - keptIds.length
37
+ if (dropped <= 0) return { pruned: false, kept: keptIds.length, dropped: 0 }
38
+ const keptRaws = keptIds.map(id => rawById.get(id)).filter(raw => raw !== undefined)
39
+ await writeJsonlLines(eventsFilePath, keptRaws)
40
+ invalidateTopologicalOrderMemo(eventsFilePath)
41
+ return { pruned: true, kept: keptIds.length, dropped }
42
+ })
39
43
  }
@@ -1,13 +1,13 @@
1
- import { mkdir } from 'node:fs/promises'
2
- import { dirname } from 'node:path'
3
-
4
1
  import { topologicalCanonicalOrder } from '../dag/index.mjs'
5
- import { readJsonl, writeJsonlSynced } from '../dag/storage.mjs'
2
+ import { jsonlMutexKey, readJsonlEntries, writeJsonlLines } from '../dag/storage.mjs'
6
3
  import { invalidateTopologicalOrderMemo } from '../federation/topo_order_memo.mjs'
7
4
  import { computeDagTipIdsFromEvents, selectConsensusBranchTip } from '../governance/branch.mjs'
8
5
  import { computeRetentionKeepIds } from '../node/retention_policy.mjs'
6
+ import { withAsyncMutex } from '../utils/async_mutex.mjs'
9
7
 
10
8
  /**
9
+ * 整个读-算-写在 `jsonlMutexKey(eventsFilePath)` 锁内完成,避免与并发 append 竞态;
10
+ * 写回的是保留行的原始字节(非 sanitize 后的重序列化),保持无损失。
11
11
  * @param {string} eventsFilePath events.jsonl 路径
12
12
  * @param {object | null} checkpointHint 检查点提示
13
13
  * @param {{ maxDepth: number, maxMs: number, anchorTypes: Set<string> }} policy 保留策略
@@ -20,35 +20,39 @@ export async function enforceTimelineEventRetention(
20
20
  policy,
21
21
  sanitize = row => row,
22
22
  ) {
23
- const events = await readJsonl(eventsFilePath, { sanitize })
24
- if (!events.length) return { pruned: false, kept: 0, dropped: 0 }
25
- const maxDepth = Math.max(256, Number(policy.maxDepth) || 200_000)
26
- const maxMs = Math.max(3_600_000, Number(policy.maxMs) || 365 * 24 * 3600 * 1000)
27
- const cutoffWall = Date.now() - maxMs
28
- const byId = new Map(events.map(e => [e.id, e]))
29
- const order = topologicalCanonicalOrder(events.map(e => ({
30
- id: e.id,
31
- prev_event_ids: e.prev_event_ids,
32
- hlc: e.hlc,
33
- node_id: e.node_id,
34
- sender: e.sender,
35
- })))
36
- const tips = computeDagTipIdsFromEvents(events)
37
- const branchTipId = selectConsensusBranchTip(tips, byId)
38
- const checkpointTipId = checkpointHint?.checkpoint_event_id || null
39
- const keepIds = computeRetentionKeepIds(order, byId, {
40
- maxDepth,
41
- cutoffWall,
42
- anchorTypes: policy.anchorTypes,
43
- checkpointTipId,
44
- branchTipId,
23
+ return withAsyncMutex(jsonlMutexKey(eventsFilePath), async () => {
24
+ const entries = await readJsonlEntries(eventsFilePath, { sanitize })
25
+ if (!entries.length) return { pruned: false, kept: 0, dropped: 0 }
26
+ const maxDepth = Math.max(256, Number(policy.maxDepth) || 200_000)
27
+ const maxMs = Math.max(3_600_000, Number(policy.maxMs) || 365 * 24 * 3600 * 1000)
28
+ const cutoffWall = Date.now() - maxMs
29
+ const events = entries.map(entry => entry.row)
30
+ const rawById = new Map(entries.map(entry => [entry.row.id, entry.raw]))
31
+ const byId = new Map(events.map(e => [e.id, e]))
32
+ const order = topologicalCanonicalOrder(events.map(e => ({
33
+ id: e.id,
34
+ prev_event_ids: e.prev_event_ids,
35
+ hlc: e.hlc,
36
+ node_id: e.node_id,
37
+ sender: e.sender,
38
+ })))
39
+ const tips = computeDagTipIdsFromEvents(events)
40
+ const branchTipId = selectConsensusBranchTip(tips, byId)
41
+ const checkpointTipId = checkpointHint?.checkpoint_event_id || null
42
+ const keepIds = computeRetentionKeepIds(order, byId, {
43
+ maxDepth,
44
+ cutoffWall,
45
+ anchorTypes: policy.anchorTypes,
46
+ checkpointTipId,
47
+ branchTipId,
48
+ })
49
+ if (keepIds.size >= events.length) return { pruned: false, kept: events.length, dropped: 0 }
50
+ const keptIds = order.filter(id => keepIds.has(id))
51
+ const dropped = events.length - keptIds.length
52
+ if (dropped <= 0) return { pruned: false, kept: keptIds.length, dropped: 0 }
53
+ const keptRaws = keptIds.map(id => rawById.get(id)).filter(raw => raw !== undefined)
54
+ await writeJsonlLines(eventsFilePath, keptRaws)
55
+ invalidateTopologicalOrderMemo(eventsFilePath)
56
+ return { pruned: true, kept: keptIds.length, dropped }
45
57
  })
46
- if (keepIds.size >= events.length) return { pruned: false, kept: events.length, dropped: 0 }
47
- const kept = order.map(id => byId.get(id)).filter(ev => ev && keepIds.has(ev.id))
48
- const dropped = events.length - kept.length
49
- if (dropped <= 0) return { pruned: false, kept: kept.length, dropped: 0 }
50
- await mkdir(dirname(eventsFilePath), { recursive: true })
51
- await writeJsonlSynced(eventsFilePath, kept)
52
- invalidateTopologicalOrderMemo(eventsFilePath)
53
- return { pruned: true, kept: kept.length, dropped }
54
58
  }
@@ -25,13 +25,21 @@ function mutexState(lockKey) {
25
25
  */
26
26
  function releaseMutex(lockKey, state) {
27
27
  const next = state.queue.shift()
28
- if (next) setTimeout(next, 0)
28
+ if (next) queueMicrotask(next)
29
29
  else {
30
30
  state.locked = false
31
31
  mutexes.delete(lockKey)
32
32
  }
33
33
  }
34
34
 
35
+ /**
36
+ * 当前存活的互斥键数量(仅供测试观察锁表是否回落到基线)。
37
+ * @returns {number} `mutexes` 中的键数量
38
+ */
39
+ export function activeMutexCount() {
40
+ return mutexes.size
41
+ }
42
+
35
43
  /**
36
44
  * @param {string} key 锁键
37
45
  * @param {() => Promise<T>} criticalSection 临界区
@@ -39,15 +47,14 @@ function releaseMutex(lockKey, state) {
39
47
  * @template T
40
48
  */
41
49
  export async function withAsyncMutex(key, criticalSection) {
42
- const lockKey = key
43
- const state = mutexState(lockKey)
50
+ const state = mutexState(key)
44
51
  return new Promise((resolve, reject) => {
45
52
  /** 执行临界区并最终释放锁 */
46
53
  const run = () => {
47
54
  Promise.resolve()
48
55
  .then(criticalSection)
49
56
  .then(resolve, reject)
50
- .finally(() => releaseMutex(lockKey, state))
57
+ .finally(() => releaseMutex(key, state))
51
58
  }
52
59
  if (state.locked) state.queue.push(run)
53
60
  else {
package/utils/shuffle.mjs CHANGED
@@ -7,7 +7,7 @@
7
7
  export function shuffleInPlace(arr) {
8
8
  for (let i = arr.length - 1; i > 0; i--) {
9
9
  const j = Math.floor(Math.random() * (i + 1))
10
- ;[arr[i], arr[j]] = [arr[j], arr[i]]
10
+ ;[arr[i], arr[j]] = [arr[j], arr[i]]
11
11
  }
12
12
  return arr
13
13
  }
package/wire/adapter.mjs CHANGED
@@ -10,4 +10,4 @@
10
10
  */
11
11
 
12
12
  /** Wire 适配器与上下文的类型导出(运行时代码见各 wire 模块)。 */
13
- export {}
13
+ export { }