@steve02081504/fount-p2p 0.0.44 → 0.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -3
- package/dag/storage.mjs +157 -79
- package/discovery/nostr/census.mjs +5 -5
- package/discovery/nostr/census_monitor.mjs +2 -2
- package/discovery/nostr/constants.mjs +3 -0
- package/discovery/nostr/index.mjs +1 -1
- package/docs/nostr_relay_discovery.md +2 -0
- package/docs/signaling.md +1 -0
- package/federation/part_query/runtime.mjs +4 -4
- package/mailbox/store.mjs +3 -22
- package/package.json +1 -1
- package/pages/index.mjs +4 -0
- package/timeline/prune.mjs +27 -23
- package/timeline/retention.mjs +38 -34
- package/utils/async_mutex.mjs +11 -4
- package/utils/shuffle.mjs +1 -1
- package/wire/adapter.mjs +1 -1
package/AGENTS.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
## Package layers (`@steve02081504/fount-p2p`)
|
|
4
4
|
|
|
5
5
|
| Layer | Directory | Role |
|
|
6
|
-
|
|
6
|
+
| --- | --- | --- |
|
|
7
7
|
| L0 | `core/` | IDs, logical entity, canonical JSON, bytes |
|
|
8
8
|
| L1 | `crypto/`, `wire/`, `schemas/` | Crypto, wire ingress, canonical validation |
|
|
9
9
|
| L2 | `node/` | `initNode`, identity, entity store, denylist, reputation, storage plugins |
|
|
@@ -19,7 +19,7 @@ Detail docs: [transports](docs/transports.md) · [mesh](docs/mesh.md) · [signal
|
|
|
19
19
|
### Runtime: isomorphic vs Node
|
|
20
20
|
|
|
21
21
|
| Surface | Runtime | Notes |
|
|
22
|
-
|
|
22
|
+
| --- | --- | --- |
|
|
23
23
|
| `core/*` | Node + browser | No `node:*` builtins |
|
|
24
24
|
| `crypto/crypto.mjs` | Node + browser | `@noble/hashes` + `@noble/curves` only — never `node:crypto` |
|
|
25
25
|
| `crypto/key.mjs` / `crypto/channel.mjs`, disk I/O, LAN/BT, `ws`, CLI / `startNode` | Node (+ Deno bridge) | Do not load the whole package via esm.sh in the browser |
|
|
@@ -31,6 +31,7 @@ Deno / native / BT: [runtime.md](docs/runtime.md).
|
|
|
31
31
|
## Conventions
|
|
32
32
|
|
|
33
33
|
- Prefer shared helpers under `utils/`, `core/`, `wire/subscribe`, `wire/adapter` — do not reimplement LRU/TTL/inflight/atomic-fs/shuffle.
|
|
34
|
+
- **JSONL read-modify-write:** do it inside `withAsyncMutex(jsonlMutexKey(path))`. `rewriteJsonlKeeping` / `appendJsonlSynced` / `writeJsonlSynced` take that per-file mutex themselves and are **not reentrant** — never call them while already holding the same key (use lock-free `writeJsonl` / `writeJsonlLines` instead). Rewrites preserve the original raw line (a `sanitize` passed to `rewriteJsonlKeeping` / `readJsonlEntries` only affects the value handed to the predicate) and skip the rename when nothing is dropped.
|
|
34
35
|
- **No pure-forward aliases:** do not add `fooText`/`fooAlias` that only `return foo(sameArgs)` when the callee already accepts those types (e.g. never wrap `sha256Hex` as `sha256TextHex`). Domain names must add logic or type narrowing, not just rename.
|
|
35
36
|
- **Heterogeneous backends:** normalize at the load boundary (e.g. `link/rtc/ice_local_hostname.mjs` wraps W3C RTC backends); call sites speak one contract.
|
|
36
37
|
- **File naming:** parent directory is scope — short child names. Tunables default `<dir>/tunables.json` (exception: `schemas/part_query.tunables.json`). Subpath `package.json` exports mirror filenames.
|
|
@@ -76,7 +77,7 @@ Deno / native / BT: [runtime.md](docs/runtime.md).
|
|
|
76
77
|
## Tunables JSON
|
|
77
78
|
|
|
78
79
|
| File | Directory |
|
|
79
|
-
|
|
80
|
+
| --- | --- |
|
|
80
81
|
| `tunables.json` | `transport/`, `infra/`, `reputation/`, `trust_graph/`, `mailbox/`, `governance/`, `dag/` |
|
|
81
82
|
| `part_query.tunables.json` | `schemas/` |
|
|
82
83
|
|
package/dag/storage.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer'
|
|
2
2
|
import { createReadStream, createWriteStream } from 'node:fs'
|
|
3
|
-
import { appendFile, mkdir, open,
|
|
4
|
-
import { dirname } from 'node:path'
|
|
3
|
+
import { appendFile, mkdir, open, unlink, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { dirname, resolve } from 'node:path'
|
|
5
5
|
import { createInterface } from 'node:readline'
|
|
6
6
|
import { Readable } from 'node:stream'
|
|
7
7
|
import { pipeline } from 'node:stream/promises'
|
|
@@ -29,26 +29,52 @@ function parseJsonlLine(line, sanitize) {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
/**
|
|
32
|
-
*
|
|
32
|
+
* 逐行读取 JSONL 原始行(不解析、不净化)。文件缺失(ENOENT)视为空流,
|
|
33
|
+
* 兼容 cleanup 竞态:群目录被删后台仍在尾巴上读它;其他读错误抛出。
|
|
34
|
+
* @param {string} filePath 文件路径
|
|
35
|
+
* @returns {AsyncGenerator<string>} 原始行(readline 已去掉行尾换行)
|
|
36
|
+
*/
|
|
37
|
+
async function* readJsonlRawLines(filePath) {
|
|
38
|
+
const input = createReadStream(filePath, { encoding: 'utf8' })
|
|
39
|
+
// stream 内部异步 open 失败会触发 'error' 事件;提前订阅避免 unhandled error,
|
|
40
|
+
// 真实错误仍由下方 for-await 抛出,由本函数统一收口。
|
|
41
|
+
input.on('error', () => { })
|
|
42
|
+
const lines = createInterface({ input, crlfDelay: Infinity })
|
|
43
|
+
try {
|
|
44
|
+
for await (const line of lines) yield line
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error?.code !== 'ENOENT') throw error
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* 读取 JSONL 为 `{ row, raw }` 列表,保留原始行以便无损写回。
|
|
53
|
+
* 空行/解析失败/净化抛错的行跳过;文件缺失(ENOENT)返回空数组,其他读错误抛出。
|
|
54
|
+
* @param {string} filePath 文件路径
|
|
55
|
+
* @param {{ sanitize?: (row: object) => object }} [options] 行净化
|
|
56
|
+
* @returns {Promise<Array<{ row: object, raw: string }>>} 行条目列表
|
|
57
|
+
*/
|
|
58
|
+
export async function readJsonlEntries(filePath, options = {}) {
|
|
59
|
+
const sanitize = options.sanitize ?? (row => row)
|
|
60
|
+
/** @type {Array<{ row: object, raw: string }>} */
|
|
61
|
+
const entries = []
|
|
62
|
+
for await (const raw of readJsonlRawLines(filePath)) {
|
|
63
|
+
const row = parseJsonlLine(raw, sanitize)
|
|
64
|
+
if (row != null) entries.push({ row, raw })
|
|
65
|
+
}
|
|
66
|
+
return entries
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* 读取 JSONL 文件并解析为对象数组;缺失(ENOENT)返回空数组,其他读错误抛出。
|
|
33
71
|
* @param {string} filePath 文件系统路径
|
|
34
72
|
* @param {{ sanitize?: (row: object) => object }} [options] 可选净化函数
|
|
35
73
|
* @returns {Promise<object[]>} 各行解析后的对象列表
|
|
36
74
|
*/
|
|
37
75
|
export async function readJsonl(filePath, options = {}) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const sanitize = options.sanitize ?? (row => row)
|
|
41
|
-
/** @type {object[]} */
|
|
42
|
-
const rows = []
|
|
43
|
-
for (const line of text.split('\n')) {
|
|
44
|
-
const row = parseJsonlLine(line, sanitize)
|
|
45
|
-
if (row) rows.push(row)
|
|
46
|
-
}
|
|
47
|
-
return rows
|
|
48
|
-
}
|
|
49
|
-
catch {
|
|
50
|
-
return []
|
|
51
|
-
}
|
|
76
|
+
const entries = await readJsonlEntries(filePath, options)
|
|
77
|
+
return entries.map(entry => entry.row).filter(Boolean)
|
|
52
78
|
}
|
|
53
79
|
|
|
54
80
|
/**
|
|
@@ -60,69 +86,93 @@ export async function readJsonl(filePath, options = {}) {
|
|
|
60
86
|
*/
|
|
61
87
|
export async function* readJsonlStream(filePath, options = {}) {
|
|
62
88
|
const sanitize = options.sanitize ?? (row => row)
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
input.on('error', () => { })
|
|
67
|
-
const lines = createInterface({ input, crlfDelay: Infinity })
|
|
68
|
-
try {
|
|
69
|
-
for await (const line of lines) {
|
|
70
|
-
const row = parseJsonlLine(line, sanitize)
|
|
71
|
-
if (row) yield row
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
if (error?.code !== 'ENOENT') throw error
|
|
89
|
+
for await (const line of readJsonlRawLines(filePath)) {
|
|
90
|
+
const row = parseJsonlLine(line, sanitize)
|
|
91
|
+
if (row) yield row
|
|
76
92
|
}
|
|
77
93
|
}
|
|
78
94
|
|
|
79
95
|
/**
|
|
80
|
-
* 流式过滤重写 JSONL:保留 `keep(row)===true`
|
|
96
|
+
* 流式过滤重写 JSONL:保留 `keep(row)===true` 的行,写回的是原始行字节,
|
|
97
|
+
* `sanitize` 只影响传给 `keep` 的值。
|
|
98
|
+
*
|
|
99
|
+
* 在 `jsonlMutexKey(filePath)` 互斥锁内执行;调用方不得已持有同一文件的锁(不可重入)。
|
|
81
100
|
* @param {string} filePath 目标路径
|
|
82
|
-
* @param {(row: object) => boolean} keep
|
|
101
|
+
* @param {(row: object) => boolean | Promise<boolean>} keep 保留谓词(可异步)
|
|
83
102
|
* @param {{ sanitize?: (row: object) => object }} [options] 读行净化
|
|
84
103
|
* @returns {Promise<{ kept: number, dropped: number }>} 统计
|
|
85
104
|
*/
|
|
86
105
|
export async function rewriteJsonlKeeping(filePath, keep, options = {}) {
|
|
87
|
-
|
|
88
|
-
|
|
106
|
+
return withAsyncMutex(jsonlMutexKey(filePath), () => rewriteJsonlKeepingUnlocked(filePath, keep, options))
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* `rewriteJsonlKeeping` 的无锁实现,调用方须已持有该文件互斥锁。
|
|
111
|
+
* @param {string} filePath 目标路径
|
|
112
|
+
* @param {(row: object) => boolean | Promise<boolean>} keep 保留谓词(可异步)
|
|
113
|
+
* @param {{ sanitize?: (row: object) => object }} [options] 读行净化
|
|
114
|
+
* @returns {Promise<{ kept: number, dropped: number }>} 统计
|
|
115
|
+
*/
|
|
116
|
+
async function rewriteJsonlKeepingUnlocked(filePath, keep, options = {}) {
|
|
117
|
+
const sanitize = options.sanitize ?? (row => row)
|
|
89
118
|
const temporaryPath = atomicTemporaryPath(filePath)
|
|
90
|
-
|
|
119
|
+
let tempCreated = false
|
|
120
|
+
/** @type {string[]} */
|
|
91
121
|
const buffer = []
|
|
92
122
|
let kept = 0
|
|
93
123
|
let dropped = 0
|
|
94
|
-
/** @returns {Promise<void>} */
|
|
124
|
+
/** @returns {Promise<void>} 将缓冲区原始行写入临时文件 */
|
|
95
125
|
const flush = async () => {
|
|
96
126
|
if (!buffer.length) return
|
|
97
|
-
|
|
98
|
-
for (const row of buffer)
|
|
99
|
-
block += `${JSON.stringify(row)}\n`
|
|
127
|
+
const block = buffer.map(line => `${line}\n`).join('')
|
|
100
128
|
await appendFile(temporaryPath, block, 'utf8')
|
|
129
|
+
tempCreated = true
|
|
101
130
|
buffer.length = 0
|
|
102
131
|
}
|
|
103
132
|
try {
|
|
104
|
-
for await (const
|
|
105
|
-
if (
|
|
106
|
-
|
|
133
|
+
for await (const raw of readJsonlRawLines(filePath)) {
|
|
134
|
+
if (!raw.trim()) continue
|
|
135
|
+
const row = parseJsonlLine(raw, sanitize)
|
|
136
|
+
if (row == null) {
|
|
137
|
+
dropped++
|
|
138
|
+
continue
|
|
139
|
+
}
|
|
140
|
+
if (await keep(row)) {
|
|
141
|
+
buffer.push(raw)
|
|
107
142
|
kept++
|
|
108
|
-
if (buffer.length >= WRITE_JSONL_CHUNK_LINES)
|
|
109
|
-
await flush()
|
|
143
|
+
if (buffer.length >= WRITE_JSONL_CHUNK_LINES) await flush()
|
|
110
144
|
}
|
|
111
145
|
else dropped++
|
|
146
|
+
}
|
|
112
147
|
await flush()
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
148
|
+
if (dropped === 0) {
|
|
149
|
+
if (tempCreated) await unlink(temporaryPath).catch(() => { })
|
|
150
|
+
return { kept, dropped }
|
|
151
|
+
}
|
|
152
|
+
await mkdir(dirname(filePath), { recursive: true })
|
|
153
|
+
if (!tempCreated) {
|
|
154
|
+
await writeFile(temporaryPath, '', 'utf8')
|
|
155
|
+
tempCreated = true
|
|
156
|
+
}
|
|
157
|
+
const fileHandle = await open(temporaryPath, 'r+')
|
|
158
|
+
try {
|
|
159
|
+
await fileHandle.sync()
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
await fileHandle.close()
|
|
163
|
+
}
|
|
116
164
|
await finalizeAtomicRename(temporaryPath, filePath)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
165
|
+
return { kept, dropped }
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
if (tempCreated) await unlink(temporaryPath).catch(() => { })
|
|
169
|
+
throw error
|
|
170
|
+
}
|
|
122
171
|
}
|
|
123
172
|
|
|
124
173
|
/**
|
|
125
|
-
* 读取 JSONL
|
|
174
|
+
* 读取 JSONL 末条可解析事件的 `id`(DAG tip);从文件尾部最多读 1 MiB,
|
|
175
|
+
* 跳过撕裂/坏行后取最后一个非空且 `id` 非 null 的行;空文件或缺失为 null。
|
|
126
176
|
* @param {string} filePath 文件路径
|
|
127
177
|
* @returns {Promise<string | null>} tip event id
|
|
128
178
|
*/
|
|
@@ -132,14 +182,20 @@ export async function readJsonlTipId(filePath) {
|
|
|
132
182
|
try {
|
|
133
183
|
const { size } = await fh.stat()
|
|
134
184
|
if (!size) return null
|
|
135
|
-
const chunk = Math.min(size,
|
|
185
|
+
const chunk = Math.min(size, 1_048_576)
|
|
136
186
|
const buffer = Buffer.alloc(chunk)
|
|
137
187
|
await fh.read(buffer, 0, chunk, size - chunk)
|
|
138
|
-
const lines = buffer.toString('utf8').split('\n')
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
188
|
+
const lines = buffer.toString('utf8').split('\n')
|
|
189
|
+
for (let index = lines.length - 1; index >= 0; index--) {
|
|
190
|
+
const line = lines[index].trim()
|
|
191
|
+
if (!line) continue
|
|
192
|
+
try {
|
|
193
|
+
const row = JSON.parse(line)
|
|
194
|
+
if (row && typeof row === 'object' && row.id != null) return String(row.id)
|
|
195
|
+
}
|
|
196
|
+
catch { /* 撕裂/坏行:继续向前找 */ }
|
|
197
|
+
}
|
|
198
|
+
return null
|
|
143
199
|
}
|
|
144
200
|
finally {
|
|
145
201
|
await fh.close()
|
|
@@ -151,30 +207,50 @@ export async function readJsonlTipId(filePath) {
|
|
|
151
207
|
}
|
|
152
208
|
|
|
153
209
|
/**
|
|
154
|
-
*
|
|
210
|
+
* 流式写入原始 JSONL 行(临时文件 + fsync + rename),每行自动补 `\n`。
|
|
211
|
+
* 不取锁——调用方需自行持有 `jsonlMutexKey(filePath)`。
|
|
155
212
|
* @param {string} filePath 目标路径
|
|
156
|
-
* @param {
|
|
213
|
+
* @param {Iterable<string>} lines 原始行(不含换行)
|
|
157
214
|
* @returns {Promise<void>}
|
|
158
215
|
*/
|
|
159
|
-
export async function
|
|
216
|
+
export async function writeJsonlLines(filePath, lines) {
|
|
160
217
|
const dir = dirname(filePath)
|
|
161
218
|
await mkdir(dir, { recursive: true })
|
|
162
219
|
const temporaryPath = atomicTemporaryPath(filePath)
|
|
163
|
-
/** @returns {Generator<string>}
|
|
164
|
-
function*
|
|
165
|
-
for (const
|
|
166
|
-
yield `${
|
|
220
|
+
/** @returns {Generator<string>} 带换行的行 */
|
|
221
|
+
function* withEol() {
|
|
222
|
+
for (const line of lines)
|
|
223
|
+
yield `${line}\n`
|
|
224
|
+
}
|
|
225
|
+
await pipeline(Readable.from(withEol()), createWriteStream(temporaryPath, { encoding: 'utf8' }))
|
|
226
|
+
const fileHandle = await open(temporaryPath, 'r+')
|
|
227
|
+
try {
|
|
228
|
+
await fileHandle.sync()
|
|
229
|
+
}
|
|
230
|
+
finally {
|
|
231
|
+
await fileHandle.close()
|
|
167
232
|
}
|
|
168
|
-
await pipeline(Readable.from(lines()), createWriteStream(temporaryPath, { encoding: 'utf8' }))
|
|
169
233
|
await finalizeAtomicRename(temporaryPath, filePath)
|
|
170
234
|
}
|
|
171
235
|
|
|
236
|
+
/**
|
|
237
|
+
* 流式重写 JSONL(临时文件 + fsync + rename),避免大数组 join 的内存峰值。
|
|
238
|
+
* 不取锁——`mailbox/store.mjs` 等调用方在自己的 `jsonlMutexKey` 锁内调用;
|
|
239
|
+
* 需要锁时用 `writeJsonlSynced`。
|
|
240
|
+
* @param {string} filePath 目标路径
|
|
241
|
+
* @param {object[]} records 行对象列表
|
|
242
|
+
* @returns {Promise<void>}
|
|
243
|
+
*/
|
|
244
|
+
export async function writeJsonl(filePath, records) {
|
|
245
|
+
return writeJsonlLines(filePath, records.map(record => JSON.stringify(record)))
|
|
246
|
+
}
|
|
247
|
+
|
|
172
248
|
/**
|
|
173
249
|
* @param {string} filePath JSONL 路径
|
|
174
|
-
* @returns {string}
|
|
250
|
+
* @returns {string} 进程内互斥键(路径已规范化为绝对路径)
|
|
175
251
|
*/
|
|
176
252
|
export function jsonlMutexKey(filePath) {
|
|
177
|
-
return `jsonl:${filePath}`
|
|
253
|
+
return `jsonl:${resolve(filePath)}`
|
|
178
254
|
}
|
|
179
255
|
|
|
180
256
|
/**
|
|
@@ -188,21 +264,23 @@ export async function writeJsonlSynced(filePath, records) {
|
|
|
188
264
|
}
|
|
189
265
|
|
|
190
266
|
/**
|
|
191
|
-
* 追加一行 JSONL 并 `fsync
|
|
267
|
+
* 追加一行 JSONL 并 `fsync`;与 rewrite 共用同一 per-file 互斥锁。
|
|
192
268
|
* @param {string} filePath 目标路径
|
|
193
269
|
* @param {object} record 记录对象
|
|
194
270
|
* @returns {Promise<void>}
|
|
195
271
|
*/
|
|
196
272
|
export async function appendJsonlSynced(filePath, record) {
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
273
|
+
return withAsyncMutex(jsonlMutexKey(filePath), async () => {
|
|
274
|
+
await mkdir(dirname(filePath), { recursive: true })
|
|
275
|
+
const fh = await open(filePath, 'a')
|
|
276
|
+
try {
|
|
277
|
+
await fh.appendFile(`${JSON.stringify(record)}\n`, 'utf8')
|
|
278
|
+
await fh.sync()
|
|
279
|
+
}
|
|
280
|
+
finally {
|
|
281
|
+
await fh.close()
|
|
282
|
+
}
|
|
283
|
+
})
|
|
206
284
|
}
|
|
207
285
|
|
|
208
286
|
/**
|
|
@@ -21,17 +21,17 @@ import { ensureNodeSeed, getNodeHash } from '../../node/identity.mjs'
|
|
|
21
21
|
import { getP2PFeatures, isNodeInitialized } from '../../node/instance.mjs'
|
|
22
22
|
import { nodeDebug } from '../../node/log.mjs'
|
|
23
23
|
|
|
24
|
-
import {
|
|
25
|
-
CENSUS_TAG_FOUNT,
|
|
26
|
-
CENSUS_TAG_X,
|
|
27
|
-
NOSTR_CENSUS_KIND,
|
|
28
|
-
} from './constants.mjs'
|
|
29
24
|
import {
|
|
30
25
|
CENSUS_TARGET_EVENTS,
|
|
31
26
|
estimatePopulation,
|
|
32
27
|
nextInclusionProbability,
|
|
33
28
|
} from './census_math.mjs'
|
|
34
29
|
import { CENSUS_TTL_MS, buildCensusMessage, verifyCensusBytes } from './census_verify.mjs'
|
|
30
|
+
import {
|
|
31
|
+
CENSUS_TAG_FOUNT,
|
|
32
|
+
CENSUS_TAG_X,
|
|
33
|
+
NOSTR_CENSUS_KIND,
|
|
34
|
+
} from './constants.mjs'
|
|
35
35
|
import { resolveRelayConnectTarget } from './relays.mjs'
|
|
36
36
|
|
|
37
37
|
/** census 订阅标签数组(发布用)。 */
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { base64ToBytes } from '../../core/bytes_codec.mjs'
|
|
12
12
|
|
|
13
|
+
import { estimatePopulation } from './census_math.mjs'
|
|
14
|
+
import { CENSUS_TTL_MS, verifyCensusBytes } from './census_verify.mjs'
|
|
13
15
|
import {
|
|
14
16
|
CENSUS_TAG_FOUNT,
|
|
15
17
|
CENSUS_TAG_X,
|
|
@@ -17,8 +19,6 @@ import {
|
|
|
17
19
|
NIP66_BOOTSTRAP_RELAYS,
|
|
18
20
|
NOSTR_CENSUS_KIND,
|
|
19
21
|
} from './constants.mjs'
|
|
20
|
-
import { estimatePopulation } from './census_math.mjs'
|
|
21
|
-
import { CENSUS_TTL_MS, verifyCensusBytes } from './census_verify.mjs'
|
|
22
22
|
|
|
23
23
|
/** NIP-66 relay 注册 kind。 */
|
|
24
24
|
const NIP66_KIND = 30166
|
|
@@ -18,8 +18,8 @@ import {
|
|
|
18
18
|
} from '../internal/signal_crypto.mjs'
|
|
19
19
|
import { noteDiscoveryPeerClue } from '../peer_clue.mjs'
|
|
20
20
|
|
|
21
|
-
import { DEFAULT_RELAY_URLS } from './constants.mjs'
|
|
22
21
|
import { createNostrCensus } from './census.mjs'
|
|
22
|
+
import { DEFAULT_RELAY_URLS } from './constants.mjs'
|
|
23
23
|
import {
|
|
24
24
|
getListenRelays,
|
|
25
25
|
getPeerRoute,
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
Nostr is the fount-network fallback transport (`link` level −∞, discovery `priority` 100). This document covers the **relay pool**, NIP-66 discovery, health scoring, advert relay-field signing, and handshake routing. Top-level surface: [signaling.md](signaling.md) · [transports.md](transports.md) · [mesh.md](mesh.md).
|
|
4
4
|
|
|
5
5
|
Source layout (`discovery/nostr/`):
|
|
6
|
+
|
|
6
7
|
- `index.mjs` — provider (presence / signal / advert subscriptions)
|
|
7
8
|
- `relays.mjs` — pool, health, NIP-66 discovery, normalization, persistence
|
|
8
9
|
- `selection.mjs` — handshake routing, backoff, fanout
|
|
@@ -72,6 +73,7 @@ Lower is better. `recordProbeSuccess` / `recordProbeFailure` / `recordPublishRes
|
|
|
72
73
|
## Handshake routing (`selection.mjs`)
|
|
73
74
|
|
|
74
75
|
`handshakeTargets(nodeHash, attempt)`:
|
|
76
|
+
|
|
75
77
|
- **Round 0**: peer-claimed `listenRelays` top 4 by composite score (own health + peer rtt); else local `workingRelays` top 4; else pinned top 4.
|
|
76
78
|
- **Round ≥1**: backoff `min(2000 · 2^(attempt−1), 60000)`; base on `lastGoodNostrRelays` (expanded via `expandFromHistory` ≤ 16), or weighted-random sample of `workingRelays` (weight `1/score`); round-0 core always included; fanout capped at `MAX_ROUTING_FANOUT` (64).
|
|
77
79
|
- Retries ≤ `MAX_ROUTING_ATTEMPTS` (4).
|
package/docs/signaling.md
CHANGED
|
@@ -25,6 +25,7 @@ When set: send final offer/answer after ICE gathering, dedupe remote signals, qu
|
|
|
25
25
|
`channels` selects which discovery/link media are active — `nostr`, `lan`, `bt`, `webrtc`. Semantics per channel key: `false` **disables**; any other value (`undefined`, `true`, or an object) **enables** — an object is merged over that channel's default config. Channels not mentioned keep their default enabled state.
|
|
26
26
|
|
|
27
27
|
Channel → components:
|
|
28
|
+
|
|
28
29
|
- `nostr` (nostr discovery + nostr link) — config `relay` **replaces** the default public relay list (do not merge defaults back in).
|
|
29
30
|
- `lan` (lan discovery + lan_tcp link)
|
|
30
31
|
- `bt` (bt discovery + ble_gatt link)
|
|
@@ -227,7 +227,7 @@ function mergeRowsWithSources(entries, maxHits, rowKey) {
|
|
|
227
227
|
try { return JSON.stringify(row) }
|
|
228
228
|
catch { return `\0${rows.length}` }
|
|
229
229
|
})
|
|
230
|
-
for (const entry of entries)
|
|
230
|
+
for (const entry of entries)
|
|
231
231
|
for (const row of entry.rows || []) {
|
|
232
232
|
const key = keyOf(row)
|
|
233
233
|
if (!seen.has(key)) {
|
|
@@ -240,7 +240,7 @@ function mergeRowsWithSources(entries, maxHits, rowKey) {
|
|
|
240
240
|
}
|
|
241
241
|
if (entry.sourceNodeHash) sources.get(key).add(entry.sourceNodeHash)
|
|
242
242
|
}
|
|
243
|
-
|
|
243
|
+
|
|
244
244
|
return { rows, sources }
|
|
245
245
|
}
|
|
246
246
|
|
|
@@ -325,10 +325,10 @@ function filterRowsBySource(rows, sources, rowKey, isSourceBlocked) {
|
|
|
325
325
|
return rows.filter(row => {
|
|
326
326
|
let key
|
|
327
327
|
if (rowKey) key = rowKey(row)
|
|
328
|
-
else
|
|
328
|
+
else
|
|
329
329
|
try { key = JSON.stringify(row) }
|
|
330
330
|
catch { return true }
|
|
331
|
-
|
|
331
|
+
|
|
332
332
|
const set = sources.get(key)
|
|
333
333
|
if (!set || set.size === 0) return true
|
|
334
334
|
for (const source of set) if (!isSourceBlocked(source)) return true
|
package/mailbox/store.mjs
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import { mkdir, readFile } from 'node:fs/promises'
|
|
2
|
-
import { dirname } from 'node:path'
|
|
3
|
-
|
|
4
1
|
import { isHex64 } from '../core/hexIds.mjs'
|
|
5
|
-
import { jsonlMutexKey, writeJsonl } from '../dag/storage.mjs'
|
|
2
|
+
import { jsonlMutexKey, readJsonl, writeJsonl } from '../dag/storage.mjs'
|
|
6
3
|
import { mailboxStorePath } from '../node/user_paths.mjs'
|
|
7
4
|
import { withAsyncMutex } from '../utils/async_mutex.mjs'
|
|
8
5
|
|
|
@@ -103,21 +100,7 @@ export function relayHopAfterWireIngress(wireHop, existingStoredHop) {
|
|
|
103
100
|
* @returns {Promise<MailboxRecord[]>} 全部有效记录
|
|
104
101
|
*/
|
|
105
102
|
async function readAll() {
|
|
106
|
-
|
|
107
|
-
const text = await readFile(mailboxStorePath(), 'utf8')
|
|
108
|
-
/** @type {MailboxRecord[]} */
|
|
109
|
-
const rows = []
|
|
110
|
-
for (const line of text.split('\n')) {
|
|
111
|
-
const trimmed = line.trim()
|
|
112
|
-
if (!trimmed) continue
|
|
113
|
-
try {
|
|
114
|
-
rows.push(JSON.parse(trimmed))
|
|
115
|
-
}
|
|
116
|
-
catch { /* skip corrupt line */ }
|
|
117
|
-
}
|
|
118
|
-
return rows
|
|
119
|
-
}
|
|
120
|
-
catch { return [] }
|
|
103
|
+
return /** @type {MailboxRecord[]} */ await readJsonl(mailboxStorePath())
|
|
121
104
|
}
|
|
122
105
|
|
|
123
106
|
/**
|
|
@@ -125,13 +108,11 @@ async function readAll() {
|
|
|
125
108
|
* @returns {Promise<void>}
|
|
126
109
|
*/
|
|
127
110
|
async function writeAll(rows) {
|
|
128
|
-
const filePath = mailboxStorePath()
|
|
129
|
-
await mkdir(dirname(filePath), { recursive: true })
|
|
130
111
|
const now = Date.now()
|
|
131
112
|
const kept = pruneMailboxGlobalFair(
|
|
132
113
|
pruneMailboxBuckets(sortMailboxForRetention(rows.filter(record => record.expiresAt > now))),
|
|
133
114
|
)
|
|
134
|
-
await writeJsonl(
|
|
115
|
+
await writeJsonl(mailboxStorePath(), kept)
|
|
135
116
|
}
|
|
136
117
|
|
|
137
118
|
/**
|
package/package.json
CHANGED
package/pages/index.mjs
CHANGED
|
@@ -32,6 +32,10 @@ function startLive() {
|
|
|
32
32
|
try {
|
|
33
33
|
monitor = createPopulationMonitor({
|
|
34
34
|
relays: relayUrl ? [relayUrl] : undefined,
|
|
35
|
+
/**
|
|
36
|
+
* @param {{ estimate: number, sampleSize: number, eventsInWindow: number, relayUrl: string, relays: number }} snapshot 人口快照
|
|
37
|
+
* @returns {void}
|
|
38
|
+
*/
|
|
35
39
|
onUpdate: snapshot => {
|
|
36
40
|
render(snapshot)
|
|
37
41
|
statusEl.textContent = `监听 ${snapshot.relayUrl}(census,共 ${snapshot.relays} 个 relay)`
|
package/timeline/prune.mjs
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { mkdir } from 'node:fs/promises'
|
|
2
|
-
import { dirname } from 'node:path'
|
|
3
|
-
|
|
4
1
|
import { topologicalCanonicalOrder } from '../dag/index.mjs'
|
|
5
|
-
import {
|
|
2
|
+
import { jsonlMutexKey, readJsonlEntries, writeJsonlLines } from '../dag/storage.mjs'
|
|
6
3
|
import { invalidateTopologicalOrderMemo } from '../federation/topo_order_memo.mjs'
|
|
7
4
|
import { descendantClosureFromTip } from '../governance/branch.mjs'
|
|
5
|
+
import { withAsyncMutex } from '../utils/async_mutex.mjs'
|
|
8
6
|
|
|
9
7
|
/**
|
|
10
8
|
* 将 events.jsonl 裁剪为 checkpoint 起后代闭包(连通子图,非拓扑下标切片)。
|
|
9
|
+
*
|
|
10
|
+
* 整个读-算-写在 `jsonlMutexKey(eventsFilePath)` 锁内完成,避免与并发 append 竞态;
|
|
11
|
+
* 写回的是保留行的原始字节(非 sanitize 后的重序列化),保持无损失。
|
|
11
12
|
* @param {string} eventsFilePath events.jsonl 路径
|
|
12
13
|
* @param {object | null} checkpoint 含 checkpoint_event_id 的快照
|
|
13
14
|
* @param {(row: object) => object} [sanitize] 行规范化
|
|
@@ -16,24 +17,27 @@ import { descendantClosureFromTip } from '../governance/branch.mjs'
|
|
|
16
17
|
export async function pruneEventsJsonlAfterCheckpoint(eventsFilePath, checkpoint, sanitize = row => row) {
|
|
17
18
|
const tipId = checkpoint?.checkpoint_event_id
|
|
18
19
|
if (!tipId) return { pruned: false, kept: 0, dropped: 0 }
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
return withAsyncMutex(jsonlMutexKey(eventsFilePath), async () => {
|
|
21
|
+
const entries = await readJsonlEntries(eventsFilePath, { sanitize })
|
|
22
|
+
if (!entries.length) return { pruned: false, kept: 0, dropped: 0 }
|
|
23
|
+
const byId = new Map(entries.map(entry => [entry.row.id, entry.row]))
|
|
24
|
+
const rawById = new Map(entries.map(entry => [entry.row.id, entry.raw]))
|
|
25
|
+
if (!byId.has(tipId)) return { pruned: false, kept: entries.length, dropped: 0 }
|
|
23
26
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
27
|
+
const keepIds = descendantClosureFromTip(tipId, byId)
|
|
28
|
+
const order = topologicalCanonicalOrder(entries.map(entry => ({
|
|
29
|
+
id: entry.row.id,
|
|
30
|
+
prev_event_ids: entry.row.prev_event_ids,
|
|
31
|
+
hlc: entry.row.hlc,
|
|
32
|
+
node_id: entry.row.node_id,
|
|
33
|
+
sender: entry.row.sender,
|
|
34
|
+
})))
|
|
35
|
+
const keptIds = order.filter(id => keepIds.has(id))
|
|
36
|
+
const dropped = entries.length - keptIds.length
|
|
37
|
+
if (dropped <= 0) return { pruned: false, kept: keptIds.length, dropped: 0 }
|
|
38
|
+
const keptRaws = keptIds.map(id => rawById.get(id)).filter(raw => raw !== undefined)
|
|
39
|
+
await writeJsonlLines(eventsFilePath, keptRaws)
|
|
40
|
+
invalidateTopologicalOrderMemo(eventsFilePath)
|
|
41
|
+
return { pruned: true, kept: keptIds.length, dropped }
|
|
42
|
+
})
|
|
39
43
|
}
|
package/timeline/retention.mjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { mkdir } from 'node:fs/promises'
|
|
2
|
-
import { dirname } from 'node:path'
|
|
3
|
-
|
|
4
1
|
import { topologicalCanonicalOrder } from '../dag/index.mjs'
|
|
5
|
-
import {
|
|
2
|
+
import { jsonlMutexKey, readJsonlEntries, writeJsonlLines } from '../dag/storage.mjs'
|
|
6
3
|
import { invalidateTopologicalOrderMemo } from '../federation/topo_order_memo.mjs'
|
|
7
4
|
import { computeDagTipIdsFromEvents, selectConsensusBranchTip } from '../governance/branch.mjs'
|
|
8
5
|
import { computeRetentionKeepIds } from '../node/retention_policy.mjs'
|
|
6
|
+
import { withAsyncMutex } from '../utils/async_mutex.mjs'
|
|
9
7
|
|
|
10
8
|
/**
|
|
9
|
+
* 整个读-算-写在 `jsonlMutexKey(eventsFilePath)` 锁内完成,避免与并发 append 竞态;
|
|
10
|
+
* 写回的是保留行的原始字节(非 sanitize 后的重序列化),保持无损失。
|
|
11
11
|
* @param {string} eventsFilePath events.jsonl 路径
|
|
12
12
|
* @param {object | null} checkpointHint 检查点提示
|
|
13
13
|
* @param {{ maxDepth: number, maxMs: number, anchorTypes: Set<string> }} policy 保留策略
|
|
@@ -20,35 +20,39 @@ export async function enforceTimelineEventRetention(
|
|
|
20
20
|
policy,
|
|
21
21
|
sanitize = row => row,
|
|
22
22
|
) {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
23
|
+
return withAsyncMutex(jsonlMutexKey(eventsFilePath), async () => {
|
|
24
|
+
const entries = await readJsonlEntries(eventsFilePath, { sanitize })
|
|
25
|
+
if (!entries.length) return { pruned: false, kept: 0, dropped: 0 }
|
|
26
|
+
const maxDepth = Math.max(256, Number(policy.maxDepth) || 200_000)
|
|
27
|
+
const maxMs = Math.max(3_600_000, Number(policy.maxMs) || 365 * 24 * 3600 * 1000)
|
|
28
|
+
const cutoffWall = Date.now() - maxMs
|
|
29
|
+
const events = entries.map(entry => entry.row)
|
|
30
|
+
const rawById = new Map(entries.map(entry => [entry.row.id, entry.raw]))
|
|
31
|
+
const byId = new Map(events.map(e => [e.id, e]))
|
|
32
|
+
const order = topologicalCanonicalOrder(events.map(e => ({
|
|
33
|
+
id: e.id,
|
|
34
|
+
prev_event_ids: e.prev_event_ids,
|
|
35
|
+
hlc: e.hlc,
|
|
36
|
+
node_id: e.node_id,
|
|
37
|
+
sender: e.sender,
|
|
38
|
+
})))
|
|
39
|
+
const tips = computeDagTipIdsFromEvents(events)
|
|
40
|
+
const branchTipId = selectConsensusBranchTip(tips, byId)
|
|
41
|
+
const checkpointTipId = checkpointHint?.checkpoint_event_id || null
|
|
42
|
+
const keepIds = computeRetentionKeepIds(order, byId, {
|
|
43
|
+
maxDepth,
|
|
44
|
+
cutoffWall,
|
|
45
|
+
anchorTypes: policy.anchorTypes,
|
|
46
|
+
checkpointTipId,
|
|
47
|
+
branchTipId,
|
|
48
|
+
})
|
|
49
|
+
if (keepIds.size >= events.length) return { pruned: false, kept: events.length, dropped: 0 }
|
|
50
|
+
const keptIds = order.filter(id => keepIds.has(id))
|
|
51
|
+
const dropped = events.length - keptIds.length
|
|
52
|
+
if (dropped <= 0) return { pruned: false, kept: keptIds.length, dropped: 0 }
|
|
53
|
+
const keptRaws = keptIds.map(id => rawById.get(id)).filter(raw => raw !== undefined)
|
|
54
|
+
await writeJsonlLines(eventsFilePath, keptRaws)
|
|
55
|
+
invalidateTopologicalOrderMemo(eventsFilePath)
|
|
56
|
+
return { pruned: true, kept: keptIds.length, dropped }
|
|
45
57
|
})
|
|
46
|
-
if (keepIds.size >= events.length) return { pruned: false, kept: events.length, dropped: 0 }
|
|
47
|
-
const kept = order.map(id => byId.get(id)).filter(ev => ev && keepIds.has(ev.id))
|
|
48
|
-
const dropped = events.length - kept.length
|
|
49
|
-
if (dropped <= 0) return { pruned: false, kept: kept.length, dropped: 0 }
|
|
50
|
-
await mkdir(dirname(eventsFilePath), { recursive: true })
|
|
51
|
-
await writeJsonlSynced(eventsFilePath, kept)
|
|
52
|
-
invalidateTopologicalOrderMemo(eventsFilePath)
|
|
53
|
-
return { pruned: true, kept: kept.length, dropped }
|
|
54
58
|
}
|
package/utils/async_mutex.mjs
CHANGED
|
@@ -25,13 +25,21 @@ function mutexState(lockKey) {
|
|
|
25
25
|
*/
|
|
26
26
|
function releaseMutex(lockKey, state) {
|
|
27
27
|
const next = state.queue.shift()
|
|
28
|
-
if (next)
|
|
28
|
+
if (next) queueMicrotask(next)
|
|
29
29
|
else {
|
|
30
30
|
state.locked = false
|
|
31
31
|
mutexes.delete(lockKey)
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* 当前存活的互斥键数量(仅供测试观察锁表是否回落到基线)。
|
|
37
|
+
* @returns {number} `mutexes` 中的键数量
|
|
38
|
+
*/
|
|
39
|
+
export function activeMutexCount() {
|
|
40
|
+
return mutexes.size
|
|
41
|
+
}
|
|
42
|
+
|
|
35
43
|
/**
|
|
36
44
|
* @param {string} key 锁键
|
|
37
45
|
* @param {() => Promise<T>} criticalSection 临界区
|
|
@@ -39,15 +47,14 @@ function releaseMutex(lockKey, state) {
|
|
|
39
47
|
* @template T
|
|
40
48
|
*/
|
|
41
49
|
export async function withAsyncMutex(key, criticalSection) {
|
|
42
|
-
const
|
|
43
|
-
const state = mutexState(lockKey)
|
|
50
|
+
const state = mutexState(key)
|
|
44
51
|
return new Promise((resolve, reject) => {
|
|
45
52
|
/** 执行临界区并最终释放锁 */
|
|
46
53
|
const run = () => {
|
|
47
54
|
Promise.resolve()
|
|
48
55
|
.then(criticalSection)
|
|
49
56
|
.then(resolve, reject)
|
|
50
|
-
.finally(() => releaseMutex(
|
|
57
|
+
.finally(() => releaseMutex(key, state))
|
|
51
58
|
}
|
|
52
59
|
if (state.locked) state.queue.push(run)
|
|
53
60
|
else {
|
package/utils/shuffle.mjs
CHANGED
package/wire/adapter.mjs
CHANGED