@steve02081504/fount-p2p 0.0.18 → 0.0.20

package/dag/storage.mjs

@@ -1,62 +1,18 @@
 import { Buffer } from 'node:buffer'
-import { randomUUID } from 'node:crypto'
 import { createReadStream, createWriteStream } from 'node:fs'
-import { appendFile, mkdir, open, readFile, rename, unlink, writeFile } from 'node:fs/promises'
+import { appendFile, mkdir, open, readFile, writeFile } from 'node:fs/promises'
 import { dirname } from 'node:path'
 import { createInterface } from 'node:readline'
 import { Readable } from 'node:stream'
 import { pipeline } from 'node:stream/promises'
-import { setTimeout as sleep } from 'node:timers/promises'
 
+import { atomicTemporaryPath, finalizeAtomicRename } from '../utils/atomic_fs.mjs'
 import { withAsyncMutex } from '../utils/async_mutex.mjs'
 
+export { finalizeAtomicRename } from '../utils/atomic_fs.mjs'
+
 /** 流式重写 JSONL 时分块写入的行数上限 */
 const WRITE_JSONL_CHUNK_LINES = 1000
-/** Windows 上 rename 可能被短暂占用，做几次短退避重试。 */
-const ATOMIC_RENAME_RETRY_DELAYS_MS = [0, 10, 25, 50, 100]
-const ATOMIC_RENAME_RETRY_CODES = new Set(['EPERM', 'EBUSY', 'EACCES'])
-
-/**
- * @param {string} filePath 目标路径
- * @returns {string} 唯一临时文件路径
- */
-function atomicTemporaryPath(filePath) {
-	return `${filePath}.tmp.${process.pid}.${randomUUID()}`
-}
-
-/**
- * @param {string} temporaryPath 临时文件路径
- * @returns {Promise<void>}
- */
-async function cleanupAtomicTemporary(temporaryPath) {
-	try { await unlink(temporaryPath) } catch { /* ok */ }
-}
-
-/**
- * 完成原子写的最终 rename；若目标目录已在 cleanup 中消失，则清理残余临时文件后静默返回。
- * @param {string} temporaryPath 临时文件路径
- * @param {string} filePath 最终目标路径
- * @returns {Promise<boolean>} 是否已成功落到目标路径
- */
-export async function finalizeAtomicRename(temporaryPath, filePath) {
-	/** @type {NodeJS.ErrnoException | undefined} */
-	let lastError
-	for (const delayMs of ATOMIC_RENAME_RETRY_DELAYS_MS) {
-		if (delayMs) await sleep(delayMs)
-		try {
-			await rename(temporaryPath, filePath)
-			return true
-		}
-		catch (error) {
-			lastError = error
-			if (ATOMIC_RENAME_RETRY_CODES.has(error?.code)) continue
-			break
-		}
-	}
-	await cleanupAtomicTemporary(temporaryPath)
-	if (lastError?.code === 'ENOENT') return false
-	throw lastError
-}
 
 /**
  * @param {string} line JSONL 单行

package/files/chunk_fetch.mjs

@@ -3,15 +3,24 @@ import { randomUUID } from 'node:crypto'
 import { bytesToBase64 } from '../core/bytes_codec.mjs'
 import {
 	MAX_PENDING_CHUNK_FETCHES,
-	pendingChunkFetches,
 	registerChunkFetchWait,
 } from '../federation/chunk_fetch_pending.mjs'
+import { ms } from '../utils/duration.mjs'
+import { createInflightTable } from '../utils/inflight_table.mjs'
 
 import { verifiedChunkBytes } from './chunk_fetch_verify.mjs'
 import { fetchFederationChunk, resolveNodeHash } from './chunk_provider_registry.mjs'
 import { getChunk, hasChunk, putChunk } from './chunk_store.mjs'
 import { fanoutFedFetch } from './fetch_fanout.mjs'
 
+const DEFAULT_CHUNK_FETCH_TIMEOUT_MS = ms('8s')
+
+/** 同 username+hash 共享一次 fanout；队满只丢已超基础超时的队首。 */
+const chunkInflight = createInflightTable({
+	maxSize: MAX_PENDING_CHUNK_FETCHES,
+	baseTimeoutMs: DEFAULT_CHUNK_FETCH_TIMEOUT_MS,
+})
+
 /**
  * @typedef {{
  *   username: string,
@@ -42,19 +51,27 @@ export async function fetchChunk(context) {
 		}
 	}
 
-	if (pendingChunkFetches.size >= MAX_PENDING_CHUNK_FETCHES) return null
+	const inflightKey = `${username}\0${hash}`
+	const shared = chunkInflight.acquire(inflightKey, () => {
+		const requestId = randomUUID()
+		const wait = registerChunkFetchWait(requestId, hash, DEFAULT_CHUNK_FETCH_TIMEOUT_MS)
+		void (async () => {
+			try {
+				const { nodeHash } = await resolveNodeHash(username)
+				await fanoutFedFetch(username, 'fed_chunk_get', {
+					requestId,
+					nodeHash,
+					chunkHash: hash,
+					ownerEntityHash: context.ownerEntityHash,
+				})
+			}
+			catch { /* pending wait 超时/cancel 负责 settle */ }
+		})()
+		return { done: wait.done, cancel: wait.cancel }
+	})
+	if (!shared) return null
 
-	const requestId = randomUUID()
-	const { done } = registerChunkFetchWait(requestId, hash, 8000)
-	const { nodeHash } = await resolveNodeHash(username)
-	const payload = {
-		requestId,
-		nodeHash,
-		chunkHash: hash,
-		ownerEntityHash: context.ownerEntityHash,
-	}
-	await fanoutFedFetch(username, 'fed_chunk_get', payload)
-	const result = await done
+	const result = await shared
 	const verified = verifiedChunkBytes(hash, result)
 	if (verified) {
 		await putChunk(hash, verified)

package/files/manifest_fetch.mjs

@@ -3,11 +3,12 @@ import { randomUUID } from 'node:crypto'
 import {
 	manifestFetchExpectedKey,
 	MAX_PENDING_MANIFEST_FETCHES,
-	pendingManifestFetches,
 	registerManifestFetchWait,
 } from '../federation/manifest_fetch_pending.mjs'
 import { isWritableLocalEntity } from '../node/identity.mjs'
 import { getEntityStore } from '../node/instance.mjs'
+import { ms } from '../utils/duration.mjs'
+import { createInflightTable } from '../utils/inflight_table.mjs'
 
 import { resolveNodeHash } from './chunk_provider_registry.mjs'
 import { loadFileManifest, saveFileManifest } from './evfs.mjs'
@@ -15,11 +16,18 @@ import { fanoutFedFetch } from './fetch_fanout.mjs'
 import { normalizeFileManifest } from './manifest.mjs'
 import { shouldPreferIncomingPublicManifest } from './public_manifest.mjs'
 
-const DEFAULT_MANIFEST_FETCH_TIMEOUT_MS = 8000
+const DEFAULT_MANIFEST_FETCH_TIMEOUT_MS = ms('8s')
+
+/** 同 username+owner+path 共享一次 fanout；队满只丢已超基础超时的队首。 */
+const manifestInflight = createInflightTable({
+	maxSize: MAX_PENDING_MANIFEST_FETCHES,
+	baseTimeoutMs: DEFAULT_MANIFEST_FETCH_TIMEOUT_MS,
+})
 
 /**
  * 拉取公开 manifest；默认不写盘。`cache: true` 或 `cachePublicManifest` 才缓存。
  * 本地已有 publicSig 时仍会 fanout 再校验，按 publishedAt 择新；超时则回退本地。
+ * 同 key in-flight 去重；调用方外层超时不 abort，后台继续填缓存。
  * @param {{ username: string, ownerEntityHash: string, logicalPath: string, cache?: boolean, timeoutMs?: number }} context - 拉取上下文
  * @returns {Promise<import('./manifest.mjs').FileManifest | null>} 验签后的 manifest，失败为 null
  */
@@ -29,30 +37,37 @@ export async function fetchPublicManifest(context) {
 	const { username } = context
 	if (!ownerEntityHash || !logicalPath || !username) return null
 
-	const local = await loadFileManifest(ownerEntityHash, logicalPath)
-	const hasLocalPublic = local?.transferKeyDescriptor?.type === 'public' && !!local?.meta?.publicSig
-
-	if (pendingManifestFetches.size >= MAX_PENDING_MANIFEST_FETCHES)
-		return hasLocalPublic ? local : null
-
 	const timeoutMs = Number(context.timeoutMs) > 0
 		? Number(context.timeoutMs)
 		: DEFAULT_MANIFEST_FETCH_TIMEOUT_MS
-	const requestId = randomUUID()
-	const { done } = registerManifestFetchWait(
-		requestId,
-		manifestFetchExpectedKey(ownerEntityHash, logicalPath),
-		timeoutMs,
-	)
-	const { nodeHash } = await resolveNodeHash(username)
-	const payload = {
-		requestId,
-		nodeHash,
-		ownerEntityHash,
-		logicalPath,
-	}
-	await fanoutFedFetch(username, 'fed_manifest_get', payload)
-	const result = await done
+	const expectedKey = manifestFetchExpectedKey(ownerEntityHash, logicalPath)
+	const inflightKey = `${username}\0${expectedKey}`
+
+	// 先挂 in-flight（同步），再读本地 — 避免并发调用在 await 间隙各自 start
+	const localPromise = loadFileManifest(ownerEntityHash, logicalPath)
+	const shared = manifestInflight.acquire(inflightKey, () => {
+		const requestId = randomUUID()
+		const wait = registerManifestFetchWait(requestId, expectedKey, timeoutMs)
+		void (async () => {
+			try {
+				const { nodeHash } = await resolveNodeHash(username)
+				await fanoutFedFetch(username, 'fed_manifest_get', {
+					requestId,
+					nodeHash,
+					ownerEntityHash,
+					logicalPath,
+				})
+			}
+			catch { /* pending wait 超时/cancel 负责 settle */ }
+		})()
+		return { done: wait.done, cancel: wait.cancel }
+	})
+
+	const local = await localPromise
+	const hasLocalPublic = local?.transferKeyDescriptor?.type === 'public' && !!local?.meta?.publicSig
+	if (!shared) return hasLocalPublic ? local : null
+
+	const result = await shared
 
 	if (result && (!hasLocalPublic || shouldPreferIncomingPublicManifest(local, result))) {
 		if (context.cache === true)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@steve02081504/fount-p2p",
-	"version": "0.0.18",
+	"version": "0.0.20",
 	"description": "fount federation P2P layer — link, trust graph, mailbox, DAG, EVFS.",
 	"keywords": [
 		"network",

package/utils/atomic_fs.mjs

@@ -0,0 +1,87 @@
+/**
+ * 原子落盘：唯一临时路径 + rename（Windows 短暂占用时短退避重试）。
+ */
+import { randomUUID } from 'node:crypto'
+import fs from 'node:fs'
+import { rename, unlink } from 'node:fs/promises'
+import { setTimeout as sleep } from 'node:timers/promises'
+
+/** Windows 上 rename 可能被短暂占用，做几次短退避重试。 */
+const ATOMIC_RENAME_RETRY_DELAYS_MS = [0, 10, 25, 50, 100]
+const ATOMIC_RENAME_RETRY_CODES = new Set(['EPERM', 'EBUSY', 'EACCES'])
+
+/**
+ * @param {string} filePath 目标路径
+ * @returns {string} 唯一临时文件路径
+ */
+export function atomicTemporaryPath(filePath) {
+	return `${filePath}.tmp.${process.pid}.${randomUUID()}`
+}
+
+/**
+ * @param {string} temporaryPath 临时文件路径
+ * @returns {Promise<void>}
+ */
+async function cleanupAtomicTemporary(temporaryPath) {
+	try { await unlink(temporaryPath) } catch { /* ok */ }
+}
+
+/**
+ * @param {string} temporaryPath 临时文件路径
+ * @returns {void}
+ */
+function cleanupAtomicTemporarySync(temporaryPath) {
+	try { fs.unlinkSync(temporaryPath) } catch { /* ok */ }
+}
+
+/**
+ * 完成原子写的最终 rename；若目标目录已在 cleanup 中消失，则清理残余临时文件后静默返回。
+ * @param {string} temporaryPath 临时文件路径
+ * @param {string} filePath 最终目标路径
+ * @returns {Promise<boolean>} 是否已成功落到目标路径
+ */
+export async function finalizeAtomicRename(temporaryPath, filePath) {
+	/** @type {NodeJS.ErrnoException | undefined} */
+	let lastError
+	for (const delayMs of ATOMIC_RENAME_RETRY_DELAYS_MS) {
+		if (delayMs) await sleep(delayMs)
+		try {
+			await rename(temporaryPath, filePath)
+			return true
+		}
+		catch (error) {
+			lastError = error
+			if (ATOMIC_RENAME_RETRY_CODES.has(error?.code)) continue
+			break
+		}
+	}
+	await cleanupAtomicTemporary(temporaryPath)
+	if (lastError?.code === 'ENOENT') return false
+	throw lastError
+}
+
+/**
+ * `finalizeAtomicRename` 的同步版。
+ * @param {string} temporaryPath 临时文件路径
+ * @param {string} filePath 最终目标路径
+ * @returns {boolean} 是否已成功落到目标路径
+ */
+export function finalizeAtomicRenameSync(temporaryPath, filePath) {
+	/** @type {NodeJS.ErrnoException | undefined} */
+	let lastError
+	for (const delayMs of ATOMIC_RENAME_RETRY_DELAYS_MS) {
+		if (delayMs) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delayMs)
+		try {
+			fs.renameSync(temporaryPath, filePath)
+			return true
+		}
+		catch (error) {
+			lastError = error
+			if (ATOMIC_RENAME_RETRY_CODES.has(error?.code)) continue
+			break
+		}
+	}
+	cleanupAtomicTemporarySync(temporaryPath)
+	if (lastError?.code === 'ENOENT') return false
+	throw lastError
+}

package/utils/inflight_table.mjs

@@ -0,0 +1,95 @@
+/**
+ * In-flight 去重表：同 key 复用并 touch 到队尾；队满时仅淘汰「已超过 baseTimeout」的队首。
+ * 双窗口：size >= maxSize 且 entry 年龄 >= baseTimeoutMs 才 cancel。
+ *
+ * @template T
+ * @param {{ maxSize: number, baseTimeoutMs: number, now?: () => number }} options 容量与基础超时
+ * @returns {{
+ *   size: () => number,
+ *   has: (key: string) => boolean,
+ *   acquire: (key: string, start: () => { done: Promise<T>, cancel: () => void }) => Promise<T> | null,
+ *   clear: () => void,
+ * }} 表句柄
+ */
+export function createInflightTable(options) {
+	const maxSize = Math.max(1, Math.floor(Number(options.maxSize) || 1))
+	const baseTimeoutMs = Math.max(0, Number(options.baseTimeoutMs) || 0)
+	const now = options.now || Date.now
+
+	/** @type {Map<string, { done: Promise<T>, cancel: () => void, startedAt: number }>} */
+	const map = new Map()
+
+	/**
+	 * 队满时从队首取消已超时项。
+	 * @returns {void}
+	 */
+	function pruneAgedOverCap() {
+		const t = now()
+		while (map.size >= maxSize) {
+			const oldestKey = map.keys().next().value
+			const entry = map.get(oldestKey)
+			if (!entry || t - entry.startedAt < baseTimeoutMs) break
+			map.delete(oldestKey)
+			entry.cancel()
+		}
+	}
+
+	/**
+	 * @param {string} key 逻辑键
+	 * @param {{ done: Promise<T>, cancel: () => void, startedAt: number }} entry 条目
+	 * @returns {void}
+	 */
+	function track(key, entry) {
+		entry.done.finally(() => {
+			if (map.get(key) === entry) map.delete(key)
+		})
+		map.set(key, entry)
+	}
+
+	return {
+		/**
+		 * @returns {number} 当前 in-flight 数
+		 */
+		size: () => map.size,
+		/**
+		 * @param {string} key 逻辑键
+		 * @returns {boolean} 是否在飞
+		 */
+		has: key => map.has(key),
+		/**
+		 * 复用或启动；队满且无法淘汰超时项时返回 null（拒绝新开）。
+		 * @param {string} key 逻辑键
+		 * @param {() => { done: Promise<T>, cancel: () => void }} start 仅在未命中时调用
+		 * @returns {Promise<T> | null} 共享 Promise，或拒绝新开
+		 */
+		acquire(key, start) {
+			const existing = map.get(key)
+			if (existing) {
+				map.delete(key)
+				map.set(key, existing)
+				pruneAgedOverCap()
+				return existing.done
+			}
+
+			pruneAgedOverCap()
+			if (map.size >= maxSize) return null
+
+			const started = start()
+			const entry = {
+				done: started.done,
+				cancel: started.cancel,
+				startedAt: now(),
+			}
+			track(key, entry)
+			return entry.done
+		},
+		/**
+		 * 取消全部并清空（测试用）。
+		 * @returns {void}
+		 */
+		clear() {
+			for (const entry of map.values()) entry.cancel()
+			map.clear()
+		},
+	}
+}

package/utils/json_io.mjs

@@ -2,6 +2,8 @@ import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import path from 'node:path'
 
+import { atomicTemporaryPath, finalizeAtomicRename, finalizeAtomicRenameSync } from './atomic_fs.mjs'
+
 /**
  * @param {string} filePath 绝对路径
  * @returns {Promise<object | null>} JSON 或 null
@@ -24,9 +26,10 @@ export async function readJsonFile(filePath) {
  */
 export async function writeJsonFile(filePath, data) {
 	await fsp.mkdir(path.dirname(filePath), { recursive: true })
-	const temporaryPath = `${filePath}.tmp`
+	const temporaryPath = atomicTemporaryPath(filePath)
 	await fsp.writeFile(temporaryPath, `${JSON.stringify(data, null, 2)}\n`, 'utf8')
-	await fsp.rename(temporaryPath, filePath)
+	if (!await finalizeAtomicRename(temporaryPath, filePath))
+		throw Object.assign(new Error(`ENOENT: atomic rename failed for ${filePath}`), { code: 'ENOENT' })
 }
 
 /**
@@ -51,7 +54,8 @@ export function readJsonFileSync(filePath) {
  */
 export function writeJsonFileSync(filePath, data) {
 	fs.mkdirSync(path.dirname(filePath), { recursive: true })
-	const temporaryPath = `${filePath}.tmp`
+	const temporaryPath = atomicTemporaryPath(filePath)
 	fs.writeFileSync(temporaryPath, `${JSON.stringify(data, null, 2)}\n`, 'utf8')
-	fs.renameSync(temporaryPath, filePath)
+	if (!finalizeAtomicRenameSync(temporaryPath, filePath))
+		throw Object.assign(new Error(`ENOENT: atomic rename failed for ${filePath}`), { code: 'ENOENT' })
 }