@armadra/agent 0.7.4 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/CHANGELOG.zh-CN.md +11 -0
- package/dist/ai/apis/anthropic-request.js +6 -1
- package/dist/ai/apis/cache-params.js +2 -1
- package/dist/ai/apis/google-request.js +2 -1
- package/dist/ai/apis/openai-request.js +3 -1
- package/dist/ai/apis/openai-responses-request.js +2 -1
- package/dist/ai/json-body.d.ts +33 -1
- package/dist/ai/json-body.js +96 -18
- package/dist/bundle/ama.cjs +699 -352
- package/dist/cli/bootstrap.js +13 -5
- package/dist/cli/deps.d.ts +1 -1
- package/dist/modes/acp/acp-server.js +17 -15
- package/dist/modes/acp/acp-sessions.d.ts +14 -1
- package/dist/modes/acp/acp-sessions.js +44 -1
- package/dist/session/manager.d.ts +16 -0
- package/dist/session/manager.js +70 -14
- package/dist/session/offload.d.ts +51 -0
- package/dist/session/offload.js +195 -0
- package/dist/session/store.d.ts +12 -4
- package/dist/session/store.js +18 -8
- package/dist/tools/read-lines.d.ts +39 -0
- package/dist/tools/read-lines.js +163 -86
- package/dist/tools/read.js +3 -3
- package/docs/acp.md +6 -3
- package/docs/en/acp.md +2 -2
- package/docs/session-format.md +10 -0
- package/package.json +1 -1
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 会话图片卸载(#170):被 `context_edit` 改写的旧消息不再进上下文,内存里只留它在会话文件中的行位置,
|
|
3
|
+
* 图片 base64 需要时再从文件回读。JSONL 格式不变。
|
|
4
|
+
*
|
|
5
|
+
* - 对象:活动分支上、被活动分支上的 `context_edit`(任何 reason)改写的 `message` 条目里的 `image` 块;
|
|
6
|
+
* 不在活动分支上的条目一律保持原文(rewind 回到编辑点之前、`/tree` 换分支时回读);
|
|
7
|
+
* - 打开文件时先扫一遍 `context_edit` 行(只解码行首判断类型)收集目标,读条目时目标里的图片解析完
|
|
8
|
+
* 随即剥掉,峰值不再含全部旧图;随后按活动分支对齐(不在分支上的编辑目标回读);
|
|
9
|
+
* - 卸载**不改共享的 ImageBlock**(驻留 block 被其它条目 / 会话共享):`entry.message` 换成浅拷贝,
|
|
10
|
+
* `content` 换新数组,image 块换成 `data: ""` 的占位;
|
|
11
|
+
* - 回读:按行位置 `readSync` 一行、`JSON.parse`,校验 id 与内容块数,图片经 `internImage` 驻留后
|
|
12
|
+
* **就地**放回占位所在的数组(之前拿到同一消息对象的调用方随之看到原图);读不回(文件被外部改写)时
|
|
13
|
+
* 保留 `""` 并告警,不再重试;
|
|
14
|
+
* - `original(entry)`:给 `getEntries` / `fork` 的原文副本,不改内存状态。
|
|
15
|
+
*/
|
|
16
|
+
import { closeSync, openSync, readSync } from "node:fs";
|
|
17
|
+
import { internImage } from "../ai/image-intern.js";
|
|
18
|
+
import { forEachLineSync, lineTypeOf } from "./line-reader.js";
|
|
19
|
+
import { collectContextEdits } from "./projection.js";
|
|
20
|
+
function isImage(block) {
|
|
21
|
+
return (typeof block === "object" &&
|
|
22
|
+
block !== null &&
|
|
23
|
+
block.type === "image" &&
|
|
24
|
+
typeof block.data === "string");
|
|
25
|
+
}
|
|
26
|
+
function contentOf(entry) {
|
|
27
|
+
if (entry.type !== "message")
|
|
28
|
+
return undefined;
|
|
29
|
+
const content = entry.message.content;
|
|
30
|
+
return Array.isArray(content) ? content : undefined;
|
|
31
|
+
}
|
|
32
|
+
/** `message` 条目的内容里有图片块。 */
|
|
33
|
+
export function hasImages(entry) {
|
|
34
|
+
return contentOf(entry)?.some(isImage) === true;
|
|
35
|
+
}
|
|
36
|
+
/** 读文件里 `locator` 处的一行并解析。 */
|
|
37
|
+
export function readLineAt(file, locator) {
|
|
38
|
+
const fd = openSync(file, "r");
|
|
39
|
+
try {
|
|
40
|
+
const buf = Buffer.allocUnsafe(locator.length);
|
|
41
|
+
let read = 0;
|
|
42
|
+
while (read < locator.length) {
|
|
43
|
+
const n = readSync(fd, buf, read, locator.length - read, locator.offset + read);
|
|
44
|
+
if (n === 0)
|
|
45
|
+
throw new Error("unexpected end of file");
|
|
46
|
+
read += n;
|
|
47
|
+
}
|
|
48
|
+
return JSON.parse(buf.toString("utf8"));
|
|
49
|
+
}
|
|
50
|
+
finally {
|
|
51
|
+
closeSync(fd);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/** 文件里全部 `context_edit` 的目标 id(只解析 context_edit 行;读不了时为空集)。 */
|
|
55
|
+
export function editTargets(file) {
|
|
56
|
+
const targets = new Set();
|
|
57
|
+
try {
|
|
58
|
+
forEachLineSync(file, (line) => {
|
|
59
|
+
if (lineTypeOf(line)?.type !== "context_edit")
|
|
60
|
+
return;
|
|
61
|
+
try {
|
|
62
|
+
const id = JSON.parse(line.toString("utf8")).targetId;
|
|
63
|
+
if (typeof id === "string")
|
|
64
|
+
targets.add(id);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
// 末尾半行:随后的 readSessionLines 修复
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
// 读不了:由随后的 readSessionLines 报错
|
|
73
|
+
}
|
|
74
|
+
return targets;
|
|
75
|
+
}
|
|
76
|
+
const placeholder = (block) => ({ ...block, data: "" });
|
|
77
|
+
export class ImageOffload {
|
|
78
|
+
warn;
|
|
79
|
+
/** 含图 message 条目的行位置(只有已落盘的条目才有)。 */
|
|
80
|
+
locators = new Map();
|
|
81
|
+
offloaded = new Set();
|
|
82
|
+
constructor(warn = () => { }) {
|
|
83
|
+
this.warn = warn;
|
|
84
|
+
}
|
|
85
|
+
/** 记下条目的行位置;不含图片的条目忽略。 */
|
|
86
|
+
track(entry, locator) {
|
|
87
|
+
if (hasImages(entry))
|
|
88
|
+
this.locators.set(entry.id, locator);
|
|
89
|
+
}
|
|
90
|
+
/** 读文件时逐行调用:记下行位置;`strip` 时就地换成占位(刚解析的对象,别处没有引用)。 */
|
|
91
|
+
load(line, locator, strip) {
|
|
92
|
+
if (typeof line.id !== "string" || !hasImages(line))
|
|
93
|
+
return;
|
|
94
|
+
this.locators.set(line.id, locator);
|
|
95
|
+
if (!strip)
|
|
96
|
+
return;
|
|
97
|
+
const content = contentOf(line);
|
|
98
|
+
content.forEach((block, i) => {
|
|
99
|
+
if (isImage(block))
|
|
100
|
+
content[i] = placeholder(block);
|
|
101
|
+
});
|
|
102
|
+
this.offloaded.add(line.id);
|
|
103
|
+
}
|
|
104
|
+
isOffloaded(id) {
|
|
105
|
+
return this.offloaded.has(id);
|
|
106
|
+
}
|
|
107
|
+
/** 已卸载的条目数(测试与诊断用)。 */
|
|
108
|
+
get size() {
|
|
109
|
+
return this.offloaded.size;
|
|
110
|
+
}
|
|
111
|
+
/** 把条目里的图片换成占位;没有行位置(未落盘、不含图)或已卸载时返回 false。 */
|
|
112
|
+
offload(entry) {
|
|
113
|
+
if (entry.type !== "message" || this.offloaded.has(entry.id))
|
|
114
|
+
return false;
|
|
115
|
+
const content = contentOf(entry);
|
|
116
|
+
if (content === undefined || !this.locators.has(entry.id))
|
|
117
|
+
return false;
|
|
118
|
+
entry.message = {
|
|
119
|
+
...entry.message,
|
|
120
|
+
content: content.map((block) => (isImage(block) ? placeholder(block) : block)),
|
|
121
|
+
};
|
|
122
|
+
this.offloaded.add(entry.id);
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
/** 就地放回图片;读不回时返回 false(占位保留,之后不再卸载 / 回读这一条)。 */
|
|
126
|
+
hydrate(file, entry) {
|
|
127
|
+
if (!this.offloaded.has(entry.id))
|
|
128
|
+
return true;
|
|
129
|
+
this.offloaded.delete(entry.id);
|
|
130
|
+
const source = this.read(file, entry);
|
|
131
|
+
const content = contentOf(entry);
|
|
132
|
+
if (source === undefined || content === undefined)
|
|
133
|
+
return false;
|
|
134
|
+
const originals = contentOf(source) ?? [];
|
|
135
|
+
content.forEach((block, i) => {
|
|
136
|
+
const original = originals[i];
|
|
137
|
+
if (isImage(block) && block.data === "" && isImage(original)) {
|
|
138
|
+
content[i] = internImage(original);
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
/** 原文:已卸载的条目从文件读一份新副本(图片驻留),否则原样返回。读不回时返回占位版本。 */
|
|
144
|
+
original(file, entry) {
|
|
145
|
+
if (!this.offloaded.has(entry.id))
|
|
146
|
+
return entry;
|
|
147
|
+
const copy = this.read(file, entry);
|
|
148
|
+
if (copy === undefined)
|
|
149
|
+
return entry;
|
|
150
|
+
const content = contentOf(copy) ?? [];
|
|
151
|
+
content.forEach((block, i) => {
|
|
152
|
+
if (isImage(block))
|
|
153
|
+
content[i] = internImage(block);
|
|
154
|
+
});
|
|
155
|
+
return copy;
|
|
156
|
+
}
|
|
157
|
+
/** 按活动分支对齐:分支上被生效编辑改写的含图条目卸载,其余已卸载的回读。 */
|
|
158
|
+
sync(file, branch, index) {
|
|
159
|
+
const edits = collectContextEdits(branch);
|
|
160
|
+
const onBranch = new Set();
|
|
161
|
+
for (const entry of branch)
|
|
162
|
+
if (edits.has(entry.id))
|
|
163
|
+
onBranch.add(entry.id);
|
|
164
|
+
for (const id of [...this.offloaded]) {
|
|
165
|
+
const entry = index.get(id);
|
|
166
|
+
if (entry !== undefined && !onBranch.has(id))
|
|
167
|
+
this.hydrate(file, entry);
|
|
168
|
+
}
|
|
169
|
+
for (const id of onBranch) {
|
|
170
|
+
const entry = index.get(id);
|
|
171
|
+
if (entry !== undefined)
|
|
172
|
+
this.offload(entry);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/** 从文件读回条目(新对象);不成功时告警、忘掉行位置并返回 undefined。 */
|
|
176
|
+
read(file, entry) {
|
|
177
|
+
const locator = this.locators.get(entry.id);
|
|
178
|
+
try {
|
|
179
|
+
if (locator === undefined)
|
|
180
|
+
throw new Error("no line locator");
|
|
181
|
+
const line = readLineAt(file, locator);
|
|
182
|
+
const content = line.id === entry.id ? contentOf(line) : undefined;
|
|
183
|
+
if (content === undefined || content.length !== contentOf(entry)?.length) {
|
|
184
|
+
throw new Error("line does not match the entry");
|
|
185
|
+
}
|
|
186
|
+
return line;
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
this.locators.delete(entry.id);
|
|
190
|
+
this.offloaded.delete(entry.id);
|
|
191
|
+
this.warn(`cannot read session entry ${entry.id} back from ${file}: ${String(error)}`);
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
package/dist/session/store.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* - 锁:`<file>.lock`(O_EXCL 创建,内容为 pid);持锁进程已不在 → 视为陈旧锁并接管。
|
|
8
8
|
* - prune:移到 `<sessionsRoot>/.trash/`,文件名前缀删除时间;`purgeTrash` 清理超过 7 天的。
|
|
9
9
|
*/
|
|
10
|
+
import type { LineLocator } from "./offload.js";
|
|
10
11
|
import type { SessionLine } from "./types.js";
|
|
11
12
|
export declare const SESSION_FILE_SUFFIX = ".jsonl";
|
|
12
13
|
export declare const TRASH_DIR_NAME = ".trash";
|
|
@@ -22,6 +23,8 @@ export interface ReadResult {
|
|
|
22
23
|
lines: SessionLine[];
|
|
23
24
|
/** 末尾半行被截掉时为 true。 */
|
|
24
25
|
repairedTail: boolean;
|
|
26
|
+
/** 给了 `onLine` 时:读完(含修复)后的文件字节数。 */
|
|
27
|
+
bytes?: number;
|
|
25
28
|
}
|
|
26
29
|
/** 空白行(同旧口径 `trim() === ""`):ASCII 空白直接判定,遇到非 ASCII 字节才解码再 trim。 */
|
|
27
30
|
export declare function isBlankLine(line: Buffer): boolean;
|
|
@@ -31,11 +34,16 @@ export declare function isBlankLine(line: Buffer): boolean;
|
|
|
31
34
|
*/
|
|
32
35
|
export declare function readSessionLines(file: string, options?: {
|
|
33
36
|
repair?: boolean;
|
|
37
|
+
/** 每解析出一行就回调(行位置不含行尾;#170 图片卸载在这里就地剥掉图片)。 */
|
|
38
|
+
onLine?: (line: SessionLine, locator: LineLocator) => void;
|
|
34
39
|
}): ReadResult;
|
|
35
|
-
/** 追加若干行(一次 write,O_APPEND
|
|
36
|
-
export declare function appendLines(file: string, lines: readonly SessionLine[]):
|
|
37
|
-
/**
|
|
38
|
-
|
|
40
|
+
/** 追加若干行(一次 write,O_APPEND),返回写入的字节数。 */
|
|
41
|
+
export declare function appendLines(file: string, lines: readonly SessionLine[]): number;
|
|
42
|
+
/**
|
|
43
|
+
* 新建文件:先写临时文件再 rename,避免出现只有半个头的文件。返回每行的字节数(不含行尾,与
|
|
44
|
+
* `lines` 同序)。
|
|
45
|
+
*/
|
|
46
|
+
export declare function writeNewSessionFile(file: string, lines: readonly SessionLine[]): number[];
|
|
39
47
|
export interface SessionLock {
|
|
40
48
|
readonly path: string;
|
|
41
49
|
release(): void;
|
package/dist/session/store.js
CHANGED
|
@@ -57,8 +57,9 @@ export function readSessionLines(file, options = {}) {
|
|
|
57
57
|
unterminated = true;
|
|
58
58
|
if (isBlankLine(buf))
|
|
59
59
|
return;
|
|
60
|
+
let line;
|
|
60
61
|
try {
|
|
61
|
-
|
|
62
|
+
line = JSON.parse(buf.toString("utf8"));
|
|
62
63
|
}
|
|
63
64
|
catch (error) {
|
|
64
65
|
if (last) {
|
|
@@ -71,6 +72,8 @@ export function readSessionLines(file, options = {}) {
|
|
|
71
72
|
cause: error,
|
|
72
73
|
});
|
|
73
74
|
}
|
|
75
|
+
options.onLine?.(line, { offset, length: buf.length });
|
|
76
|
+
lines.push(line);
|
|
74
77
|
});
|
|
75
78
|
}
|
|
76
79
|
catch (error) {
|
|
@@ -82,32 +85,39 @@ export function readSessionLines(file, options = {}) {
|
|
|
82
85
|
// 最后一行完整但缺 LF:补上,保证后续追加另起一行。
|
|
83
86
|
appendRaw(file, "\n");
|
|
84
87
|
}
|
|
85
|
-
|
|
88
|
+
if (options.onLine === undefined)
|
|
89
|
+
return { lines, repairedTail };
|
|
90
|
+
return { lines, repairedTail, bytes: statSync(file).size };
|
|
86
91
|
}
|
|
87
92
|
function appendRaw(file, text) {
|
|
88
93
|
const fd = openSync(file, "a");
|
|
89
94
|
try {
|
|
90
|
-
writeSync(fd, text);
|
|
95
|
+
return writeSync(fd, text);
|
|
91
96
|
}
|
|
92
97
|
finally {
|
|
93
98
|
closeSync(fd);
|
|
94
99
|
}
|
|
95
100
|
}
|
|
96
|
-
/** 追加若干行(一次 write,O_APPEND
|
|
101
|
+
/** 追加若干行(一次 write,O_APPEND),返回写入的字节数。 */
|
|
97
102
|
export function appendLines(file, lines) {
|
|
98
103
|
if (lines.length === 0)
|
|
99
|
-
return;
|
|
100
|
-
appendRaw(file, lines.map((line) => JSON.stringify(line)).join("\n") + "\n");
|
|
104
|
+
return 0;
|
|
105
|
+
return appendRaw(file, lines.map((line) => JSON.stringify(line)).join("\n") + "\n");
|
|
101
106
|
}
|
|
102
|
-
/**
|
|
107
|
+
/**
|
|
108
|
+
* 新建文件:先写临时文件再 rename,避免出现只有半个头的文件。返回每行的字节数(不含行尾,与
|
|
109
|
+
* `lines` 同序)。
|
|
110
|
+
*/
|
|
103
111
|
export function writeNewSessionFile(file, lines) {
|
|
104
112
|
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
|
105
113
|
if (existsSync(file)) {
|
|
106
114
|
throw new AmaError("session_exists", `session file already exists: ${file}`);
|
|
107
115
|
}
|
|
108
116
|
const tmp = `${file}.${process.pid}.tmp`;
|
|
109
|
-
|
|
117
|
+
const texts = lines.map((line) => JSON.stringify(line));
|
|
118
|
+
writeFileSync(tmp, texts.join("\n") + "\n", { mode: 0o600 });
|
|
110
119
|
renameSync(tmp, file);
|
|
120
|
+
return texts.map((text) => Buffer.byteLength(text));
|
|
111
121
|
}
|
|
112
122
|
// ---------------------------------------------------------------------------
|
|
113
123
|
// 锁
|
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
* - 按行解码:终止符都是 ASCII,不会落在多字节序列中间,无效字节的替换结果与整段解码相同;
|
|
11
11
|
* - 窗口内单行超过 `maxBytes + LINE_SLACK` 字节时只留开头(这一行必然触发字节截断,
|
|
12
12
|
* `truncateHead` 至多取它开头 `maxBytes` 字节),超长行不会整行进内存。
|
|
13
|
+
*
|
|
14
|
+
* 解析状态机是 `LineScanner`,同步版 `readLineWindow` 与异步版 `readLineWindowAsync` 共用(#171):
|
|
15
|
+
* 异步版每块一次 `await`,大文件扫描期间不独占事件循环;峰值内存相同(仍一块缓冲)。
|
|
13
16
|
*/
|
|
14
17
|
/** 超过这个大小的文本文件走字节窗口;不超过的保留整读路径。 */
|
|
15
18
|
export declare const STREAM_READ_THRESHOLD: number;
|
|
@@ -25,8 +28,44 @@ export interface LineWindow {
|
|
|
25
28
|
}
|
|
26
29
|
/** 文件开头至多 `bytes` 字节(一次 `readSync`,用于二进制嗅探)。 */
|
|
27
30
|
export declare function readHead(abs: string, bytes: number): Buffer;
|
|
31
|
+
/** `readHead` 的异步版(#171)。 */
|
|
32
|
+
export declare function readHeadAsync(abs: string, bytes: number): Promise<Buffer>;
|
|
33
|
+
/**
|
|
34
|
+
* 按块喂入文件字节的行扫描状态机:第 `first` 行(0 起)到第 `stop` 行(不含)之间的行被收集,
|
|
35
|
+
* 收集的行累计 `字节 + 1` 超过 `maxBytes` 后停止收集(含越界的那一行),其余行只计数。
|
|
36
|
+
*/
|
|
37
|
+
export declare class LineScanner {
|
|
38
|
+
private readonly first;
|
|
39
|
+
private readonly stop;
|
|
40
|
+
private readonly maxBytes;
|
|
41
|
+
private readonly lineCap;
|
|
42
|
+
private readonly lines;
|
|
43
|
+
private truncatedBy;
|
|
44
|
+
private collected;
|
|
45
|
+
private index;
|
|
46
|
+
private parts;
|
|
47
|
+
private partBytes;
|
|
48
|
+
private pending;
|
|
49
|
+
private skipLF;
|
|
50
|
+
private bom;
|
|
51
|
+
private firstChunk;
|
|
52
|
+
constructor(first: number, stop: number, maxBytes: number);
|
|
53
|
+
/**
|
|
54
|
+
* 处理 `buf[0, end)`;`eof` 表示这是最后一块。返回调用方下次读取前应保留在 `buf` 开头的字节数
|
|
55
|
+
* (只在首块不足 3 字节、还判断不了 BOM 时非零)。
|
|
56
|
+
*/
|
|
57
|
+
feed(buf: Buffer, end: number, eof: boolean): number;
|
|
58
|
+
/** 文件读完:收尾最后一行并给出结果。 */
|
|
59
|
+
finish(): LineWindow;
|
|
60
|
+
private collecting;
|
|
61
|
+
private take;
|
|
62
|
+
private endLine;
|
|
63
|
+
}
|
|
28
64
|
/**
|
|
29
65
|
* 读取 `abs` 第 `offset` 行(1 起)开始的至多 `limit` 行(`undefined` 不限,`0` 只计数);
|
|
30
66
|
* 收集的行累计 `字节 + 1` 超过 `maxBytes` 后停止收集(含越界的那一行)。`chunkBytes` 只供测试。
|
|
67
|
+
* 同步版,保留给单测与口径比对;`read` 工具用 `readLineWindowAsync`。
|
|
31
68
|
*/
|
|
32
69
|
export declare function readLineWindow(abs: string, offset: number, limit: number | undefined, maxBytes: number, chunkBytes?: number): LineWindow;
|
|
70
|
+
/** `readLineWindow` 的异步版(#171):每块一次 `await`,结果与同步版相同。 */
|
|
71
|
+
export declare function readLineWindowAsync(abs: string, offset: number, limit: number | undefined, maxBytes: number, chunkBytes?: number): Promise<LineWindow>;
|
package/dist/tools/read-lines.js
CHANGED
|
@@ -10,8 +10,12 @@
|
|
|
10
10
|
* - 按行解码:终止符都是 ASCII,不会落在多字节序列中间,无效字节的替换结果与整段解码相同;
|
|
11
11
|
* - 窗口内单行超过 `maxBytes + LINE_SLACK` 字节时只留开头(这一行必然触发字节截断,
|
|
12
12
|
* `truncateHead` 至多取它开头 `maxBytes` 字节),超长行不会整行进内存。
|
|
13
|
+
*
|
|
14
|
+
* 解析状态机是 `LineScanner`,同步版 `readLineWindow` 与异步版 `readLineWindowAsync` 共用(#171):
|
|
15
|
+
* 异步版每块一次 `await`,大文件扫描期间不独占事件循环;峰值内存相同(仍一块缓冲)。
|
|
13
16
|
*/
|
|
14
17
|
import { closeSync, openSync, readSync } from "node:fs";
|
|
18
|
+
import { open } from "node:fs/promises";
|
|
15
19
|
/** 超过这个大小的文本文件走字节窗口;不超过的保留整读路径。 */
|
|
16
20
|
export const STREAM_READ_THRESHOLD = 1024 * 1024;
|
|
17
21
|
const CHUNK_BYTES = 64 * 1024;
|
|
@@ -37,111 +41,184 @@ export function readHead(abs, bytes) {
|
|
|
37
41
|
closeSync(fd);
|
|
38
42
|
}
|
|
39
43
|
}
|
|
44
|
+
/** `readHead` 的异步版(#171)。 */
|
|
45
|
+
export async function readHeadAsync(abs, bytes) {
|
|
46
|
+
const handle = await open(abs, "r");
|
|
47
|
+
try {
|
|
48
|
+
const buf = Buffer.alloc(bytes);
|
|
49
|
+
let got = 0;
|
|
50
|
+
while (got < bytes) {
|
|
51
|
+
const { bytesRead } = await handle.read(buf, got, bytes - got, null);
|
|
52
|
+
if (bytesRead === 0)
|
|
53
|
+
break;
|
|
54
|
+
got += bytesRead;
|
|
55
|
+
}
|
|
56
|
+
return buf.subarray(0, got);
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
await handle.close();
|
|
60
|
+
}
|
|
61
|
+
}
|
|
40
62
|
/**
|
|
41
|
-
*
|
|
42
|
-
* 收集的行累计 `字节 + 1` 超过 `maxBytes`
|
|
63
|
+
* 按块喂入文件字节的行扫描状态机:第 `first` 行(0 起)到第 `stop` 行(不含)之间的行被收集,
|
|
64
|
+
* 收集的行累计 `字节 + 1` 超过 `maxBytes` 后停止收集(含越界的那一行),其余行只计数。
|
|
43
65
|
*/
|
|
44
|
-
export
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
66
|
+
export class LineScanner {
|
|
67
|
+
first;
|
|
68
|
+
stop;
|
|
69
|
+
maxBytes;
|
|
70
|
+
lineCap;
|
|
71
|
+
lines = [];
|
|
72
|
+
truncatedBy;
|
|
73
|
+
collected = 0;
|
|
74
|
+
index = 0; // 已完成的行数
|
|
75
|
+
parts = [];
|
|
76
|
+
partBytes = 0;
|
|
77
|
+
pending = false; // 最后一个终止符之后已有字节
|
|
78
|
+
skipLF = false; // 上一块以 `\r` 结尾:下一块开头的 `\n` 属于同一终止符
|
|
79
|
+
bom = false;
|
|
80
|
+
firstChunk = true;
|
|
81
|
+
constructor(first, stop, maxBytes) {
|
|
82
|
+
this.first = first;
|
|
83
|
+
this.stop = stop;
|
|
84
|
+
this.maxBytes = maxBytes;
|
|
85
|
+
this.lineCap = maxBytes + LINE_SLACK;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* 处理 `buf[0, end)`;`eof` 表示这是最后一块。返回调用方下次读取前应保留在 `buf` 开头的字节数
|
|
89
|
+
* (只在首块不足 3 字节、还判断不了 BOM 时非零)。
|
|
90
|
+
*/
|
|
91
|
+
feed(buf, end, eof) {
|
|
92
|
+
let pos = 0;
|
|
93
|
+
if (this.firstChunk) {
|
|
94
|
+
if (end < 3 && !eof)
|
|
95
|
+
return end;
|
|
96
|
+
this.firstChunk = false;
|
|
97
|
+
if (end >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf) {
|
|
98
|
+
this.bom = true;
|
|
99
|
+
pos = 3;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (this.skipLF && pos < end) {
|
|
103
|
+
if (buf[pos] === LF)
|
|
104
|
+
pos++;
|
|
105
|
+
this.skipLF = false;
|
|
106
|
+
}
|
|
107
|
+
let nextLF = buf.indexOf(LF, pos);
|
|
108
|
+
let nextCR = buf.indexOf(CR, pos);
|
|
109
|
+
while (pos < end) {
|
|
110
|
+
if (nextLF !== -1 && nextLF < pos)
|
|
111
|
+
nextLF = buf.indexOf(LF, pos);
|
|
112
|
+
if (nextCR !== -1 && nextCR < pos)
|
|
113
|
+
nextCR = buf.indexOf(CR, pos);
|
|
114
|
+
const lf = nextLF === -1 || nextLF >= end ? end : nextLF;
|
|
115
|
+
const cr = nextCR === -1 || nextCR >= end ? end : nextCR;
|
|
116
|
+
const at = Math.min(lf, cr);
|
|
117
|
+
if (at > pos) {
|
|
118
|
+
this.pending = true;
|
|
119
|
+
if (this.collecting())
|
|
120
|
+
this.take(buf, pos, at);
|
|
121
|
+
}
|
|
122
|
+
if (at === end)
|
|
123
|
+
break;
|
|
124
|
+
this.endLine();
|
|
125
|
+
pos = at + 1;
|
|
126
|
+
if (at === cr) {
|
|
127
|
+
if (pos < end) {
|
|
128
|
+
if (buf[pos] === LF)
|
|
129
|
+
pos++;
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
this.skipLF = true;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
/** 文件读完:收尾最后一行并给出结果。 */
|
|
139
|
+
finish() {
|
|
140
|
+
if (this.pending)
|
|
141
|
+
this.endLine();
|
|
142
|
+
const { lines, truncatedBy, bom } = this;
|
|
143
|
+
return { lines, totalLines: this.index, ...(truncatedBy ? { truncatedBy } : {}), bom };
|
|
144
|
+
}
|
|
145
|
+
collecting() {
|
|
146
|
+
return this.truncatedBy === undefined && this.index >= this.first && this.index < this.stop;
|
|
147
|
+
}
|
|
148
|
+
take(buf, from, to) {
|
|
149
|
+
if (to <= from || this.partBytes >= this.lineCap)
|
|
60
150
|
return;
|
|
61
|
-
const end = Math.min(to, from + lineCap - partBytes);
|
|
62
|
-
parts.push(Buffer.from(buf.subarray(from, end)));
|
|
63
|
-
partBytes += end - from;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
if (collecting()) {
|
|
151
|
+
const end = Math.min(to, from + this.lineCap - this.partBytes);
|
|
152
|
+
this.parts.push(Buffer.from(buf.subarray(from, end)));
|
|
153
|
+
this.partBytes += end - from;
|
|
154
|
+
}
|
|
155
|
+
endLine() {
|
|
156
|
+
if (this.collecting()) {
|
|
157
|
+
const { parts } = this;
|
|
67
158
|
const line = (parts.length === 1 ? parts[0] : Buffer.concat(parts)).toString("utf8");
|
|
68
|
-
lines.push(line);
|
|
69
|
-
collected += Buffer.byteLength(line) + 1;
|
|
70
|
-
if (collected > maxBytes)
|
|
71
|
-
truncatedBy = "bytes";
|
|
159
|
+
this.lines.push(line);
|
|
160
|
+
this.collected += Buffer.byteLength(line) + 1;
|
|
161
|
+
if (this.collected > this.maxBytes)
|
|
162
|
+
this.truncatedBy = "bytes";
|
|
72
163
|
}
|
|
73
|
-
if (partBytes > 0)
|
|
74
|
-
parts = []; // 窗口外的行不分配:40 万行时每行一个空数组就是十几 MB 临时垃圾
|
|
75
|
-
partBytes = 0;
|
|
76
|
-
pending = false;
|
|
77
|
-
index++;
|
|
78
|
-
}
|
|
164
|
+
if (this.partBytes > 0)
|
|
165
|
+
this.parts = []; // 窗口外的行不分配:40 万行时每行一个空数组就是十几 MB 临时垃圾
|
|
166
|
+
this.partBytes = 0;
|
|
167
|
+
this.pending = false;
|
|
168
|
+
this.index++;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
function scanner(offset, limit, maxBytes) {
|
|
172
|
+
const first = offset - 1;
|
|
173
|
+
return new LineScanner(first, limit === undefined ? Infinity : first + limit, maxBytes);
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* 读取 `abs` 第 `offset` 行(1 起)开始的至多 `limit` 行(`undefined` 不限,`0` 只计数);
|
|
177
|
+
* 收集的行累计 `字节 + 1` 超过 `maxBytes` 后停止收集(含越界的那一行)。`chunkBytes` 只供测试。
|
|
178
|
+
* 同步版,保留给单测与口径比对;`read` 工具用 `readLineWindowAsync`。
|
|
179
|
+
*/
|
|
180
|
+
export function readLineWindow(abs, offset, limit, maxBytes, chunkBytes = CHUNK_BYTES) {
|
|
181
|
+
const scan = scanner(offset, limit, maxBytes);
|
|
79
182
|
const size = Math.max(3, chunkBytes);
|
|
80
183
|
const fd = openSync(abs, "r");
|
|
81
184
|
try {
|
|
82
185
|
const buf = Buffer.alloc(size);
|
|
83
186
|
let carry = 0; // 首块凑满 3 字节判断 BOM
|
|
84
|
-
let firstChunk = true;
|
|
85
187
|
for (;;) {
|
|
86
188
|
const n = readSync(fd, buf, carry, size - carry, null);
|
|
87
189
|
const end = carry + n;
|
|
88
|
-
|
|
89
|
-
if (n === 0 && end === 0)
|
|
190
|
+
if (end === 0)
|
|
90
191
|
break;
|
|
91
|
-
|
|
92
|
-
if (firstChunk) {
|
|
93
|
-
if (end < 3 && n > 0) {
|
|
94
|
-
carry = end;
|
|
95
|
-
continue;
|
|
96
|
-
}
|
|
97
|
-
firstChunk = false;
|
|
98
|
-
if (end >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf) {
|
|
99
|
-
bom = true;
|
|
100
|
-
pos = 3;
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
if (skipLF && pos < end) {
|
|
104
|
-
if (buf[pos] === LF)
|
|
105
|
-
pos++;
|
|
106
|
-
skipLF = false;
|
|
107
|
-
}
|
|
108
|
-
let nextLF = buf.indexOf(LF, pos);
|
|
109
|
-
let nextCR = buf.indexOf(CR, pos);
|
|
110
|
-
while (pos < end) {
|
|
111
|
-
if (nextLF !== -1 && nextLF < pos)
|
|
112
|
-
nextLF = buf.indexOf(LF, pos);
|
|
113
|
-
if (nextCR !== -1 && nextCR < pos)
|
|
114
|
-
nextCR = buf.indexOf(CR, pos);
|
|
115
|
-
const lf = nextLF === -1 || nextLF >= end ? end : nextLF;
|
|
116
|
-
const cr = nextCR === -1 || nextCR >= end ? end : nextCR;
|
|
117
|
-
const at = Math.min(lf, cr);
|
|
118
|
-
if (at > pos) {
|
|
119
|
-
pending = true;
|
|
120
|
-
if (collecting())
|
|
121
|
-
take(buf, pos, at);
|
|
122
|
-
}
|
|
123
|
-
if (at === end)
|
|
124
|
-
break;
|
|
125
|
-
finish();
|
|
126
|
-
pos = at + 1;
|
|
127
|
-
if (at === cr) {
|
|
128
|
-
if (pos < end) {
|
|
129
|
-
if (buf[pos] === LF)
|
|
130
|
-
pos++;
|
|
131
|
-
}
|
|
132
|
-
else {
|
|
133
|
-
skipLF = true;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
}
|
|
192
|
+
carry = scan.feed(buf, end, n === 0);
|
|
137
193
|
if (n === 0)
|
|
138
194
|
break;
|
|
139
195
|
}
|
|
140
|
-
if (pending)
|
|
141
|
-
finish();
|
|
142
196
|
}
|
|
143
197
|
finally {
|
|
144
198
|
closeSync(fd);
|
|
145
199
|
}
|
|
146
|
-
return
|
|
200
|
+
return scan.finish();
|
|
201
|
+
}
|
|
202
|
+
/** `readLineWindow` 的异步版(#171):每块一次 `await`,结果与同步版相同。 */
|
|
203
|
+
export async function readLineWindowAsync(abs, offset, limit, maxBytes, chunkBytes = CHUNK_BYTES) {
|
|
204
|
+
const scan = scanner(offset, limit, maxBytes);
|
|
205
|
+
const size = Math.max(3, chunkBytes);
|
|
206
|
+
const handle = await open(abs, "r");
|
|
207
|
+
try {
|
|
208
|
+
const buf = Buffer.alloc(size);
|
|
209
|
+
let carry = 0;
|
|
210
|
+
for (;;) {
|
|
211
|
+
const { bytesRead: n } = await handle.read(buf, carry, size - carry, null);
|
|
212
|
+
const end = carry + n;
|
|
213
|
+
if (end === 0)
|
|
214
|
+
break;
|
|
215
|
+
carry = scan.feed(buf, end, n === 0);
|
|
216
|
+
if (n === 0)
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
finally {
|
|
221
|
+
await handle.close();
|
|
222
|
+
}
|
|
223
|
+
return scan.finish();
|
|
147
224
|
}
|
package/dist/tools/read.js
CHANGED
|
@@ -13,7 +13,7 @@ import { internImage } from "../ai/image-intern.js";
|
|
|
13
13
|
import { displayPath, resolvePath } from "./paths.js";
|
|
14
14
|
import { DEFAULT_MAX_LINES, formatSize, toolOutputBytes, truncateHead } from "./truncate.js";
|
|
15
15
|
import { normalizeToLF, splitBom } from "./edit-fuzzy.js";
|
|
16
|
-
import { STREAM_READ_THRESHOLD,
|
|
16
|
+
import { STREAM_READ_THRESHOLD, readHeadAsync, readLineWindowAsync } from "./read-lines.js";
|
|
17
17
|
import { MAX_IMAGE_FILE_BYTES, fitImage, imageMimeFromPath, imageSize, sniffImageMime, } from "./image-file.js";
|
|
18
18
|
export { imageSize } from "./image-file.js";
|
|
19
19
|
const SNIFF_BYTES = 8000;
|
|
@@ -99,10 +99,10 @@ export async function executeRead(input, ctx, options = {}) {
|
|
|
99
99
|
let selected;
|
|
100
100
|
if (info.size > (options.streamThreshold ?? STREAM_READ_THRESHOLD)) {
|
|
101
101
|
// [M-E] 大文件按字节窗口读:只解码要显示的行,结果与整读逐字节相同(D1)
|
|
102
|
-
if (isBinary(
|
|
102
|
+
if (isBinary(await readHeadAsync(abs, SNIFF_BYTES)))
|
|
103
103
|
return error(binaryMessage(shown));
|
|
104
104
|
const want = Math.min(input.limit ?? Infinity, DEFAULT_MAX_LINES + 1);
|
|
105
|
-
const win =
|
|
105
|
+
const win = await readLineWindowAsync(abs, badOffset ? 1 : offset, badOffset || badLimit ? 0 : want, maxBytes);
|
|
106
106
|
totalLines = win.totalLines;
|
|
107
107
|
selected = win.lines;
|
|
108
108
|
}
|