@piagent/platform 1.43.4 → 1.43.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/docs/command-reference-vietnamese.md +2 -2
- package/docs/distribution-standard.md +6 -6
- package/docs/operator-manual-vietnamese.md +7 -7
- package/docs/publishing-for-teams.md +2 -0
- package/docs/release-install-policy.md +2 -2
- package/docs/team-onboarding.md +2 -2
- package/docs-site/architecture.html +1 -1
- package/docs-site/assets/search-en.json +1 -1
- package/docs-site/assets/search-vi.json +1 -1
- package/docs-site/benchmark.html +1 -1
- package/docs-site/commands.html +1 -1
- package/docs-site/context-engine.html +1 -1
- package/docs-site/ecosystem.html +1 -1
- package/docs-site/en/architecture.html +1 -1
- package/docs-site/en/benchmark.html +1 -1
- package/docs-site/en/commands.html +1 -1
- package/docs-site/en/context-engine.html +1 -1
- package/docs-site/en/ecosystem.html +1 -1
- package/docs-site/en/index.html +15 -15
- package/docs-site/en/mcp-approval.html +1 -1
- package/docs-site/en/mcp-auth.html +1 -1
- package/docs-site/en/mcp-servers.html +1 -1
- package/docs-site/en/mcp.html +1 -1
- package/docs-site/en/permissions.html +1 -1
- package/docs-site/en/profiles.html +1 -1
- package/docs-site/en/quickstart.html +1 -1
- package/docs-site/en/runtime.html +1 -1
- package/docs-site/en/scope.html +1 -1
- package/docs-site/en/security.html +1 -1
- package/docs-site/en/team.html +1 -1
- package/docs-site/en/update.html +3 -3
- package/docs-site/en/whats-new.html +52 -31
- package/docs-site/en/windows.html +1 -1
- package/docs-site/en/workflows.html +1 -1
- package/docs-site/index.html +15 -15
- package/docs-site/mcp-approval.html +1 -1
- package/docs-site/mcp-auth.html +1 -1
- package/docs-site/mcp-servers.html +1 -1
- package/docs-site/mcp.html +1 -1
- package/docs-site/permissions.html +1 -1
- package/docs-site/profiles.html +1 -1
- package/docs-site/quickstart.html +1 -1
- package/docs-site/releases.json +62 -0
- package/docs-site/runtime.html +1 -1
- package/docs-site/scope.html +1 -1
- package/docs-site/security.html +1 -1
- package/docs-site/team.html +1 -1
- package/docs-site/update.html +3 -3
- package/docs-site/whats-new.html +52 -31
- package/docs-site/windows.html +1 -1
- package/docs-site/workflows.html +1 -1
- package/evals/private-holdout-v1/public-exposure.v2.json +1 -1
- package/package.json +1 -1
- package/packages/piagent-core/extensions/document-intake.ts +265 -19
- package/packages/piagent-core/managed/compaction.mjs +26 -0
- package/packages/piagent-core/managed/session.mjs +4 -4
- package/packages/piagent-core/package.json +1 -1
- package/packages/piagent-core/runtime/input/attachment-store.ts +10 -10
- package/packages/piagent-core/runtime/input/attachment-types.ts +18 -0
- package/packages/piagent-core/runtime/inspection/document-workspace-projection.ts +1 -1
- package/packages/piagent-webui/dist/client/assets/{ActivityPanel-DRAiHliI.js → ActivityPanel-CBbfhN-m.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/App-C633xHZS.js +5 -0
- package/packages/piagent-webui/dist/client/assets/GatewayApp-cQ6cd7fw.js +97 -0
- package/packages/piagent-webui/dist/client/assets/{InspectionApp-C00GhY47.js → InspectionApp-B5eKxLEs.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/{SessionAgentWorkspace-CBbWJmh7.js → SessionAgentWorkspace-N79svKS3.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/{SessionInspectorDrawer-CMgkZoNX.js → SessionInspectorDrawer-mZqwulvp.js} +2 -2
- package/packages/piagent-webui/dist/client/assets/{SettingsNavigation-CM0s3QUC.js → SettingsNavigation-DzboWgoR.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/SettingsPage-DX2QIvRC.js +1 -0
- package/packages/piagent-webui/dist/client/assets/{TerminalPanel-DNKKZsMX.js → TerminalPanel-CiDM7fY8.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/{WorkspaceCodeEditor-DUffKV2S.js → WorkspaceCodeEditor-B5KeO5jf.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/{WorkspaceExplorer-D5iHXMqE.js → WorkspaceExplorer-C5M5c2uW.js} +3 -3
- package/packages/piagent-webui/dist/client/assets/{api-BUjrtT1B.js → api-BQS4Dtrr.js} +1 -1
- package/packages/piagent-webui/dist/client/assets/{index-D6GtlEqC.js → index-CrpbZF1r.js} +2 -2
- package/packages/piagent-webui/dist/client/assets/transcript-view-model-BCyNr6KF.js +3 -0
- package/packages/piagent-webui/dist/client/index.html +1 -1
- package/packages/piagent-webui/extension/piagent-webui.ts +2 -0
- package/packages/piagent-webui/server/core-inspection-provider.ts +6 -1
- package/packages/piagent-webui/server/ipc-read-model-client.ts +1 -0
- package/packages/piagent-webui/server/read-model-provider.ts +1 -0
- package/packages/piagent-webui/server/read-only-router.ts +6 -0
- package/packages/piagent-webui/server/transcript-projection.ts +20 -4
- package/schemas/piagent-webui/attachment-v1.schema.json +8 -1
- package/schemas/piagent-webui/document-workspace-v1.schema.json +3 -3
- package/packages/piagent-webui/dist/client/assets/App-CarM9drO.js +0 -5
- package/packages/piagent-webui/dist/client/assets/GatewayApp-uqpLU6gs.js +0 -93
- package/packages/piagent-webui/dist/client/assets/SettingsPage-Cr6M2FVg.js +0 -1
- package/packages/piagent-webui/dist/client/assets/transcript-view-model-BNGQCkzY.js +0 -3
package/docs-site/whats-new.html
CHANGED
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
<span class="brand-mark"><img src="/assets/piagent-logo.svg" alt="" width="28" height="28" /></span>
|
|
38
38
|
<span class="brand-name"><b>Piagent</b> <span>Platform</span></span>
|
|
39
39
|
</a>
|
|
40
|
-
<a class="version-badge" href="/whats-new" data-docs-version="v1.43.
|
|
40
|
+
<a class="version-badge" href="/whats-new" data-docs-version="v1.43.6" title="v1.43.6 docs · VI"><span class="dot"></span>v1.43.6</a>
|
|
41
41
|
<nav class="top-nav" aria-label="Tài liệu">
|
|
42
42
|
<a href="/quickstart">Tài liệu</a>
|
|
43
43
|
<a href="/whats-new" aria-current="page">Changelog</a>
|
|
@@ -108,7 +108,31 @@
|
|
|
108
108
|
<div class="release-filter" role="toolbar" aria-label="Lọc theo loại thay đổi"><button type="button" data-filter-kind="all" aria-pressed="true">Tất cả</button><button type="button" data-filter-kind="new" aria-pressed="false">Mới</button><button type="button" data-filter-kind="improved" aria-pressed="false">Cải thiện</button><button type="button" data-filter-kind="fixed" aria-pressed="false">Sửa lỗi</button><button type="button" data-filter-kind="security" aria-pressed="false">Bảo mật</button></div>
|
|
109
109
|
<section class="section releases" data-releases>
|
|
110
110
|
<h2 id="latest" class="sr-only">Mới nhất</h2>
|
|
111
|
-
<article class="release latest" id="v1-43-
|
|
111
|
+
<article class="release latest" id="v1-43-6" data-release>
|
|
112
|
+
<div class="release-meta"><a class="release-version" href="#v1-43-6">v1.43.6</a><time class="release-date" datetime="2026-10-27">27/10/2026</time></div>
|
|
113
|
+
<div class="release-body">
|
|
114
|
+
<h3 data-toc-skip id="v1-43-6-title">Xem thay đổi của agent ngay khi đang chạy</h3>
|
|
115
|
+
<p>Thay đổi và Review cập nhật theo lượt đang chạy, thấy sai thì dừng sớm được, và lượt còn chạy không còn trông như đã xong.</p>
|
|
116
|
+
<ul class="release-items">
|
|
117
|
+
<li data-kind="improved"><span class="kind kind-improved">Cải thiện</span><span>Panel Workspace liệt kê file thay đổi và mở được Review khi agent đang chạy, tự làm mới vài giây một lần; Review cũng cập nhật theo.</span></li>
|
|
118
|
+
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Câu trả lời bị harness trả lại để chạy check được ghi là giữa chừng, và lượt vẫn hiện đang chạy kèm nút Dừng cho tới khi check xong.</span></li>
|
|
119
|
+
</ul>
|
|
120
|
+
</div>
|
|
121
|
+
</article>
|
|
122
|
+
<article class="release" id="v1-43-5" data-release>
|
|
123
|
+
<div class="release-meta"><a class="release-version" href="#v1-43-5">v1.43.5</a><time class="release-date" datetime="2026-10-27">27/10/2026</time></div>
|
|
124
|
+
<div class="release-body">
|
|
125
|
+
<h3 data-toc-skip id="v1-43-5-title">Xem trước ảnh và đính kèm file văn phòng</h3>
|
|
126
|
+
<p>Ảnh đã gửi hiện thành hình, bảng tính và slide đính kèm trực tiếp được, task chụp nhiều screenshot không còn dừng vì giới hạn dung lượng request của Studio.</p>
|
|
127
|
+
<ul class="release-items">
|
|
128
|
+
<li data-kind="new"><span class="kind kind-new">Mới</span><span>Ảnh đính kèm và ảnh đã gửi hiện thành hình trong hội thoại, bấm để xem cỡ lớn; chip trong khung soạn có ảnh thu nhỏ.</span></li>
|
|
129
|
+
<li data-kind="new"><span class="kind kind-new">Mới</span><span>Đính kèm được .xlsx, .xlsm, .pptx, .odt, .ods, .odp, .rtf: bảng tính thành các dòng cách nhau bằng tab, slide theo thứ tự kèm ghi chú, không cần cài converter.</span></li>
|
|
130
|
+
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Task công ty chụp nhiều screenshot không còn lỗi request_too_large: mỗi request chỉ gửi lại ảnh mới nhất trong 2 MiB, ảnh cũ hơn thành một dòng ghi chú.</span></li>
|
|
131
|
+
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Câu trả lời đang stream của lượt có dùng tool xuống đoạn theo từng message của model, không còn dính các câu thành một khối.</span></li>
|
|
132
|
+
</ul>
|
|
133
|
+
</div>
|
|
134
|
+
</article>
|
|
135
|
+
<article class="release" id="v1-43-4" data-release>
|
|
112
136
|
<div class="release-meta"><a class="release-version" href="#v1-43-4">v1.43.4</a><time class="release-date" datetime="2026-10-26">26/10/2026</time></div>
|
|
113
137
|
<div class="release-body">
|
|
114
138
|
<h3 data-toc-skip id="v1-43-4-title">Dev chọn workflow, harness hỗ trợ BE/FE theo project</h3>
|
|
@@ -400,40 +424,37 @@
|
|
|
400
424
|
<p class="release-requires"><strong>Cần:</strong> Agent Studio 0.34, Agent Watch 0.15.9 (Windows 0.1.15)</p>
|
|
401
425
|
</div>
|
|
402
426
|
</article>
|
|
403
|
-
<article class="release" id="v1-33-0" data-release>
|
|
404
|
-
<div class="release-meta"><a class="release-version" href="#v1-33-0">v1.33.0</a><time class="release-date" datetime="2026-10-20">20/10/2026</time></div>
|
|
405
|
-
<div class="release-body">
|
|
406
|
-
<h3 data-toc-skip id="v1-33-0-title">Session công ty và cá nhân chạy trên Pi 1.3.0</h3>
|
|
407
|
-
<p><code>piagent-setup</code> và <code>piagent-update</code> chuyển Piagent từ Pi 0.87.1 lên Pi 1.3.0, từ chối lời gọi tool hỏng và tự thử lại lỗi gateway.</p>
|
|
408
|
-
<ul class="release-items">
|
|
409
|
-
<li data-kind="breaking"><span class="kind kind-breaking">Thay đổi lớn</span><span>Lệnh <code>pi</code> cá nhân cũng lên 1.3.0: tham chiếu <code>$schema</code> của theme đã đổi, và URL gốc ghi đè cho <code>kimi-coding</code> phải kết thúc bằng <code>/v1</code>.</span></li>
|
|
410
|
-
<li data-kind="improved"><span class="kind kind-improved">Cải thiện</span><span>Lời gọi tool có tham số JSON hỏng bị từ chối và trả lại cho model thay vì chạy với phần đọc được trước chỗ lỗi.</span></li>
|
|
411
|
-
<li data-kind="improved"><span class="kind kind-improved">Cải thiện</span><span>Lỗi gateway được tự thử lại, session mở lại giữ đúng các tool đang bật, và đổi model sẽ bỏ thinking của model trước.</span></li>
|
|
412
|
-
<li data-kind="improved"><span class="kind kind-improved">Cải thiện</span><span>GPT-6.1 Sol trong session công ty vẫn giữ cửa sổ 272k đã duyệt, vì phần trên 272k bị tính giá gấp đôi.</span></li>
|
|
413
|
-
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Lượt chạy bắt đầu từ một tin nhắn tùy chỉnh nay nhận system prompt như thường và không tạo thêm lượt, task hay context pack.</span></li>
|
|
414
|
-
<li data-kind="security"><span class="kind kind-security">Bảo mật</span><span>Ba cảnh báo bảo mật dependency từng được chấp nhận đã được xử lý: Pi 1.3.0 cài bản đã sửa.</span></li>
|
|
415
|
-
</ul>
|
|
416
|
-
<p class="release-requires"><strong>Cần:</strong> Pi 1.3.0</p>
|
|
417
|
-
</div>
|
|
418
|
-
</article>
|
|
419
|
-
<article class="release" id="v1-32-1" data-release>
|
|
420
|
-
<div class="release-meta"><a class="release-version" href="#v1-32-1">v1.32.1</a><time class="release-date" datetime="2026-10-19">19/10/2026</time></div>
|
|
421
|
-
<div class="release-body">
|
|
422
|
-
<h3 data-toc-skip id="v1-32-1-title">Bypass được giữ sau khi tải lại, tab đang dùng không bị đăng xuất</h3>
|
|
423
|
-
<p>Bypass và Fast bật ngay sau câu trả lời được giữ, nút quyền truy cập theo đúng cuộc trò chuyện đang mở, và tab đang dùng không còn bị đăng xuất.</p>
|
|
424
|
-
<ul class="release-items">
|
|
425
|
-
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Bật Bypass hoặc Fast ngay sau câu trả lời từng bị từ chối, nên tải lại vẫn thấy "Hỏi trước"; giờ lệnh được gửi lại theo trạng thái mới nhất.</span></li>
|
|
426
|
-
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Mở cuộc trò chuyện khác cũng đang Bypass từng hiện "Hỏi trước"; nút quyền và Cài đặt → Quyền truy cập giờ theo cuộc trò chuyện đang mở.</span></li>
|
|
427
|
-
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Phiên trình duyệt từng hết hạn 8 giờ sau khi mở dù tab vẫn đang dùng; giờ được gia hạn tới 7 ngày, chỉ tab bỏ không 8 giờ mới bị đăng xuất.</span></li>
|
|
428
|
-
</ul>
|
|
429
|
-
</div>
|
|
430
|
-
</article>
|
|
431
427
|
</section>
|
|
432
428
|
<section class="section archive">
|
|
433
429
|
<h2 id="archive">Các bản trước</h2>
|
|
434
430
|
<p class="section-lead">Bản cũ hơn, gom theo tháng. Mở một tháng để xem từng thay đổi.</p>
|
|
435
431
|
<details class="release-archive">
|
|
436
|
-
<summary><span>Tháng 10/2026</span><span class="archive-count">
|
|
432
|
+
<summary><span>Tháng 10/2026</span><span class="archive-count">47</span></summary>
|
|
433
|
+
<div class="archive-row" id="v1-33-0">
|
|
434
|
+
<div class="archive-head"><a class="release-version" href="#v1-33-0">v1.33.0</a> <span class="release-date">20/10/2026</span></div>
|
|
435
|
+
<div class="archive-body">
|
|
436
|
+
<p><strong>Session công ty và cá nhân chạy trên Pi 1.3.0</strong> — <code>piagent-setup</code> và <code>piagent-update</code> chuyển Piagent từ Pi 0.87.1 lên Pi 1.3.0, từ chối lời gọi tool hỏng và tự thử lại lỗi gateway.</p>
|
|
437
|
+
<ul class="release-items">
|
|
438
|
+
<li data-kind="breaking"><span class="kind kind-breaking">Thay đổi lớn</span><span>Lệnh <code>pi</code> cá nhân cũng lên 1.3.0: tham chiếu <code>$schema</code> của theme đã đổi, và URL gốc ghi đè cho <code>kimi-coding</code> phải kết thúc bằng <code>/v1</code>.</span></li>
|
|
439
|
+
<li data-kind="improved"><span class="kind kind-improved">Cải thiện</span><span>Lời gọi tool có tham số JSON hỏng bị từ chối và trả lại cho model thay vì chạy với phần đọc được trước chỗ lỗi.</span></li>
|
|
440
|
+
<li data-kind="improved"><span class="kind kind-improved">Cải thiện</span><span>Lỗi gateway được tự thử lại, session mở lại giữ đúng các tool đang bật, và đổi model sẽ bỏ thinking của model trước.</span></li>
|
|
441
|
+
<li data-kind="improved"><span class="kind kind-improved">Cải thiện</span><span>GPT-6.1 Sol trong session công ty vẫn giữ cửa sổ 272k đã duyệt, vì phần trên 272k bị tính giá gấp đôi.</span></li>
|
|
442
|
+
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Lượt chạy bắt đầu từ một tin nhắn tùy chỉnh nay nhận system prompt như thường và không tạo thêm lượt, task hay context pack.</span></li>
|
|
443
|
+
<li data-kind="security"><span class="kind kind-security">Bảo mật</span><span>Ba cảnh báo bảo mật dependency từng được chấp nhận đã được xử lý: Pi 1.3.0 cài bản đã sửa.</span></li>
|
|
444
|
+
</ul>
|
|
445
|
+
</div>
|
|
446
|
+
</div>
|
|
447
|
+
<div class="archive-row" id="v1-32-1">
|
|
448
|
+
<div class="archive-head"><a class="release-version" href="#v1-32-1">v1.32.1</a> <span class="release-date">19/10/2026</span></div>
|
|
449
|
+
<div class="archive-body">
|
|
450
|
+
<p><strong>Bypass được giữ sau khi tải lại, tab đang dùng không bị đăng xuất</strong> — Bypass và Fast bật ngay sau câu trả lời được giữ, nút quyền truy cập theo đúng cuộc trò chuyện đang mở, và tab đang dùng không còn bị đăng xuất.</p>
|
|
451
|
+
<ul class="release-items">
|
|
452
|
+
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Bật Bypass hoặc Fast ngay sau câu trả lời từng bị từ chối, nên tải lại vẫn thấy "Hỏi trước"; giờ lệnh được gửi lại theo trạng thái mới nhất.</span></li>
|
|
453
|
+
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Mở cuộc trò chuyện khác cũng đang Bypass từng hiện "Hỏi trước"; nút quyền và Cài đặt → Quyền truy cập giờ theo cuộc trò chuyện đang mở.</span></li>
|
|
454
|
+
<li data-kind="fixed"><span class="kind kind-fixed">Sửa lỗi</span><span>Phiên trình duyệt từng hết hạn 8 giờ sau khi mở dù tab vẫn đang dùng; giờ được gia hạn tới 7 ngày, chỉ tab bỏ không 8 giờ mới bị đăng xuất.</span></li>
|
|
455
|
+
</ul>
|
|
456
|
+
</div>
|
|
457
|
+
</div>
|
|
437
458
|
<div class="archive-row" id="v1-32-0">
|
|
438
459
|
<div class="archive-head"><a class="release-version" href="#v1-32-0">v1.32.0</a> <span class="release-date">19/10/2026</span></div>
|
|
439
460
|
<div class="archive-body">
|
package/docs-site/windows.html
CHANGED
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
<span class="brand-mark"><img src="/assets/piagent-logo.svg" alt="" width="28" height="28" /></span>
|
|
38
38
|
<span class="brand-name"><b>Piagent</b> <span>Platform</span></span>
|
|
39
39
|
</a>
|
|
40
|
-
<a class="version-badge" href="/whats-new" data-docs-version="v1.43.
|
|
40
|
+
<a class="version-badge" href="/whats-new" data-docs-version="v1.43.6" title="v1.43.6 docs · VI"><span class="dot"></span>v1.43.6</a>
|
|
41
41
|
<nav class="top-nav" aria-label="Tài liệu">
|
|
42
42
|
<a href="/quickstart">Tài liệu</a>
|
|
43
43
|
<a href="/whats-new">Changelog</a>
|
package/docs-site/workflows.html
CHANGED
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
<span class="brand-mark"><img src="/assets/piagent-logo.svg" alt="" width="28" height="28" /></span>
|
|
38
38
|
<span class="brand-name"><b>Piagent</b> <span>Platform</span></span>
|
|
39
39
|
</a>
|
|
40
|
-
<a class="version-badge" href="/whats-new" data-docs-version="v1.43.
|
|
40
|
+
<a class="version-badge" href="/whats-new" data-docs-version="v1.43.6" title="v1.43.6 docs · VI"><span class="dot"></span>v1.43.6</a>
|
|
41
41
|
<nav class="top-nav" aria-label="Tài liệu">
|
|
42
42
|
<a href="/quickstart">Tài liệu</a>
|
|
43
43
|
<a href="/whats-new">Changelog</a>
|
|
@@ -257,7 +257,7 @@
|
|
|
257
257
|
{
|
|
258
258
|
"path": "tests",
|
|
259
259
|
"fileCount": 626,
|
|
260
|
-
"sha256": "
|
|
260
|
+
"sha256": "9cae03de80f76faf46814a6b2f30e649e35d7629d8b453954d28a472f27e9b4e"
|
|
261
261
|
}
|
|
262
262
|
],
|
|
263
263
|
"custodianComparisonRules": {
|
package/package.json
CHANGED
|
@@ -24,8 +24,12 @@ import zlib from "node:zlib";
|
|
|
24
24
|
// Everything here is read-only by construction. There is no write path.
|
|
25
25
|
|
|
26
26
|
export const DOCUMENT_EXTENSIONS = Object.freeze([
|
|
27
|
-
"md", "markdown", "txt", "text", "csv", "tsv", "json", "yaml", "yml", "pdf", "docx"
|
|
27
|
+
"md", "markdown", "txt", "text", "csv", "tsv", "json", "yaml", "yml", "pdf", "docx",
|
|
28
|
+
"xlsx", "xlsm", "pptx", "odt", "ods", "odp", "rtf"
|
|
28
29
|
]);
|
|
30
|
+
// Office formats read without a converter: the OOXML and OpenDocument ones are
|
|
31
|
+
// ZIP archives of XML, and RTF is text with control words.
|
|
32
|
+
export const OFFICE_EXTENSIONS = Object.freeze(["xlsx", "xlsm", "pptx", "odt", "ods", "odp", "rtf"]);
|
|
29
33
|
|
|
30
34
|
// Large enough for a real specification, small enough that one file cannot
|
|
31
35
|
// swallow a context window before the budget check ever sees it.
|
|
@@ -48,7 +52,7 @@ export type DocumentResolution =
|
|
|
48
52
|
| { status: "error"; reason: string };
|
|
49
53
|
|
|
50
54
|
export type DocumentExtraction =
|
|
51
|
-
| { status: "ok"; text: string; truncated: boolean; kind: "text" | "docx" | "pdf" }
|
|
55
|
+
| { status: "ok"; text: string; truncated: boolean; kind: "text" | "docx" | "pdf" | "office" }
|
|
52
56
|
| { status: "error"; reason: string };
|
|
53
57
|
|
|
54
58
|
export type DocumentCommandResult = { status: number | null; stdout: string; stderr: string; error?: Error };
|
|
@@ -246,23 +250,42 @@ type ZipEntryResult =
|
|
|
246
250
|
// Every refusal names what was actually wrong. "Not a readable .docx" for a
|
|
247
251
|
// 4097-entry archive, a corrupt entry, and a file that is not a ZIP at all sends
|
|
248
252
|
// the operator looking in three different wrong places.
|
|
249
|
-
function malformed(detail: string): ZipEntryResult {
|
|
250
|
-
return { status: "error", reason: `not a readable
|
|
253
|
+
function malformed(detail: string, format = "docx"): ZipEntryResult & { status: "error" } {
|
|
254
|
+
return { status: "error", reason: `not a readable .${format}: ${detail}` };
|
|
251
255
|
}
|
|
252
256
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
257
|
+
// Names of every entry, for the formats that keep one part per sheet or slide.
|
|
258
|
+
function zipEntryNames(buffer: Buffer, format: string): { status: "ok"; names: string[] } | { status: "error"; reason: string } {
|
|
259
|
+
const eocd = buffer.length >= 22 ? findEndOfCentralDirectory(buffer) : undefined;
|
|
260
|
+
if (eocd === undefined) return malformed("no ZIP end-of-central-directory record was found", format);
|
|
261
|
+
const entryCount = buffer.readUInt16LE(eocd + 10);
|
|
262
|
+
if (entryCount > MAX_ZIP_ENTRIES) return malformed(`the archive declares ${entryCount} entries, over the ${MAX_ZIP_ENTRIES} entry limit`, format);
|
|
263
|
+
const names: string[] = [];
|
|
264
|
+
let offset = buffer.readUInt32LE(eocd + 16);
|
|
265
|
+
for (let index = 0; index < entryCount; index += 1) {
|
|
266
|
+
if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== ZIP_CENTRAL_SIGNATURE) return malformed("the central directory is corrupt", format);
|
|
267
|
+
const nameLength = buffer.readUInt16LE(offset + 28);
|
|
268
|
+
if (offset + 46 + nameLength > buffer.length) return malformed("an entry name runs past the end of the file", format);
|
|
269
|
+
names.push(buffer.toString("utf8", offset + 46, offset + 46 + nameLength));
|
|
270
|
+
offset += 46 + nameLength + buffer.readUInt16LE(offset + 30) + buffer.readUInt16LE(offset + 32);
|
|
271
|
+
}
|
|
272
|
+
return { status: "ok", names };
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function readZipEntry(buffer: Buffer, entryName: string, format = "docx"): ZipEntryResult {
|
|
276
|
+
const fail = (detail: string) => malformed(detail, format);
|
|
277
|
+
const eocd = buffer.length >= 22 ? findEndOfCentralDirectory(buffer) : undefined;
|
|
278
|
+
if (eocd === undefined) return fail("no ZIP end-of-central-directory record was found");
|
|
256
279
|
|
|
257
280
|
const entryCount = buffer.readUInt16LE(eocd + 10);
|
|
258
281
|
let offset = buffer.readUInt32LE(eocd + 16);
|
|
259
282
|
if (entryCount > MAX_ZIP_ENTRIES) {
|
|
260
|
-
return
|
|
283
|
+
return fail(`the archive declares ${entryCount} entries, over the ${MAX_ZIP_ENTRIES} entry limit`);
|
|
261
284
|
}
|
|
262
285
|
|
|
263
286
|
for (let index = 0; index < entryCount; index += 1) {
|
|
264
|
-
if (offset + 46 > buffer.length) return
|
|
265
|
-
if (buffer.readUInt32LE(offset) !== ZIP_CENTRAL_SIGNATURE) return
|
|
287
|
+
if (offset + 46 > buffer.length) return fail("the central directory runs past the end of the file");
|
|
288
|
+
if (buffer.readUInt32LE(offset) !== ZIP_CENTRAL_SIGNATURE) return fail("the central directory is corrupt");
|
|
266
289
|
|
|
267
290
|
const method = buffer.readUInt16LE(offset + 10);
|
|
268
291
|
const compressedSize = buffer.readUInt32LE(offset + 20);
|
|
@@ -270,18 +293,18 @@ function readZipEntry(buffer: Buffer, entryName: string): ZipEntryResult {
|
|
|
270
293
|
const extraLength = buffer.readUInt16LE(offset + 30);
|
|
271
294
|
const commentLength = buffer.readUInt16LE(offset + 32);
|
|
272
295
|
const localOffset = buffer.readUInt32LE(offset + 42);
|
|
273
|
-
if (offset + 46 + nameLength > buffer.length) return
|
|
296
|
+
if (offset + 46 + nameLength > buffer.length) return fail("an entry name runs past the end of the file");
|
|
274
297
|
const name = buffer.toString("utf8", offset + 46, offset + 46 + nameLength);
|
|
275
298
|
|
|
276
299
|
if (name === entryName) {
|
|
277
|
-
if (localOffset + 30 > buffer.length) return
|
|
278
|
-
if (buffer.readUInt32LE(localOffset) !== ZIP_LOCAL_SIGNATURE) return
|
|
300
|
+
if (localOffset + 30 > buffer.length) return fail(`the ${entryName} entry header runs past the end of the file`);
|
|
301
|
+
if (buffer.readUInt32LE(localOffset) !== ZIP_LOCAL_SIGNATURE) return fail(`the ${entryName} entry header is corrupt`);
|
|
279
302
|
const localNameLength = buffer.readUInt16LE(localOffset + 26);
|
|
280
303
|
const localExtraLength = buffer.readUInt16LE(localOffset + 28);
|
|
281
304
|
const dataStart = localOffset + 30 + localNameLength + localExtraLength;
|
|
282
305
|
const dataEnd = dataStart + compressedSize;
|
|
283
306
|
if (dataStart > buffer.length || dataEnd > buffer.length) {
|
|
284
|
-
return
|
|
307
|
+
return fail(`the ${entryName} entry claims more data than the file holds`);
|
|
285
308
|
}
|
|
286
309
|
const data = buffer.subarray(dataStart, dataEnd);
|
|
287
310
|
if (method === 0) return { status: "ok", data: Buffer.from(data) };
|
|
@@ -289,15 +312,15 @@ function readZipEntry(buffer: Buffer, entryName: string): ZipEntryResult {
|
|
|
289
312
|
try {
|
|
290
313
|
return { status: "ok", data: zlib.inflateRawSync(data, { maxOutputLength: MAX_DOCUMENT_BYTES }) };
|
|
291
314
|
} catch {
|
|
292
|
-
return
|
|
315
|
+
return fail(`${entryName} could not be decompressed; it is corrupt or expands past the ${MAX_DOCUMENT_BYTES} byte limit`);
|
|
293
316
|
}
|
|
294
317
|
}
|
|
295
|
-
return
|
|
318
|
+
return fail(`${entryName} uses compression method ${method}, which is not supported${method === 99 ? " (the archive is encrypted)" : ""}`);
|
|
296
319
|
}
|
|
297
320
|
|
|
298
321
|
offset += 46 + nameLength + extraLength + commentLength;
|
|
299
322
|
}
|
|
300
|
-
return
|
|
323
|
+
return fail(`${entryName} is missing`);
|
|
301
324
|
}
|
|
302
325
|
|
|
303
326
|
// The numeric references here come out of a file that arrived from outside, so
|
|
@@ -369,9 +392,232 @@ export function extractDocxText(buffer: Buffer): DocumentExtraction {
|
|
|
369
392
|
return finish(decodeXmlEntities(text), "docx");
|
|
370
393
|
}
|
|
371
394
|
|
|
395
|
+
// --- office formats ------------------------------------------------------
|
|
396
|
+
//
|
|
397
|
+
// Spreadsheets, slide decks and OpenDocument files are the other formats an
|
|
398
|
+
// operator hands over as "the document", and none of them needs a converter:
|
|
399
|
+
// OOXML and OpenDocument are ZIP archives of XML, RTF is text with control
|
|
400
|
+
// words. Each comes out as plain text the model reads like a .docx — sheets as
|
|
401
|
+
// tab-separated rows, slides in order — through the same caps and the same
|
|
402
|
+
// strict UTF-8 rule.
|
|
403
|
+
|
|
404
|
+
function zipXml(buffer: Buffer, entryName: string, format: string): { status: "ok"; xml: string } | { status: "error"; reason: string } {
|
|
405
|
+
const entry = readZipEntry(buffer, entryName, format);
|
|
406
|
+
if (entry.status === "error") return entry;
|
|
407
|
+
const xml = decodeStrict(entry.data, "utf-8");
|
|
408
|
+
if (xml === undefined) return { status: "error", reason: `not a readable .${format}: ${entryName} is not valid UTF-8` };
|
|
409
|
+
if (xml.includes("\0")) return { status: "error", reason: `not a readable .${format}: ${entryName} holds binary data` };
|
|
410
|
+
return { status: "ok", xml };
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function xmlText(xml: string): string { return decodeXmlEntities(xml.replace(/<[^>]+>/g, "")); }
|
|
414
|
+
function attribute(tag: string, name: string): string | undefined {
|
|
415
|
+
const match = new RegExp(`\\s${name}="([^"]*)"`).exec(tag);
|
|
416
|
+
return match ? decodeXmlEntities(match[1]!) : undefined;
|
|
417
|
+
}
|
|
418
|
+
function numbered(names: string[], pattern: RegExp): string[] {
|
|
419
|
+
return names.map((name) => ({ name, match: pattern.exec(name) })).filter((item) => item.match)
|
|
420
|
+
.sort((left, right) => Number(left.match![1]) - Number(right.match![1])).map((item) => item.name);
|
|
421
|
+
}
|
|
422
|
+
function tidy(text: string): string {
|
|
423
|
+
return text.replace(/\r\n?/g, "\n").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
424
|
+
}
|
|
425
|
+
// Spreadsheet column letters to a zero-based index: "A" is 0, "AA" is 26.
|
|
426
|
+
function columnIndex(reference: string | undefined): number | undefined {
|
|
427
|
+
const letters = /^([A-Z]{1,3})\d*$/.exec(reference ?? "")?.[1];
|
|
428
|
+
if (!letters) return undefined;
|
|
429
|
+
let index = 0;
|
|
430
|
+
for (const letter of letters) index = index * 26 + letter.charCodeAt(0) - 64;
|
|
431
|
+
return index - 1;
|
|
432
|
+
}
|
|
433
|
+
// Wide enough for any real sheet; a cell reference past it is a forged file
|
|
434
|
+
// asking for a row of millions of tabs.
|
|
435
|
+
const MAX_SHEET_COLUMNS = 1024;
|
|
436
|
+
|
|
437
|
+
export function extractXlsxText(buffer: Buffer, format = "xlsx"): DocumentExtraction {
|
|
438
|
+
const listing = zipEntryNames(buffer, format);
|
|
439
|
+
if (listing.status === "error") return listing;
|
|
440
|
+
const shared: string[] = [];
|
|
441
|
+
if (listing.names.includes("xl/sharedStrings.xml")) {
|
|
442
|
+
const strings = zipXml(buffer, "xl/sharedStrings.xml", format);
|
|
443
|
+
if (strings.status === "error") return strings;
|
|
444
|
+
// Phonetic runs (<rPh>) are reading aids, not the cell's text.
|
|
445
|
+
for (const match of strings.xml.matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/g)) {
|
|
446
|
+
shared.push([...match[1]!.replace(/<rPh\b[\s\S]*?<\/rPh>/g, "").matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)].map((t) => decodeXmlEntities(t[1]!)).join(""));
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
// Sheet names and order come from the workbook; the part names are only a
|
|
450
|
+
// fallback when the workbook or its relationships cannot be read.
|
|
451
|
+
let sheets: Array<{ name: string; part: string }> = [];
|
|
452
|
+
const workbook = listing.names.includes("xl/workbook.xml") ? zipXml(buffer, "xl/workbook.xml", format) : undefined;
|
|
453
|
+
const rels = listing.names.includes("xl/_rels/workbook.xml.rels") ? zipXml(buffer, "xl/_rels/workbook.xml.rels", format) : undefined;
|
|
454
|
+
if (workbook?.status === "ok" && rels?.status === "ok") {
|
|
455
|
+
const targets = new Map<string, string>();
|
|
456
|
+
for (const match of rels.xml.matchAll(/<Relationship\b[^>]*>/g)) {
|
|
457
|
+
const id = attribute(match[0], "Id"), target = attribute(match[0], "Target");
|
|
458
|
+
if (id && target) targets.set(id, target.startsWith("/") ? target.slice(1) : `xl/${target}`.replace(/\/\.\//g, "/"));
|
|
459
|
+
}
|
|
460
|
+
for (const match of workbook.xml.matchAll(/<sheet\b[^>]*>/g)) {
|
|
461
|
+
const part = targets.get(attribute(match[0], "r:id") ?? "");
|
|
462
|
+
if (part && listing.names.includes(part)) sheets.push({ name: attribute(match[0], "name") ?? part, part });
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
if (!sheets.length) sheets = numbered(listing.names, /^xl\/worksheets\/sheet(\d+)\.xml$/).map((part, index) => ({ name: `Sheet${index + 1}`, part }));
|
|
466
|
+
if (!sheets.length) return { status: "error", reason: `not a readable .${format}: the workbook has no worksheets` };
|
|
467
|
+
const out: string[] = [];
|
|
468
|
+
let length = 0;
|
|
469
|
+
for (const sheet of sheets) {
|
|
470
|
+
const part = zipXml(buffer, sheet.part, format);
|
|
471
|
+
if (part.status === "error") return part;
|
|
472
|
+
const rows: string[] = [];
|
|
473
|
+
for (const row of part.xml.matchAll(/<row\b[^>]*>([\s\S]*?)<\/row>/g)) {
|
|
474
|
+
const cells: string[] = [];
|
|
475
|
+
for (const cell of row[1]!.matchAll(/<c\b([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) {
|
|
476
|
+
const tag = `<c${cell[1]}>`, body = cell[2] ?? "", type = attribute(tag, "t");
|
|
477
|
+
const value = /<v\b[^>]*>([\s\S]*?)<\/v>/.exec(body)?.[1];
|
|
478
|
+
const text = type === "s" ? shared[Number(value)] ?? ""
|
|
479
|
+
: type === "inlineStr" ? [...body.matchAll(/<t\b[^>]*>([\s\S]*?)<\/t>/g)].map((t) => decodeXmlEntities(t[1]!)).join("")
|
|
480
|
+
: type === "b" ? (value === "1" ? "TRUE" : "FALSE") : value === undefined ? "" : decodeXmlEntities(value);
|
|
481
|
+
const column = columnIndex(attribute(tag, "r")) ?? cells.length;
|
|
482
|
+
if (column >= MAX_SHEET_COLUMNS) continue;
|
|
483
|
+
while (cells.length < column) cells.push("");
|
|
484
|
+
cells[column] = text.replace(/[\t\r\n]+/g, " ");
|
|
485
|
+
}
|
|
486
|
+
while (cells.length && !cells[cells.length - 1]) cells.pop();
|
|
487
|
+
rows.push(cells.join("\t"));
|
|
488
|
+
length += rows[rows.length - 1]!.length + 1;
|
|
489
|
+
if (length > MAX_EXTRACTED_CHARS) break;
|
|
490
|
+
}
|
|
491
|
+
while (rows.length && !rows[rows.length - 1]) rows.pop();
|
|
492
|
+
out.push(`# Sheet: ${sheet.name}\n${rows.join("\n")}`);
|
|
493
|
+
if (length > MAX_EXTRACTED_CHARS) break;
|
|
494
|
+
}
|
|
495
|
+
return finish(out.join("\n\n").trim(), "office");
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
// DrawingML text: a paragraph per <a:p>, line breaks and tabs kept.
|
|
499
|
+
function drawingText(xml: string): string {
|
|
500
|
+
return tidy(xmlText(xml.replace(/<a:br\b[^>]*\/?>/g, "\n").replace(/<a:tab\b[^>]*\/?>/g, "\t").replace(/<\/a:p>/g, "\n")));
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
export function extractPptxText(buffer: Buffer): DocumentExtraction {
|
|
504
|
+
const listing = zipEntryNames(buffer, "pptx");
|
|
505
|
+
if (listing.status === "error") return listing;
|
|
506
|
+
const slides = numbered(listing.names, /^ppt\/slides\/slide(\d+)\.xml$/);
|
|
507
|
+
if (!slides.length) return { status: "error", reason: "not a readable .pptx: the presentation has no slides" };
|
|
508
|
+
const out: string[] = [];
|
|
509
|
+
for (const [index, slide] of slides.entries()) {
|
|
510
|
+
const part = zipXml(buffer, slide, "pptx");
|
|
511
|
+
if (part.status === "error") return part;
|
|
512
|
+
const notesName = `ppt/notesSlides/notesSlide${/slide(\d+)\.xml$/.exec(slide)![1]}.xml`;
|
|
513
|
+
const notes = listing.names.includes(notesName) ? zipXml(buffer, notesName, "pptx") : undefined;
|
|
514
|
+
// Speaker notes carry a slide-number field; only their prose is kept.
|
|
515
|
+
const notesText = notes?.status === "ok" ? drawingText(notes.xml.replace(/<a:fld\b[^>]*type="slidenum"[^>]*>[\s\S]*?<\/a:fld>/g, "")) : "";
|
|
516
|
+
out.push(`## Slide ${index + 1}\n${drawingText(part.xml)}${notesText ? `\n\nNotes:\n${notesText}` : ""}`);
|
|
517
|
+
}
|
|
518
|
+
return finish(out.join("\n\n").trim(), "office");
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
export function extractOpenDocumentText(buffer: Buffer, format: "odt" | "ods" | "odp"): DocumentExtraction {
|
|
522
|
+
const content = zipXml(buffer, "content.xml", format);
|
|
523
|
+
if (content.status === "error") return content;
|
|
524
|
+
const body = /<office:body\b[^>]*>([\s\S]*)<\/office:body>/.exec(content.xml)?.[1] ?? content.xml;
|
|
525
|
+
let slide = 0;
|
|
526
|
+
const text = body
|
|
527
|
+
// Tracked deletions and annotations are not what the document says.
|
|
528
|
+
.replace(/<text:tracked-changes\b[\s\S]*?<\/text:tracked-changes>/g, "")
|
|
529
|
+
.replace(/<office:annotation\b[\s\S]*?<\/office:annotation>/g, "")
|
|
530
|
+
.replace(/<table:table(?=[\s>])[^>]*>/g, (tag) => format === "ods" ? `\n# Sheet: ${attribute(tag, "table:name") ?? ""}\n` : "\n")
|
|
531
|
+
.replace(/<draw:page(?=[\s>])[^>]*>/g, () => `\n## Slide ${++slide}\n`)
|
|
532
|
+
.replace(/<text:s\b[^>]*\/>/g, (tag) => " ".repeat(Math.min(Number(attribute(tag, "text:c") ?? 1) || 1, 64)))
|
|
533
|
+
.replace(/<text:tab\b[^>]*\/?>/g, "\t")
|
|
534
|
+
.replace(/<text:line-break\b[^>]*\/?>/g, "\n")
|
|
535
|
+
.replace(/<\/table:table-cell>|<table:table-cell\b[^>]*\/>/g, "\t")
|
|
536
|
+
.replace(/<\/table:table-row>/g, "\n")
|
|
537
|
+
.replace(/<\/text:p>|<\/text:h>/g, (tag) => format === "ods" ? " " : "\n")
|
|
538
|
+
.replace(/<[^>]+>/g, "");
|
|
539
|
+
const rows = decodeXmlEntities(text).split("\n").map((line) => line.replace(/[ \t]+$/g, "").replace(/ +\t/g, "\t"));
|
|
540
|
+
return finish(tidy(rows.join("\n")), "office");
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// RTF readers proper are large; text extraction is a small state machine over
|
|
544
|
+
// groups and control words. Destinations that hold no body text (font and
|
|
545
|
+
// colour tables, pictures, metadata, anything marked \*) are skipped whole.
|
|
546
|
+
const RTF_SKIP = new Set(["fonttbl", "colortbl", "stylesheet", "info", "pict", "object", "header", "footer", "headerl", "headerr", "headerf",
|
|
547
|
+
"footerl", "footerr", "footerf", "listtable", "listoverridetable", "rsidtbl", "generator", "xmlnstbl", "themedata", "colorschememapping",
|
|
548
|
+
"datastore", "latentstyles", "filetbl", "revtbl", "pgdsctbl", "fldinst", "bkmkstart", "bkmkend", "nonshppict", "shp"]);
|
|
549
|
+
export function extractRtfText(buffer: Buffer): DocumentExtraction {
|
|
550
|
+
const source = buffer.toString("latin1");
|
|
551
|
+
if (!source.startsWith("{\\rtf")) return { status: "error", reason: "not a readable .rtf: the file does not start with an RTF header" };
|
|
552
|
+
if (source.includes("\0")) return { status: "error", reason: "not a readable .rtf: the file holds binary data" };
|
|
553
|
+
const codepage = /\\ansicpg(\d+)/.exec(source)?.[1];
|
|
554
|
+
let decoder: TextDecoder;
|
|
555
|
+
try { decoder = new TextDecoder(codepage && codepage !== "65001" ? `windows-${codepage}` : codepage === "65001" ? "utf-8" : "windows-1252"); }
|
|
556
|
+
catch { decoder = new TextDecoder("windows-1252"); }
|
|
557
|
+
const stack: Array<{ skip: boolean; uc: number }> = [];
|
|
558
|
+
let state = { skip: false, uc: 1 }, pendingSkip = 0, text = "", bytes: number[] = [];
|
|
559
|
+
const flush = () => { if (bytes.length) { text += decoder.decode(Uint8Array.from(bytes)); bytes = []; } };
|
|
560
|
+
const emit = (value: string) => { if (state.skip) return; if (pendingSkip > 0) { pendingSkip -= 1; return; } flush(); text += value; };
|
|
561
|
+
for (let index = 0; index < source.length && text.length <= MAX_EXTRACTED_CHARS; index += 1) {
|
|
562
|
+
const char = source[index]!;
|
|
563
|
+
if (char === "{") { stack.push(state); state = { ...state }; pendingSkip = 0; continue; }
|
|
564
|
+
if (char === "}") { flush(); state = stack.pop() ?? state; pendingSkip = 0; continue; }
|
|
565
|
+
if (char === "\r" || char === "\n") continue;
|
|
566
|
+
if (char !== "\\") { emit(char); continue; }
|
|
567
|
+
const next = source[index + 1] ?? "";
|
|
568
|
+
if (next === "\\" || next === "{" || next === "}") { emit(next); index += 1; continue; }
|
|
569
|
+
if (next === "'") {
|
|
570
|
+
const code = Number.parseInt(source.slice(index + 2, index + 4), 16);
|
|
571
|
+
index += 3;
|
|
572
|
+
if (Number.isNaN(code) || state.skip) continue;
|
|
573
|
+
if (pendingSkip > 0) { pendingSkip -= 1; continue; }
|
|
574
|
+
bytes.push(code); continue;
|
|
575
|
+
}
|
|
576
|
+
if (next === "\n" || next === "\r") { emit("\n"); index += 1; continue; }
|
|
577
|
+
if (next === "*") { state.skip = true; index += 1; continue; }
|
|
578
|
+
if (next === "~") { emit(" "); index += 1; continue; }
|
|
579
|
+
if (next === "-" || next === "_") { index += 1; continue; }
|
|
580
|
+
const word = /^([a-zA-Z]{1,32})(-?\d{1,10})? ?/.exec(source.slice(index + 1, index + 48));
|
|
581
|
+
if (!word) { index += 1; continue; }
|
|
582
|
+
index += word[0].length;
|
|
583
|
+
const name = word[1]!, parameter = word[2] === undefined ? undefined : Number(word[2]);
|
|
584
|
+
if (RTF_SKIP.has(name)) { state.skip = true; continue; }
|
|
585
|
+
if (name === "uc") { state.uc = Math.max(0, Math.min(parameter ?? 1, 8)); continue; }
|
|
586
|
+
if (name === "u" && parameter !== undefined) {
|
|
587
|
+
const point = parameter < 0 ? parameter + 65_536 : parameter;
|
|
588
|
+
emit(point >= 0xd800 && point <= 0xdfff ? "" : String.fromCharCode(point));
|
|
589
|
+
pendingSkip = state.uc; continue;
|
|
590
|
+
}
|
|
591
|
+
if (name === "par" || name === "line" || name === "sect" || name === "page" || name === "row") emit("\n");
|
|
592
|
+
else if (name === "tab" || name === "cell") emit("\t");
|
|
593
|
+
else if (name === "emdash") emit("—");
|
|
594
|
+
else if (name === "endash") emit("–");
|
|
595
|
+
else if (name === "bullet") emit("•");
|
|
596
|
+
else if (name === "lquote") emit("‘");
|
|
597
|
+
else if (name === "rquote") emit("’");
|
|
598
|
+
else if (name === "ldblquote") emit("“");
|
|
599
|
+
else if (name === "rdblquote") emit("”");
|
|
600
|
+
}
|
|
601
|
+
flush();
|
|
602
|
+
return finish(tidy(text), "office");
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// One entry point for every office extension, so the attachment path and the
|
|
606
|
+
// document reader cannot disagree about which formats open.
|
|
607
|
+
export function extractOfficeText(extension: string, buffer: Buffer): DocumentExtraction | undefined {
|
|
608
|
+
switch (extension) {
|
|
609
|
+
case "docx": return extractDocxText(buffer);
|
|
610
|
+
case "xlsx": case "xlsm": return extractXlsxText(buffer, extension);
|
|
611
|
+
case "pptx": return extractPptxText(buffer);
|
|
612
|
+
case "odt": case "ods": case "odp": return extractOpenDocumentText(buffer, extension);
|
|
613
|
+
case "rtf": return extractRtfText(buffer);
|
|
614
|
+
default: return undefined;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
372
618
|
// --- extraction ----------------------------------------------------------
|
|
373
619
|
|
|
374
|
-
function finish(text: string, kind: "text" | "docx" | "pdf"): DocumentExtraction {
|
|
620
|
+
function finish(text: string, kind: "text" | "docx" | "pdf" | "office"): DocumentExtraction {
|
|
375
621
|
if (text.length > MAX_EXTRACTED_CHARS) {
|
|
376
622
|
let cut = text.slice(0, MAX_EXTRACTED_CHARS);
|
|
377
623
|
// slice counts UTF-16 code units, so the cut can land between the two halves
|
|
@@ -590,7 +836,7 @@ export function extractDocument(
|
|
|
590
836
|
}
|
|
591
837
|
const buffer = readAllFromDescriptor(opened.fd, opened.size);
|
|
592
838
|
if (resolution.extension === "docx") return extractDocxText(buffer);
|
|
593
|
-
return extractTextDocument(buffer);
|
|
839
|
+
return extractOfficeText(resolution.extension, buffer) ?? extractTextDocument(buffer);
|
|
594
840
|
} catch (error) {
|
|
595
841
|
return { status: "error", reason: error instanceof Error ? error.message : String(error) };
|
|
596
842
|
} finally {
|
|
@@ -109,6 +109,32 @@ export async function stageCompaction(session, api, sdk) {
|
|
|
109
109
|
};
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
+
// Studio takes at most 4 MiB per request body, and a summary is triggered by
|
|
113
|
+
// tokens: an image costs ~1.6k tokens but its base64 can weigh half a megabyte,
|
|
114
|
+
// so a run of screenshots overflows the body long before the window
|
|
115
|
+
// (request_too_large at 130k tokens). Each request therefore carries only the
|
|
116
|
+
// newest images within a byte and count budget; older ones become a line saying
|
|
117
|
+
// so. The session keeps every image; only what is sent shrinks.
|
|
118
|
+
export const REQUEST_IMAGE_BYTES = 2 * 1024 * 1024, REQUEST_IMAGE_COUNT = 20;
|
|
119
|
+
export function boundRequestImages(messages, budget = REQUEST_IMAGE_BYTES, count = REQUEST_IMAGE_COUNT) {
|
|
120
|
+
let bytes = 0, kept = 0;
|
|
121
|
+
const out = [...messages];
|
|
122
|
+
for (let index = out.length - 1; index >= 0; index -= 1) {
|
|
123
|
+
const message = out[index];
|
|
124
|
+
if (!Array.isArray(message?.content) || !message.content.some(part => part?.type === 'image')) continue;
|
|
125
|
+
const content = [...message.content];
|
|
126
|
+
for (let at = content.length - 1; at >= 0; at -= 1) {
|
|
127
|
+
const part = content[at];
|
|
128
|
+
if (part?.type !== 'image') continue;
|
|
129
|
+
const size = typeof part.data === 'string' ? part.data.length : 0;
|
|
130
|
+
if (kept < count && bytes + size <= budget) { bytes += size; kept += 1; continue; }
|
|
131
|
+
content[at] = { type: 'text', text: `[Image left out of this request (${part.mimeType ?? 'image'}, ${Math.ceil(size * 3 / 4 / 1024)} KB): Studio takes at most 4 MiB per request, so only the newest images are sent again. Take a new screenshot or read the file again if you need to see it.]` };
|
|
132
|
+
}
|
|
133
|
+
out[index] = { ...message, content };
|
|
134
|
+
}
|
|
135
|
+
return out;
|
|
136
|
+
}
|
|
137
|
+
|
|
112
138
|
// What a compaction did, for Studio's logs (Logs → "Lượt agent"): its summary
|
|
113
139
|
// calls name it (X-Agent-Compaction) and the main agent's next call says how
|
|
114
140
|
// it ended (X-Agent-Compacted, see compactionHeaders). Pi's own token counts
|
|
@@ -19,7 +19,7 @@ import { fastServed, fastSetter, fastState, recordFast, restoreFast } from './fa
|
|
|
19
19
|
import { runHelper } from './helper-run.mjs';
|
|
20
20
|
import { askTool } from './member-questions.mjs';
|
|
21
21
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
22
|
-
import { stageCompaction, compactEarly, backOffFailedCompaction, trackCompaction } from './compaction.mjs';
|
|
22
|
+
import { stageCompaction, compactEarly, backOffFailedCompaction, trackCompaction, boundRequestImages } from './compaction.mjs';
|
|
23
23
|
import { readPatchDigest, readPatchSnapshot, readChangedFiles, readHead } from './patch-snapshot.mjs';
|
|
24
24
|
import { memberTurn } from './continuation.mjs';
|
|
25
25
|
import { workflowPolicy, repositoryChecks, sessionPlanTool, currentPlan, workflowPrompt, RunProcess, completionGate, pendingBaseline, pendingPaths, pendingHead, processEditTools } from './workflow.mjs';
|
|
@@ -174,11 +174,11 @@ export class ManagedSession {
|
|
|
174
174
|
self.session.managedFastServed = () => { const route = self.routes.get(self.modelRuntime); return route ? fastServed(route) : null; };
|
|
175
175
|
// A helper's receipt (its tokens, the patch it read) is for the member's
|
|
176
176
|
// eyes: the agent has the helper's answer as the tool result, and a
|
|
177
|
-
// receipt in its context reads as a message from the member.
|
|
177
|
+
// receipt in its context reads as a message from the member. Old images beyond Studio's body limit are left out.
|
|
178
178
|
const context = self.session.agent.transformContext;
|
|
179
|
-
self.session.agent.transformContext = async (messages, signal) => (await prepareMemoryContext(self,
|
|
179
|
+
self.session.agent.transformContext = async (messages, signal) => boundRequestImages((await prepareMemoryContext(self,
|
|
180
180
|
context ? await context(messages, signal) : messages, signal))
|
|
181
|
-
.filter(m => !(m.role === 'custom' && ['agent-watch-helper-receipt', 'agent-watch-fast-notice'].includes(m.customType)));
|
|
181
|
+
.filter(m => !(m.role === 'custom' && ['agent-watch-helper-receipt', 'agent-watch-fast-notice'].includes(m.customType))));
|
|
182
182
|
// Pi answers unknown tools before hooks; count those failures in the run report.
|
|
183
183
|
self.session.subscribe(event => {
|
|
184
184
|
if (event?.type === 'tool_execution_end' && event.isError && textContent(event.result ?? {}) === `Tool ${event.toolName} not found`) self.run && (self.run.unknownTools += 1);
|