@sema-agent/core 7.59.0 → 7.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +564 -0
- package/dist/agents/agent-definition.d.ts +7 -0
- package/dist/agents/agent-definition.js +12 -0
- package/dist/agents/agent-transcript-tool.d.ts +2 -1
- package/dist/agents/agent-transcript-tool.js +3 -1
- package/dist/agents/child-model-seat.js +6 -3
- package/dist/agents/cumulative-stats.js +0 -1
- package/dist/agents/inherited-tool-pool.d.ts +26 -1
- package/dist/agents/inherited-tool-pool.js +22 -0
- package/dist/agents/retain-ledger.d.ts +11 -1
- package/dist/agents/retain-ledger.js +13 -2
- package/dist/agents/roster-store.d.ts +1 -1
- package/dist/agents/roster-store.js +3 -1
- package/dist/agents/send-message-tool.d.ts +8 -19
- package/dist/agents/send-message-tool.js +11 -38
- package/dist/agents/subagent.d.ts +51 -17
- package/dist/agents/subagent.js +117 -78
- package/dist/agents/tool-filter.d.ts +1 -3
- package/dist/agents/tool-filter.js +4 -12
- package/dist/bench/metrics.d.ts +3 -5
- package/dist/bench/metrics.js +0 -1
- package/dist/brain/degrading.js +3 -1
- package/dist/brain/route-adjudicator.js +3 -1
- package/dist/brain/status-sink.d.ts +7 -1
- package/dist/brain/status-sink.js +6 -0
- package/dist/brain/timeout.d.ts +3 -1
- package/dist/brain/timeout.js +18 -5
- package/dist/core/a2a.d.ts +15 -5
- package/dist/core/a2a.js +34 -19
- package/dist/core/agent-definition.d.ts +20 -0
- package/dist/core/allow-entry.d.ts +2 -0
- package/dist/core/allow-entry.js +3 -1
- package/dist/core/artifact-host.d.ts +0 -2
- package/dist/core/artifact-host.js +0 -1
- package/dist/core/auto-mode-defaults.d.ts +8 -0
- package/dist/core/auto-mode-defaults.js +1 -0
- package/dist/core/auto-mode-rebuild.d.ts +2 -1
- package/dist/core/auto-mode.d.ts +39 -4
- package/dist/core/auto-mode.js +63 -12
- package/dist/core/background-agent-store.d.ts +10 -1
- package/dist/core/background-agent-store.js +3 -2
- package/dist/core/background-shell.d.ts +12 -1
- package/dist/core/background-shell.js +12 -0
- package/dist/core/checkpoint-store.d.ts +8 -13
- package/dist/core/checkpoint-store.js +4 -3
- package/dist/core/compliance.d.ts +1 -1
- package/dist/core/credential-echo.d.ts +52 -0
- package/dist/core/credential-echo.js +65 -0
- package/dist/core/credential-vocabulary.d.ts +13 -3
- package/dist/core/credential-vocabulary.js +12 -4
- package/dist/core/cross-boundary-taint.d.ts +4 -5
- package/dist/core/engine-notice.d.ts +58 -15
- package/dist/core/frame-settlement.d.ts +5 -0
- package/dist/core/frame-settlement.js +4 -0
- package/dist/core/gate-fold.js +7 -5
- package/dist/core/gate-lanes.js +39 -8
- package/dist/core/governance-codes.d.ts +9 -1
- package/dist/core/governance-codes.js +12 -0
- package/dist/core/hands-band-screen.js +0 -1
- package/dist/core/hands-band.d.ts +0 -3
- package/dist/core/hooks.d.ts +39 -56
- package/dist/core/hooks.js +123 -77
- package/dist/core/image-admission.d.ts +104 -51
- package/dist/core/image-admission.js +170 -46
- package/dist/core/image-downsample.d.ts +36 -69
- package/dist/core/image-downsample.js +111 -73
- package/dist/core/integer-option.d.ts +21 -0
- package/dist/core/integer-option.js +13 -0
- package/dist/core/locked-config.d.ts +54 -3
- package/dist/core/locked-config.js +81 -50
- package/dist/core/mcp-call-meta.d.ts +2 -2
- package/dist/core/mcp-call-meta.js +9 -10
- package/dist/core/mcp-failure.d.ts +9 -3
- package/dist/core/mcp-failure.js +10 -0
- package/dist/core/mcp-host-seat.d.ts +92 -0
- package/dist/core/mcp-host-seat.js +131 -0
- package/dist/core/mcp-probe.d.ts +7 -2
- package/dist/core/mcp-probe.js +7 -2
- package/dist/core/mcp-server-spec.d.ts +38 -9
- package/dist/core/mcp-server-spec.js +55 -21
- package/dist/core/mcp.d.ts +17 -6
- package/dist/core/mcp.js +67 -55
- package/dist/core/memory-admission.d.ts +23 -6
- package/dist/core/memory-admission.js +19 -9
- package/dist/core/memory-engine/content-origin.d.ts +13 -0
- package/dist/core/memory-engine/content-origin.js +21 -3
- package/dist/core/memory-engine/index.d.ts +2 -1
- package/dist/core/memory-engine/index.js +2 -1
- package/dist/core/memory-engine/scope-contract.d.ts +13 -3
- package/dist/core/memory-engine/scope-contract.js +2 -2
- package/dist/core/memory-engine/sync-client.d.ts +45 -11
- package/dist/core/memory-engine/sync-client.js +82 -11
- package/dist/core/memory-engine/sync-report.d.ts +103 -0
- package/dist/core/memory-engine/sync-report.js +119 -0
- package/dist/core/memory-engine/sync.d.ts +4 -2
- package/dist/core/mode-policy.d.ts +9 -1
- package/dist/core/mode-policy.js +11 -1
- package/dist/core/model-receipt.d.ts +39 -0
- package/dist/core/model-receipt.js +139 -0
- package/dist/core/mods/agent-stations.d.ts +65 -0
- package/dist/core/mods/agent-stations.js +105 -0
- package/dist/core/mods/kernel/dispatch.d.ts +5 -1
- package/dist/core/mods/kernel/dispatch.js +4 -4
- package/dist/core/mods/kernel/events.d.ts +10 -4
- package/dist/core/mods/kernel/events.js +10 -6
- package/dist/core/mods/kernel/receive.d.ts +5 -0
- package/dist/core/mods/kernel/receive.js +10 -0
- package/dist/core/mods/lifecycle-stations.d.ts +41 -0
- package/dist/core/mods/lifecycle-stations.js +35 -0
- package/dist/core/mods/stations.d.ts +51 -59
- package/dist/core/mods/stations.js +32 -42
- package/dist/core/mods/tool-call.d.ts +10 -4
- package/dist/core/mods/tool-call.js +28 -12
- package/dist/core/mounted-mcp.d.ts +5 -0
- package/dist/core/mounted-mcp.js +2 -2
- package/dist/core/namespaced-rule-name.d.ts +7 -1
- package/dist/core/namespaced-rule-name.js +13 -2
- package/dist/core/oracle-isolation.js +1 -1
- package/dist/core/own-property-receipt.d.ts +3 -0
- package/dist/core/own-property-receipt.js +3 -0
- package/dist/core/park-selfcheck.js +8 -8
- package/dist/core/peer-seat.d.ts +63 -8
- package/dist/core/peer-seat.js +179 -34
- package/dist/core/permission-rule-consent.d.ts +0 -21
- package/dist/core/permission-rule-consent.js +1 -2
- package/dist/core/permission-rule-model.d.ts +4 -2
- package/dist/core/permission-rule-model.js +7 -6
- package/dist/core/protocol-http-auth.d.ts +14 -2
- package/dist/core/protocol-http-auth.js +16 -7
- package/dist/core/read-face-seats.d.ts +7 -5
- package/dist/core/read-face-seats.js +4 -1
- package/dist/core/redaction-marks.d.ts +4 -3
- package/dist/core/redaction-marks.js +2 -2
- package/dist/core/remote-env.d.ts +146 -98
- package/dist/core/remote-env.js +113 -14
- package/dist/core/retention.js +37 -12
- package/dist/core/retired-keys.d.ts +6 -1
- package/dist/core/retired-keys.js +4 -1
- package/dist/core/rewrite-rejudge.d.ts +12 -5
- package/dist/core/rewrite-rejudge.js +9 -5
- package/dist/core/roles.d.ts +37 -3
- package/dist/core/roles.js +126 -78
- package/dist/core/rule-owner.d.ts +3 -0
- package/dist/core/runner/assemble-result.d.ts +0 -5
- package/dist/core/runner/assemble-result.js +0 -1
- package/dist/core/runner/call-hook-stations.d.ts +17 -10
- package/dist/core/runner/call-hook-stations.js +21 -10
- package/dist/core/runner/child-tree.d.ts +7 -8
- package/dist/core/runner/child-tree.js +6 -6
- package/dist/core/runner/classifier-ladder.d.ts +10 -2
- package/dist/core/runner/classifier-ladder.js +39 -8
- package/dist/core/runner/construction-doors.js +5 -0
- package/dist/core/runner/contracts.d.ts +62 -41
- package/dist/core/runner/cross-boundary-lane.d.ts +3 -5
- package/dist/core/runner/cross-boundary-lane.js +11 -2
- package/dist/core/runner/env-capability-census.d.ts +15 -20
- package/dist/core/runner/env-capability-census.js +25 -34
- package/dist/core/runner/file-history-request.d.ts +2 -2
- package/dist/core/runner/file-history-request.js +0 -8
- package/dist/core/runner/gate-exit.d.ts +75 -7
- package/dist/core/runner/gate-exit.js +20 -1
- package/dist/core/runner/git-status-frame.d.ts +5 -2
- package/dist/core/runner/git-status-frame.js +3 -4
- package/dist/core/runner/image.d.ts +7 -2
- package/dist/core/runner/image.js +14 -8
- package/dist/core/runner/inline-image.d.ts +10 -8
- package/dist/core/runner/inline-image.js +11 -9
- package/dist/core/runner/memory-consolidation-driver.js +4 -4
- package/dist/core/runner/permission-rule-lanes.js +1 -1
- package/dist/core/runner/prepare-boundary-parks.d.ts +2 -2
- package/dist/core/runner/prepare-boundary-parks.js +3 -3
- package/dist/core/runner/prepare-caps-and-workflow.d.ts +4 -0
- package/dist/core/runner/prepare-caps-and-workflow.js +4 -3
- package/dist/core/runner/prepare-config-doors.d.ts +6 -1
- package/dist/core/runner/prepare-config-doors.js +14 -9
- package/dist/core/runner/prepare-delegation-surface.d.ts +5 -3
- package/dist/core/runner/prepare-delegation-surface.js +4 -4
- package/dist/core/runner/prepare-file-history.d.ts +10 -12
- package/dist/core/runner/prepare-file-history.js +3 -12
- package/dist/core/runner/prepare-gate-stations.d.ts +4 -3
- package/dist/core/runner/prepare-gate-stations.js +4 -3
- package/dist/core/runner/prepare-hands-readface.d.ts +3 -1
- package/dist/core/runner/prepare-hands-readface.js +15 -7
- package/dist/core/runner/prepare-inherited-gate.d.ts +5 -4
- package/dist/core/runner/prepare-inherited-gate.js +9 -5
- package/dist/core/runner/prepare-memory-engine-session.js +1 -1
- package/dist/core/runner/prepare-memory.d.ts +2 -2
- package/dist/core/runner/prepare-memory.js +2 -2
- package/dist/core/runner/prepare-park-ask.d.ts +3 -5
- package/dist/core/runner/prepare-park-ask.js +3 -8
- package/dist/core/runner/prepare-policy-chain.d.ts +5 -4
- package/dist/core/runner/prepare-policy-chain.js +120 -108
- package/dist/core/runner/prepare-project-context.js +8 -5
- package/dist/core/runner/prepare-prompt-inputs.d.ts +4 -1
- package/dist/core/runner/prepare-prompt-inputs.js +3 -2
- package/dist/core/runner/prepare-protocol-tools.d.ts +5 -3
- package/dist/core/runner/prepare-protocol-tools.js +9 -2
- package/dist/core/runner/prepare-suspend-saga.d.ts +0 -2
- package/dist/core/runner/prepare-suspend-saga.js +5 -7
- package/dist/core/runner/prepare-task.js +41 -27
- package/dist/core/runner/prepare-turn-wiring.d.ts +2 -0
- package/dist/core/runner/prepare-turn-wiring.js +7 -2
- package/dist/core/runner/prepare-wiring-manifest.d.ts +4 -3
- package/dist/core/runner/prepare-wiring-manifest.js +32 -18
- package/dist/core/runner/prepare-workspace-restore.js +3 -3
- package/dist/core/runner/resume-checkpoint-screen.js +4 -4
- package/dist/core/runner/retired-runner-deps.js +4 -1
- package/dist/core/runner/run-attachment-seats.js +3 -3
- package/dist/core/runner/run-brain-sinks.js +1 -0
- package/dist/core/runner/run-git-lane.d.ts +3 -1
- package/dist/core/runner/run-git-lane.js +1 -1
- package/dist/core/runner/run-harness-handlers.d.ts +3 -1
- package/dist/core/runner/run-harness-handlers.js +3 -1
- package/dist/core/runner/run-identity-wiring.js +1 -1
- package/dist/core/runner/run-leg.js +26 -28
- package/dist/core/runner/run-notification-lane.d.ts +3 -1
- package/dist/core/runner/run-notification-lane.js +16 -10
- package/dist/core/runner/run-settle-and-teardown.js +5 -1
- package/dist/core/runner/run-stop-and-final-verify.d.ts +4 -0
- package/dist/core/runner/run-stop-and-final-verify.js +12 -4
- package/dist/core/runner/run-turn-boundary.js +12 -7
- package/dist/core/runner/runtask.d.ts +7 -4
- package/dist/core/runner/runtask.js +19 -11
- package/dist/core/runner/skills-seat.js +2 -2
- package/dist/core/runner/sub-agent-deny-lists.js +10 -7
- package/dist/core/runner/synthetic-tools.js +2 -2
- package/dist/core/runner/tool-disclosure.js +0 -2
- package/dist/core/runner/tool-face-overlay.d.ts +1 -1
- package/dist/core/runner/tool-face-overlay.js +26 -19
- package/dist/core/runner/tool-result-media-seat.d.ts +9 -1
- package/dist/core/runner/tool-result-media-seat.js +17 -1
- package/dist/core/runner/turn-attachments.d.ts +3 -5
- package/dist/core/runner/turn-attachments.js +3 -16
- package/dist/core/runner-deps.d.ts +31 -18
- package/dist/core/secret-env.d.ts +3 -2
- package/dist/core/session-policy-store.d.ts +9 -2
- package/dist/core/session-policy-store.js +16 -1
- package/dist/core/session-store.d.ts +3 -2
- package/dist/core/session-store.js +50 -11
- package/dist/core/skill-grant.d.ts +9 -0
- package/dist/core/skill-grant.js +18 -0
- package/dist/core/store-contracts/tool-result-store-contract.d.ts +1 -1
- package/dist/core/store-contracts/tool-result-store-contract.js +37 -0
- package/dist/core/stub-env.d.ts +5 -5
- package/dist/core/stub-env.js +11 -1
- package/dist/core/swappable-deps.d.ts +17 -43
- package/dist/core/swappable-deps.js +15 -5
- package/dist/core/synthetic-read-verdict.d.ts +1 -1
- package/dist/core/task-event.d.ts +18 -2
- package/dist/core/task-notification.d.ts +88 -5
- package/dist/core/task-notification.js +130 -7
- package/dist/core/task-registry-agent.d.ts +18 -12
- package/dist/core/task-registry-agent.js +18 -6
- package/dist/core/task-registry-monitor.d.ts +7 -0
- package/dist/core/task-registry-monitor.js +28 -1
- package/dist/core/task-registry-shared.d.ts +38 -3
- package/dist/core/task-registry-shared.js +21 -7
- package/dist/core/task-registry-workflow.js +1 -1
- package/dist/core/task-registry.d.ts +50 -35
- package/dist/core/task-registry.js +64 -23
- package/dist/core/task-result.d.ts +3 -12
- package/dist/core/task-spec.d.ts +3 -6
- package/dist/core/task-stream.d.ts +1 -1
- package/dist/core/tool-catalog-entries.js +1 -1
- package/dist/core/tool-face-reader.d.ts +7 -6
- package/dist/core/tool-face-reader.js +4 -3
- package/dist/core/tool-name-aliases.d.ts +19 -0
- package/dist/core/tool-name-aliases.js +54 -2
- package/dist/core/tool-policy.d.ts +5 -4
- package/dist/core/tool-policy.js +1 -1
- package/dist/core/tool-registry.d.ts +5 -5
- package/dist/core/tool-registry.js +2 -2
- package/dist/core/tool-result-store.d.ts +88 -5
- package/dist/core/tool-result-store.js +51 -12
- package/dist/core/tool-spec.d.ts +20 -4
- package/dist/core/trace.d.ts +0 -1
- package/dist/core/types.d.ts +3 -3
- package/dist/core/types.js +1 -1
- package/dist/core/untrusted-text.js +1 -1
- package/dist/core/user-attachment.d.ts +10 -5
- package/dist/core/user-attachment.js +14 -6
- package/dist/engine/execution-env/node-execution-env.d.ts +7 -2
- package/dist/engine/execution-env/node-execution-env.js +12 -1
- package/dist/engine/harness/agent-harness.d.ts +1 -1
- package/dist/engine/harness/agent-harness.js +8 -4
- package/dist/engine/harness/types.d.ts +112 -29
- package/dist/engine/loop/agent-loop.d.ts +12 -12
- package/dist/engine/loop/agent-loop.js +153 -21
- package/dist/engine/loop/types.d.ts +9 -2
- package/dist/engine/lsp/node-lsp-manager.d.ts +3 -3
- package/dist/engine/lsp/node-lsp-manager.js +32 -13
- package/dist/index.d.ts +22 -14
- package/dist/index.js +15 -10
- package/dist/internal/harness-types.d.ts +1 -1
- package/dist/orchestration/run-workflow-tool.d.ts +9 -3
- package/dist/orchestration/run-workflow-tool.js +59 -29
- package/dist/orchestration/workflow-governance.d.ts +1 -1
- package/dist/orchestration/workflow-governance.js +2 -1
- package/dist/orchestration/workflow.d.ts +4 -0
- package/dist/orchestration/workflow.js +25 -4
- package/dist/prompts/default.d.ts +1 -10
- package/dist/prompts/default.js +0 -2
- package/dist/stores/file/background-agent-store.d.ts +1 -1
- package/dist/stores/file/background-agent-store.js +3 -1
- package/dist/stores/file/checkpoint-store.d.ts +127 -2
- package/dist/stores/file/checkpoint-store.js +19 -14
- package/dist/stores/file/file-history-store.d.ts +1 -1
- package/dist/stores/file/file-history-store.js +3 -1
- package/dist/stores/file/index.d.ts +4 -3
- package/dist/stores/file/index.js +17 -8
- package/dist/stores/file/mailbox-store.d.ts +1 -1
- package/dist/stores/file/mailbox-store.js +3 -1
- package/dist/stores/file/memory-store.d.ts +1 -1
- package/dist/stores/file/memory-store.js +3 -1
- package/dist/stores/file/session-policy-store.d.ts +1 -1
- package/dist/stores/file/session-policy-store.js +3 -1
- package/dist/stores/file/session-store.d.ts +1 -1
- package/dist/stores/file/session-store.js +3 -1
- package/dist/stores/file/shared-ledger.d.ts +26 -0
- package/dist/stores/file/shared-ledger.js +38 -9
- package/dist/stores/file/tool-result-store.d.ts +20 -2
- package/dist/stores/file/tool-result-store.js +74 -6
- package/dist/stores/file/usage-window-store.d.ts +1 -1
- package/dist/stores/file/usage-window-store.js +3 -1
- package/dist/stores/file/workflow-journal-store.d.ts +1 -11
- package/dist/stores/file/workflow-journal-store.js +3 -2
- package/dist/tools/fs/containment-face.d.ts +5 -0
- package/dist/tools/fs/encoding.d.ts +13 -4
- package/dist/tools/fs/encoding.js +17 -2
- package/dist/tools/fs/fs-bash-background-text.d.ts +7 -0
- package/dist/tools/fs/fs-bash-background-text.js +1 -0
- package/dist/tools/fs/fs-bash.d.ts +8 -4
- package/dist/tools/fs/fs-bash.js +26 -35
- package/dist/tools/fs/fs-pdf.d.ts +2 -2
- package/dist/tools/fs/fs-pdf.js +26 -21
- package/dist/tools/fs/fs-read.d.ts +5 -4
- package/dist/tools/fs/fs-read.js +35 -18
- package/dist/tools/fs/fs-search-tools.js +4 -1
- package/dist/tools/fs/fs-shared.d.ts +11 -13
- package/dist/tools/fs/fs-shared.js +11 -7
- package/dist/tools/fs/fs-write.js +5 -5
- package/dist/tools/fs/index.d.ts +14 -8
- package/dist/tools/fs/index.js +8 -4
- package/dist/tools/fs/read-floor.d.ts +44 -1
- package/dist/tools/fs/read-floor.js +35 -0
- package/dist/tools/fs/removal-targets.d.ts +9 -4
- package/dist/tools/fs/removal-targets.js +17 -9
- package/dist/tools/fs/search.d.ts +3 -3
- package/dist/tools/fs/search.js +4 -4
- package/dist/tools/monitor.d.ts +5 -4
- package/dist/tools/monitor.js +15 -8
- package/dist/tools/scheduler-tools.js +42 -4
- package/dist/tools/web.d.ts +0 -10
- package/dist/tools/web.js +1 -4
- package/package.json +4 -2
- package/test/export-surface.snapshot.json +75 -21
- package/wire-changes/core-stores-sql@7.60.0.json +30 -0
- package/wire-changes/core@7.60.0.json +3274 -0
- package/wire-changes/mods-kernel@7.60.0.json +62 -0
- package/wire-changes/primitives@7.60.0.json +30 -0
|
@@ -1,74 +1,92 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The ONE admission an image's bytes pass before they reach the model as an `ImageContent` block —
|
|
3
|
-
* whether a file supplied them (the Read tool's image lane), a caller did (`TaskSpec.images`' inline
|
|
4
|
-
* arm, a user attachment) or a tool returned them (the tool-result seat below). Four steps, in this order:
|
|
5
|
-
*
|
|
6
|
-
* ① the claimed type is one the content API accepts (png / jpeg / gif / webp — the four
|
|
7
|
-
* {@link imageMagicMatches} can verify);
|
|
8
|
-
* ② the raw bytes are within the read cap (5 MiB without a downsampler, 64 MiB with one);
|
|
9
|
-
* ③ the bytes open with the claimed type's header — its signature and the first structure the format requires
|
|
10
|
-
* ({@link IMAGE_HEADER_CHECKS}); an empty / text / mis-named "png", or an Apple-optimized PNG, would otherwise
|
|
11
|
-
* become a malformed block that 400s the NEXT whole provider request;
|
|
12
|
-
* ④ optional downsample (when a downsampler is supplied; a processing failure keeps the original
|
|
13
|
-
* bytes, CC's catch → createImageResponse(original)), then the provider's hard limit, which is on the
|
|
14
|
-
* BASE64 length, not raw bytes.
|
|
15
|
-
*
|
|
16
|
-
* The answer is structured; each carrier words its own refusal (the Read tool speaks to the model, the
|
|
17
|
-
* task door to the caller). The block it admits is built from the bytes it checked — never from a
|
|
18
|
-
* caller's string it did not decode. A document's bytes pass the same shape of admission
|
|
19
|
-
* ({@link admitDocumentBytes}: `application/pdf`, the whole-send cap, the `%PDF-` signature).
|
|
20
|
-
*
|
|
21
|
-
* THE TOOL-RESULT SEAT ({@link admitToolResultMedia}): every executed call's result passes the loop's one
|
|
22
|
-
* post-execution half (`finalizeToolOutcome`) — an engine tool's, an MCP tool's converted content, a host
|
|
23
|
-
* tool's own return, a resumed approved call's, and whatever a `tool_result` hook put in their place — so
|
|
24
|
-
* the media blocks of every supply are admitted there, once, after the last hand that can change them and
|
|
25
|
-
* before the frame and the transcript read them. A block whose bytes the admission refuses becomes a text
|
|
26
|
-
* block that names the tool, the refusal and the size — never the bytes, and never silently nothing.
|
|
27
|
-
*/
|
|
28
1
|
import type { DocumentContent, ImageContent } from "../internal/llm.js";
|
|
29
|
-
import {
|
|
2
|
+
import type { ImageDimensions, ImageDownsampler } from "./image-downsample.js";
|
|
30
3
|
import { type MediaBlock } from "./media-byte-cap.js";
|
|
31
|
-
/** Byte cap on a Read-image WITHOUT a downsampler: an image file is returned AS a visual
|
|
32
|
-
* `ImageContent` block (base64) rather than refused. Bounded so a huge image cannot blow the request.
|
|
33
|
-
* This stat-time cap only bounds the READ; the accept/refuse decision is the base64 gate below (the
|
|
34
|
-
* provider limit is on base64 LENGTH, so the effective no-sharp accept bound is raw ≤ 3.75MB). */
|
|
35
|
-
export declare const MAX_IMAGE_READ_BYTES: number;
|
|
36
|
-
/** Byte cap on a Read-image WITH a downsampler: sharp can shrink a big photo to CC's 2000×2000/3.75MB
|
|
37
|
-
* target, so the smaller hard refusal does not apply (CC readImageWithTokenBudget reads the file
|
|
38
|
-
* unbounded; we keep an OOM sanity bound — deliberate delta, camera photos fit comfortably). */
|
|
39
|
-
export declare const MAX_IMAGE_DOWNSAMPLE_INPUT_BYTES: number;
|
|
40
4
|
/** Max raw size sent whole as a document block (CC `PDF_TARGET_RAW_SIZE`). */
|
|
41
5
|
export declare const PDF_TARGET_RAW_SIZE: number;
|
|
42
|
-
/**
|
|
6
|
+
/**
|
|
7
|
+
* Every number an image passes on its way to the model, in one table — the admission and its processor read it, and so
|
|
8
|
+
* does every carrier that words a refusal; a downstream that bounds an image by the same limit reads the same field.
|
|
9
|
+
* - `hard`: what refuses. `formats` (the header rows above), `readBytes` (the raw read cap — larger when a processor can
|
|
10
|
+
* shrink what was read; the 64 MiB bound is an OOM sanity bound, CC reads the file whole), `urlBytes` (a fetched
|
|
11
|
+
* image's body), `rejectEdge` (the longest edge the bytes that ship may state — `fI` (cli309.js:253044), the 2000 box,
|
|
12
|
+
* its no-flag reject width), `imageBase64Chars` (the provider's single-image limit, on the base64 length — CC `fI`
|
|
13
|
+
* `maxBase64Size`).
|
|
14
|
+
* - `soft`: what the processor aims at, best effort. `frameEdge` and `rawBytes` are the frame ladder's box and byte rung
|
|
15
|
+
* (CC `fI` `maxWidth` / `targetRawSize`: 3.75 MB raw = 5 MiB base64); `targetBytes` is the byte-target step's goal
|
|
16
|
+
* (`SZ` (cli309.js:253044), the bytes `N$o` compresses toward).
|
|
17
|
+
* The request's media budget is not a row of this table yet.
|
|
18
|
+
*/
|
|
19
|
+
export declare const IMAGE_ADMISSION: Readonly<{
|
|
20
|
+
readonly hard: Readonly<{
|
|
21
|
+
formats: Readonly<Record<"image/jpeg" | "image/png" | "image/gif" | "image/webp", (b: Uint8Array) => boolean>>;
|
|
22
|
+
readBytes: Readonly<{
|
|
23
|
+
withoutProcessor: number;
|
|
24
|
+
withProcessor: number;
|
|
25
|
+
}>;
|
|
26
|
+
urlBytes: number;
|
|
27
|
+
rejectEdge: 2000;
|
|
28
|
+
imageBase64Chars: number;
|
|
29
|
+
}>;
|
|
30
|
+
readonly soft: Readonly<{
|
|
31
|
+
frameEdge: 2000;
|
|
32
|
+
rawBytes: number;
|
|
33
|
+
targetBytes: 512000;
|
|
34
|
+
}>;
|
|
35
|
+
}>;
|
|
36
|
+
/** Verify the raw bytes actually ARE the image format the claimed type names (`IMAGE_ADMISSION.hard.formats`). Extension
|
|
43
37
|
* alone is forgeable: a `.png` holding text / an empty file / a truncated blob / an Apple-optimized PNG would otherwise
|
|
44
38
|
* become a malformed `ImageContent` that makes the NEXT provider request fail (400). Pure — no I/O. */
|
|
45
39
|
export declare function imageMagicMatches(bytes: Uint8Array, mimeType: string): boolean;
|
|
40
|
+
/**
|
|
41
|
+
* The pixel size an image's own header states, read without decoding (`WN` (cli309.js:257745)): PNG's `IHDR`, GIF's screen
|
|
42
|
+
* descriptor, a JPEG's first start-of-frame segment, WebP's `VP8 ` / `VP8L` / `VP8X` header. `undefined` when the header
|
|
43
|
+
* states none this reader finds. Pure — the admission reads it for the bytes that ship.
|
|
44
|
+
*/
|
|
45
|
+
export declare function readImageHeaderDimensions(b: Uint8Array): {
|
|
46
|
+
width: number;
|
|
47
|
+
height: number;
|
|
48
|
+
} | undefined;
|
|
46
49
|
/** True when the bytes start with the `%PDF-` magic. Magic outranks extension both ways: a mis-named
|
|
47
50
|
* `.pdf` without it is refused, and a non-`.pdf` file carrying it is read AS a PDF. */
|
|
48
51
|
export declare function pdfMagicMatches(bytes: Uint8Array): boolean;
|
|
49
52
|
/** The image types the content API accepts — the only ones a signature check can verify. */
|
|
50
53
|
export declare const ADMITTED_IMAGE_MIME_TYPES: readonly ["image/png", "image/jpeg", "image/gif", "image/webp"];
|
|
51
|
-
/** The raw-byte read cap for an image, with or without a
|
|
52
|
-
export declare function imageReadCap(
|
|
54
|
+
/** The raw-byte read cap for an image, with or without a processor (`IMAGE_ADMISSION.hard.readBytes`). */
|
|
55
|
+
export declare function imageReadCap(processor: ImageDownsampler | "none"): number;
|
|
53
56
|
export type ImageAdmission = {
|
|
54
57
|
ok: true;
|
|
55
58
|
block: ImageContent;
|
|
56
|
-
/** The raw byte count of the bytes checked (before any
|
|
59
|
+
/** The raw byte count of the bytes checked (before any processing). */
|
|
57
60
|
originalSize: number;
|
|
58
|
-
/** Measured by the
|
|
61
|
+
/** Measured by the processor; absent when the bytes went on as they are (never fabricated). */
|
|
59
62
|
dimensions?: ImageDimensions;
|
|
60
|
-
/** True when the
|
|
63
|
+
/** True when the processor actually changed the pixel size. */
|
|
61
64
|
resized: boolean;
|
|
62
65
|
} | {
|
|
63
66
|
ok: false;
|
|
64
67
|
reason: "mime_unsupported" | "magic_mismatch";
|
|
65
68
|
rawBytes: number;
|
|
69
|
+
}
|
|
70
|
+
/** The bytes that would ship state no pixel size in their header. */
|
|
71
|
+
| {
|
|
72
|
+
ok: false;
|
|
73
|
+
reason: "dimensions_unreadable";
|
|
74
|
+
rawBytes: number;
|
|
66
75
|
} | {
|
|
67
76
|
ok: false;
|
|
68
77
|
reason: "raw_too_large";
|
|
69
78
|
rawBytes: number;
|
|
70
79
|
cap: number;
|
|
71
80
|
}
|
|
81
|
+
/** The bytes that would ship state an edge over `IMAGE_ADMISSION.hard.rejectEdge` (`limit`). */
|
|
82
|
+
| {
|
|
83
|
+
ok: false;
|
|
84
|
+
reason: "dimensions_too_large";
|
|
85
|
+
rawBytes: number;
|
|
86
|
+
width: number;
|
|
87
|
+
height: number;
|
|
88
|
+
limit: number;
|
|
89
|
+
}
|
|
72
90
|
/** `base64` is the encoding that failed the limit (the caller words the refusal from its length). */
|
|
73
91
|
| {
|
|
74
92
|
ok: false;
|
|
@@ -76,7 +94,21 @@ export type ImageAdmission = {
|
|
|
76
94
|
rawBytes: number;
|
|
77
95
|
base64: string;
|
|
78
96
|
};
|
|
79
|
-
|
|
97
|
+
/** Why the bytes that would have shipped were refused on their stated size (`dimensions_unreadable` / `dimensions_too_large`)
|
|
98
|
+
* — the one wording every carrier's refusal and the tool-result seat's placeholder read. */
|
|
99
|
+
export declare function unprocessedSizeClause(a: {
|
|
100
|
+
reason: "dimensions_unreadable";
|
|
101
|
+
} | {
|
|
102
|
+
reason: "dimensions_too_large";
|
|
103
|
+
width: number;
|
|
104
|
+
height: number;
|
|
105
|
+
limit: number;
|
|
106
|
+
}): string;
|
|
107
|
+
/** One image's bytes through the admission (see the module head). `processor` is the run's, resolved from its one seat
|
|
108
|
+
* (`resolveImageProcessor`), or `"none"`: it is never optional, so no carrier admits without saying which. */
|
|
109
|
+
export declare function admitImageBytes(bytes: Uint8Array, claimedMime: string, processor: ImageDownsampler | "none"): Promise<ImageAdmission>;
|
|
110
|
+
/** The admitted type the bytes' own header names (the first of the four header rows they open with), or `undefined`. */
|
|
111
|
+
export declare function sniffImageMime(bytes: Uint8Array): (typeof ADMITTED_IMAGE_MIME_TYPES)[number] | undefined;
|
|
80
112
|
/** The document types the content API accepts — the one a signature check can verify. */
|
|
81
113
|
export declare const ADMITTED_DOCUMENT_MIME_TYPES: readonly ["application/pdf"];
|
|
82
114
|
export type DocumentAdmission = {
|
|
@@ -98,14 +130,17 @@ export type DocumentAdmission = {
|
|
|
98
130
|
export declare function admitDocumentBytes(bytes: Uint8Array, claimedMime: string): DocumentAdmission;
|
|
99
131
|
/** Why the tool-result seat refused a media block: the admissions' own words (an image's four — a document's are among
|
|
100
132
|
* them), or `undecodable` — the block carried no base64 string to decode. */
|
|
101
|
-
export type MediaRefusalReason = "undecodable" | "mime_unsupported" | "raw_too_large" | "magic_mismatch" | "base64_too_large";
|
|
133
|
+
export type MediaRefusalReason = "undecodable" | "mime_unsupported" | "raw_too_large" | "magic_mismatch" | "dimensions_unreadable" | "dimensions_too_large" | "base64_too_large";
|
|
102
134
|
/** One media block the seat refused: its type, the refusal, its decoded size, the bound it exceeded (the raw-byte cap for
|
|
103
|
-
* `raw_too_large`, the
|
|
135
|
+
* `raw_too_large`, the reject edge for `dimensions_too_large` — with the size its header states —, the base64 length
|
|
136
|
+
* limit for `base64_too_large`), and its media type when it can be one. */
|
|
104
137
|
export interface ToolResultMediaRefusal {
|
|
105
138
|
type: MediaBlock["type"];
|
|
106
139
|
reason: MediaRefusalReason;
|
|
107
140
|
bytes: number;
|
|
108
141
|
limit?: number;
|
|
142
|
+
width?: number;
|
|
143
|
+
height?: number;
|
|
109
144
|
mimeType?: string;
|
|
110
145
|
}
|
|
111
146
|
/** What the seat made of one result's content: every media block rebuilt from its checked bytes or replaced, in its slot,
|
|
@@ -115,9 +150,27 @@ export interface ToolResultMediaAdmission {
|
|
|
115
150
|
refused: ToolResultMediaRefusal[];
|
|
116
151
|
}
|
|
117
152
|
/**
|
|
118
|
-
* The
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
153
|
+
* The decoded size of the first image block that goes to the model over `IMAGE_ADMISSION.soft.targetBytes` with no
|
|
154
|
+
* processor to bring it down — the one fact the run's operator notice reads (`config.image_processor_unavailable`), at
|
|
155
|
+
* every place images enter a run (the tool-result seat, the task's images, a person's attachments). `undefined` with a
|
|
156
|
+
* processor, or when none does.
|
|
157
|
+
*/
|
|
158
|
+
export declare function firstUnprocessedOverTarget(blocks: readonly unknown[], processor: ImageDownsampler | "none"): number | undefined;
|
|
159
|
+
/**
|
|
160
|
+
* The tool-result seat's one read of an executed call's content: its blocks, read once into a new array, when one claims a
|
|
161
|
+
* media type; `undefined` otherwise (the content stands as it is). Synchronous and total — a content whose blocks cannot be
|
|
162
|
+
* read is not media this seat can admit or refuse. A carrier resolves the run's processor only once this answers blocks,
|
|
163
|
+
* so a call with no media in its result awaits nothing.
|
|
164
|
+
*/
|
|
165
|
+
export declare function toolResultMediaBlocks(content: unknown): unknown[] | undefined;
|
|
166
|
+
/**
|
|
167
|
+
* The tool-result seat's admission over blocks {@link toolResultMediaBlocks} read: each media block admitted with the run's
|
|
168
|
+
* `processor` — rebuilt from the bytes it checked (an image a carrier already brought within the target and the frame passes
|
|
169
|
+
* unchanged), or replaced in its slot by a placeholder naming `toolName` and the refusal.
|
|
170
|
+
*/
|
|
171
|
+
export declare function admitToolResultMediaBlocks(blocks: readonly unknown[], toolName: string, processor: ImageDownsampler | "none"): Promise<ToolResultMediaAdmission>;
|
|
172
|
+
/**
|
|
173
|
+
* The tool-result seat over one executed call's content with a processor in hand: {@link toolResultMediaBlocks}, then
|
|
174
|
+
* {@link admitToolResultMediaBlocks} — `undefined` when no block claims a media type (nothing is awaited).
|
|
122
175
|
*/
|
|
123
|
-
export declare function admitToolResultMedia(content: unknown, toolName: string): Promise<ToolResultMediaAdmission> | undefined;
|
|
176
|
+
export declare function admitToolResultMedia(content: unknown, toolName: string, processor: ImageDownsampler | "none"): Promise<ToolResultMediaAdmission> | undefined;
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
2
|
import { claimedMediaType } from "./media-byte-cap.js";
|
|
3
3
|
import { readMember } from "./thrown-value.js";
|
|
4
|
-
export const MAX_IMAGE_READ_BYTES = 5 * 1024 * 1024;
|
|
5
|
-
export const MAX_IMAGE_DOWNSAMPLE_INPUT_BYTES = 64 * 1024 * 1024;
|
|
6
4
|
export const PDF_TARGET_RAW_SIZE = 20 * 1024 * 1024;
|
|
7
5
|
const spells = (b, at, text) => {
|
|
8
6
|
if (b.length < at + text.length)
|
|
@@ -14,51 +12,156 @@ const spells = (b, at, text) => {
|
|
|
14
12
|
};
|
|
15
13
|
const u32be = (b, at) => ((b[at] << 24) >>> 0) + (b[at + 1] << 16) + (b[at + 2] << 8) + b[at + 3];
|
|
16
14
|
const u16le = (b, at) => b[at] + (b[at + 1] << 8);
|
|
17
|
-
const
|
|
15
|
+
const u16be = (b, at) => (b[at] << 8) + b[at + 1];
|
|
16
|
+
const u24le = (b, at) => b[at] + (b[at + 1] << 8) + (b[at + 2] << 16);
|
|
17
|
+
const u32le = (b, at) => (u24le(b, at) + (b[at + 3] << 24)) >>> 0;
|
|
18
|
+
const IMAGE_HEADER_CHECKS = Object.freeze({
|
|
18
19
|
"image/png": (b) => b.length >= 24 && spells(b, 0, "\x89PNG\r\n\x1a\n") && u32be(b, 8) === 13 && spells(b, 12, "IHDR") && u32be(b, 16) > 0 && u32be(b, 20) > 0,
|
|
19
20
|
"image/jpeg": (b) => b.length >= 4 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff && b[3] >= 0xc0,
|
|
20
21
|
"image/gif": (b) => b.length >= 13 && (spells(b, 0, "GIF87a") || spells(b, 0, "GIF89a")) && u16le(b, 6) > 0 && u16le(b, 8) > 0,
|
|
21
22
|
"image/webp": (b) => b.length >= 16 && spells(b, 0, "RIFF") && spells(b, 8, "WEBP") && (spells(b, 12, "VP8 ") || spells(b, 12, "VP8L") || spells(b, 12, "VP8X")),
|
|
22
|
-
};
|
|
23
|
+
});
|
|
24
|
+
const MiB = 1024 * 1024;
|
|
25
|
+
export const IMAGE_ADMISSION = Object.freeze({
|
|
26
|
+
hard: Object.freeze({
|
|
27
|
+
formats: IMAGE_HEADER_CHECKS,
|
|
28
|
+
readBytes: Object.freeze({ withoutProcessor: 5 * MiB, withProcessor: 64 * MiB }),
|
|
29
|
+
urlBytes: 20 * MiB,
|
|
30
|
+
rejectEdge: 2000,
|
|
31
|
+
imageBase64Chars: 5 * MiB,
|
|
32
|
+
}),
|
|
33
|
+
soft: Object.freeze({ frameEdge: 2000, rawBytes: (5 * MiB * 3) / 4, targetBytes: 512_000 }),
|
|
34
|
+
});
|
|
23
35
|
export function imageMagicMatches(bytes, mimeType) {
|
|
24
|
-
|
|
36
|
+
const formats = IMAGE_ADMISSION.hard.formats;
|
|
37
|
+
return Object.hasOwn(formats, mimeType) && formats[mimeType](bytes);
|
|
38
|
+
}
|
|
39
|
+
function jpegFrameAt(b) {
|
|
40
|
+
if (b.length < 4 || b[0] !== 0xff || b[1] !== 0xd8)
|
|
41
|
+
return undefined;
|
|
42
|
+
let at = 2;
|
|
43
|
+
while (at + 9 < b.length) {
|
|
44
|
+
if (b[at] !== 0xff) {
|
|
45
|
+
at++;
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const marker = b[at + 1];
|
|
49
|
+
if (marker === 0xff) {
|
|
50
|
+
at++;
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc)
|
|
54
|
+
return at;
|
|
55
|
+
if ((marker >= 0xd0 && marker <= 0xd9) || marker === 0x01) {
|
|
56
|
+
at += 2;
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const length = u16be(b, at + 2);
|
|
60
|
+
if (length < 2)
|
|
61
|
+
return undefined;
|
|
62
|
+
at += 2 + length;
|
|
63
|
+
}
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
export function readImageHeaderDimensions(b) {
|
|
67
|
+
if (b.length < 10)
|
|
68
|
+
return undefined;
|
|
69
|
+
if (b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47)
|
|
70
|
+
return b.length >= 24 ? { width: u32be(b, 16), height: u32be(b, 20) } : undefined;
|
|
71
|
+
if (b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46)
|
|
72
|
+
return { width: u16le(b, 6), height: u16le(b, 8) };
|
|
73
|
+
const frame = jpegFrameAt(b);
|
|
74
|
+
if (frame !== undefined)
|
|
75
|
+
return { height: u16be(b, frame + 5), width: u16be(b, frame + 7) };
|
|
76
|
+
if (b[0] === 0xff && b[1] === 0xd8)
|
|
77
|
+
return undefined;
|
|
78
|
+
if (b.length >= 30 && spells(b, 0, "RIFF") && spells(b, 8, "WEBP")) {
|
|
79
|
+
if (spells(b, 12, "VP8 "))
|
|
80
|
+
return { width: u16le(b, 26) & 0x3fff, height: u16le(b, 28) & 0x3fff };
|
|
81
|
+
if (spells(b, 12, "VP8L")) {
|
|
82
|
+
const bits = u32le(b, 21);
|
|
83
|
+
return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 };
|
|
84
|
+
}
|
|
85
|
+
if (spells(b, 12, "VP8X"))
|
|
86
|
+
return { width: u24le(b, 24) + 1, height: u24le(b, 27) + 1 };
|
|
87
|
+
}
|
|
88
|
+
return undefined;
|
|
25
89
|
}
|
|
26
90
|
export function pdfMagicMatches(bytes) {
|
|
27
91
|
return bytes.length >= 5 && bytes[0] === 0x25 && bytes[1] === 0x50 && bytes[2] === 0x44 && bytes[3] === 0x46 && bytes[4] === 0x2d;
|
|
28
92
|
}
|
|
29
93
|
export const ADMITTED_IMAGE_MIME_TYPES = Object.freeze(["image/png", "image/jpeg", "image/gif", "image/webp"]);
|
|
30
|
-
export function imageReadCap(
|
|
31
|
-
|
|
94
|
+
export function imageReadCap(processor) {
|
|
95
|
+
const { readBytes } = IMAGE_ADMISSION.hard;
|
|
96
|
+
return processor === "none" ? readBytes.withoutProcessor : readBytes.withProcessor;
|
|
97
|
+
}
|
|
98
|
+
export function unprocessedSizeClause(a) {
|
|
99
|
+
return a.reason === "dimensions_unreadable"
|
|
100
|
+
? "it could not be processed and its header states no pixel size — convert it to PNG, JPEG, GIF, or WebP"
|
|
101
|
+
: `at ${a.width}x${a.height}px it is over the ${a.limit}x${a.limit}px limit and could not be shrunk — resize it to fit within ${a.limit}x${a.limit}px`;
|
|
32
102
|
}
|
|
33
|
-
export async function admitImageBytes(bytes, claimedMime,
|
|
103
|
+
export async function admitImageBytes(bytes, claimedMime, processor) {
|
|
34
104
|
const rawBytes = bytes.byteLength;
|
|
35
105
|
if (!ADMITTED_IMAGE_MIME_TYPES.includes(claimedMime))
|
|
36
106
|
return { ok: false, reason: "mime_unsupported", rawBytes };
|
|
37
|
-
const cap = imageReadCap(
|
|
107
|
+
const cap = imageReadCap(processor);
|
|
38
108
|
if (rawBytes > cap)
|
|
39
109
|
return { ok: false, reason: "raw_too_large", rawBytes, cap };
|
|
40
110
|
if (!imageMagicMatches(bytes, claimedMime))
|
|
41
111
|
return { ok: false, reason: "magic_mismatch", rawBytes };
|
|
42
|
-
let
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
112
|
+
let shipped;
|
|
113
|
+
const memory = processor !== "none" ? producedBy(processor) : undefined;
|
|
114
|
+
const inputDigest = memory !== undefined ? digestOf(bytes) : undefined;
|
|
115
|
+
const known = memory !== undefined && inputDigest !== undefined && memory.has(inputDigest);
|
|
116
|
+
if (processor !== "none" && !known) {
|
|
117
|
+
const out = await Promise.resolve()
|
|
118
|
+
.then(() => processor(Buffer.from(bytes), claimedMime))
|
|
119
|
+
.catch(() => undefined);
|
|
120
|
+
const outBytes = out !== undefined ? Buffer.from(out.base64, "base64") : undefined;
|
|
121
|
+
const outMime = outBytes !== undefined ? sniffImageMime(outBytes) : undefined;
|
|
122
|
+
if (out !== undefined && outBytes !== undefined && outMime !== undefined)
|
|
123
|
+
shipped = { bytes: outBytes, base64: out.base64, mimeType: outMime, ...(out.dimensions !== undefined ? { dimensions: out.dimensions } : {}) };
|
|
52
124
|
}
|
|
53
|
-
|
|
54
|
-
|
|
125
|
+
const fromProcessor = shipped !== undefined;
|
|
126
|
+
shipped ??= { bytes, base64: Buffer.from(bytes).toString("base64"), mimeType: claimedMime };
|
|
127
|
+
const header = readImageHeaderDimensions(shipped.bytes);
|
|
128
|
+
if (header === undefined)
|
|
129
|
+
return { ok: false, reason: "dimensions_unreadable", rawBytes };
|
|
130
|
+
const limit = IMAGE_ADMISSION.hard.rejectEdge;
|
|
131
|
+
if (header.width > limit || header.height > limit)
|
|
132
|
+
return { ok: false, reason: "dimensions_too_large", rawBytes, width: header.width, height: header.height, limit };
|
|
133
|
+
const { base64, mimeType, dimensions } = shipped;
|
|
134
|
+
if (base64.length > IMAGE_ADMISSION.hard.imageBase64Chars)
|
|
55
135
|
return { ok: false, reason: "base64_too_large", rawBytes, base64 };
|
|
56
136
|
const resized = dimensions !== undefined &&
|
|
57
137
|
dimensions.displayWidth > 0 &&
|
|
58
138
|
dimensions.displayHeight > 0 &&
|
|
59
139
|
(dimensions.originalWidth !== dimensions.displayWidth || dimensions.originalHeight !== dimensions.displayHeight);
|
|
140
|
+
if (memory !== undefined && (fromProcessor || known))
|
|
141
|
+
memory.add(fromProcessor ? digestOf(shipped.bytes) : inputDigest);
|
|
60
142
|
return { ok: true, block: { type: "image", data: base64, mimeType }, originalSize: rawBytes, ...(dimensions !== undefined ? { dimensions } : {}), resized };
|
|
61
143
|
}
|
|
144
|
+
const PRODUCED_KEPT = 256;
|
|
145
|
+
const produced = new WeakMap();
|
|
146
|
+
function producedBy(processor) {
|
|
147
|
+
let kept = produced.get(processor);
|
|
148
|
+
if (kept === undefined)
|
|
149
|
+
produced.set(processor, (kept = new Set()));
|
|
150
|
+
const set = kept;
|
|
151
|
+
return {
|
|
152
|
+
has: (digest) => set.has(digest),
|
|
153
|
+
add: (digest) => {
|
|
154
|
+
set.delete(digest);
|
|
155
|
+
set.add(digest);
|
|
156
|
+
if (set.size > PRODUCED_KEPT)
|
|
157
|
+
set.delete(set.values().next().value);
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
const digestOf = (bytes) => createHash("sha256").update(bytes).digest("base64");
|
|
162
|
+
export function sniffImageMime(bytes) {
|
|
163
|
+
return ADMITTED_IMAGE_MIME_TYPES.find((mime) => imageMagicMatches(bytes, mime));
|
|
164
|
+
}
|
|
62
165
|
export const ADMITTED_DOCUMENT_MIME_TYPES = Object.freeze(["application/pdf"]);
|
|
63
166
|
export function admitDocumentBytes(bytes, claimedMime) {
|
|
64
167
|
const rawBytes = bytes.byteLength;
|
|
@@ -77,6 +180,8 @@ const REFUSAL_CLAUSE = {
|
|
|
77
180
|
mime_unsupported: (r) => `${r.mimeType ?? "its media type"} is not a ${r.type} type the model accepts (${ADMITTED_FOR[r.type].join(", ")})`,
|
|
78
181
|
raw_too_large: (r) => `its ${r.bytes} bytes exceed the ${r.limit}-byte cap`,
|
|
79
182
|
magic_mismatch: (r) => `its ${r.bytes} bytes do not open with a valid ${r.mimeType ?? `${r.type}`} header`,
|
|
183
|
+
dimensions_unreadable: () => unprocessedSizeClause({ reason: "dimensions_unreadable" }),
|
|
184
|
+
dimensions_too_large: (r) => unprocessedSizeClause({ reason: "dimensions_too_large", width: r.width, height: r.height, limit: r.limit }),
|
|
80
185
|
base64_too_large: (r) => `its base64 encoding exceeds the ${r.limit}-character provider limit (${r.bytes} bytes)`,
|
|
81
186
|
};
|
|
82
187
|
const placeholderOf = (toolName, r) => ({
|
|
@@ -84,15 +189,15 @@ const placeholderOf = (toolName, r) => ({
|
|
|
84
189
|
text: `[${r.type} from the ${JSON.stringify(toolName)} tool not shown: ${REFUSAL_CLAUSE[r.reason](r)} (media admission: ${r.reason})]`,
|
|
85
190
|
});
|
|
86
191
|
const claimsMedia = (block) => claimedMediaType(block) !== undefined;
|
|
87
|
-
async function admitBlock(block) {
|
|
192
|
+
async function admitBlock(block, processor) {
|
|
88
193
|
const type = claimedMediaType(block);
|
|
89
194
|
if (type === undefined)
|
|
90
195
|
return { block };
|
|
91
196
|
const data = readMember(block, "data");
|
|
92
197
|
const mime = readMember(block, "mimeType");
|
|
93
198
|
const echo = mediaTypeEcho(mime);
|
|
94
|
-
const refusal = (reason, bytes, limit) => ({
|
|
95
|
-
refusal: { type, reason, bytes, ...(limit !== undefined ? { limit } : {}), ...(echo !== undefined ? { mimeType: echo } : {}) },
|
|
199
|
+
const refusal = (reason, bytes, limit, size) => ({
|
|
200
|
+
refusal: { type, reason, bytes, ...(limit !== undefined ? { limit } : {}), ...(size ?? {}), ...(echo !== undefined ? { mimeType: echo } : {}) },
|
|
96
201
|
});
|
|
97
202
|
if (typeof data !== "string")
|
|
98
203
|
return refusal("undecodable", 0);
|
|
@@ -100,7 +205,7 @@ async function admitBlock(block) {
|
|
|
100
205
|
try {
|
|
101
206
|
const bytes = Buffer.from(data, "base64");
|
|
102
207
|
const claimed = typeof mime === "string" ? mime : "";
|
|
103
|
-
a = type === "image" ? await admitImageBytes(bytes, claimed,
|
|
208
|
+
a = type === "image" ? await admitImageBytes(bytes, claimed, processor) : admitDocumentBytes(bytes, claimed);
|
|
104
209
|
}
|
|
105
210
|
catch {
|
|
106
211
|
return refusal("undecodable", 0);
|
|
@@ -109,35 +214,54 @@ async function admitBlock(block) {
|
|
|
109
214
|
return { block: a.block };
|
|
110
215
|
if (a.reason === "raw_too_large")
|
|
111
216
|
return refusal(a.reason, a.rawBytes, a.cap);
|
|
217
|
+
if (a.reason === "dimensions_too_large")
|
|
218
|
+
return refusal(a.reason, a.rawBytes, a.limit, { width: a.width, height: a.height });
|
|
112
219
|
if (a.reason === "base64_too_large")
|
|
113
|
-
return refusal(a.reason, a.rawBytes,
|
|
220
|
+
return refusal(a.reason, a.rawBytes, IMAGE_ADMISSION.hard.imageBase64Chars);
|
|
114
221
|
return refusal(a.reason, a.rawBytes);
|
|
115
222
|
}
|
|
116
|
-
export function
|
|
117
|
-
|
|
223
|
+
export function firstUnprocessedOverTarget(blocks, processor) {
|
|
224
|
+
if (processor !== "none")
|
|
225
|
+
return undefined;
|
|
226
|
+
for (const b of blocks) {
|
|
227
|
+
if (claimedMediaType(b) !== "image")
|
|
228
|
+
continue;
|
|
229
|
+
const data = readMember(b, "data");
|
|
230
|
+
if (typeof data !== "string")
|
|
231
|
+
continue;
|
|
232
|
+
const bytes = Buffer.byteLength(data, "base64");
|
|
233
|
+
if (bytes > IMAGE_ADMISSION.soft.targetBytes)
|
|
234
|
+
return bytes;
|
|
235
|
+
}
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
238
|
+
export function toolResultMediaBlocks(content) {
|
|
118
239
|
try {
|
|
119
240
|
if (!Array.isArray(content))
|
|
120
241
|
return undefined;
|
|
121
|
-
blocks = Array.from(content);
|
|
122
|
-
|
|
123
|
-
return undefined;
|
|
242
|
+
const blocks = Array.from(content);
|
|
243
|
+
return blocks.some(claimsMedia) ? blocks : undefined;
|
|
124
244
|
}
|
|
125
245
|
catch {
|
|
126
246
|
return undefined;
|
|
127
247
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
else {
|
|
137
|
-
refused.push(r.refusal);
|
|
138
|
-
out.push(placeholderOf(toolName, r.refusal));
|
|
139
|
-
}
|
|
248
|
+
}
|
|
249
|
+
export async function admitToolResultMediaBlocks(blocks, toolName, processor) {
|
|
250
|
+
const refused = [];
|
|
251
|
+
const out = [];
|
|
252
|
+
for (const b of blocks) {
|
|
253
|
+
const r = await admitBlock(b, processor);
|
|
254
|
+
if ("block" in r) {
|
|
255
|
+
out.push(r.block);
|
|
140
256
|
}
|
|
141
|
-
|
|
142
|
-
|
|
257
|
+
else {
|
|
258
|
+
refused.push(r.refusal);
|
|
259
|
+
out.push(placeholderOf(toolName, r.refusal));
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return { content: out, refused };
|
|
263
|
+
}
|
|
264
|
+
export function admitToolResultMedia(content, toolName, processor) {
|
|
265
|
+
const blocks = toolResultMediaBlocks(content);
|
|
266
|
+
return blocks === undefined ? undefined : admitToolResultMediaBlocks(blocks, toolName, processor);
|
|
143
267
|
}
|