@agent-native/core 0.84.0 → 0.84.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +13 -0
- package/corpus/core/docs/content/evals.mdx +34 -12
- package/corpus/core/docs/content/locales/ar-SA/evals.mdx +7 -0
- package/corpus/core/docs/content/locales/de-DE/evals.mdx +8 -0
- package/corpus/core/docs/content/locales/es-ES/evals.mdx +8 -0
- package/corpus/core/docs/content/locales/fr-FR/evals.mdx +8 -0
- package/corpus/core/docs/content/locales/hi-IN/evals.mdx +8 -0
- package/corpus/core/docs/content/locales/ja-JP/evals.mdx +7 -0
- package/corpus/core/docs/content/locales/ko-KR/evals.mdx +7 -0
- package/corpus/core/docs/content/locales/pt-BR/evals.mdx +8 -0
- package/corpus/core/docs/content/locales/zh-CN/evals.mdx +7 -0
- package/corpus/core/docs/content/locales/zh-TW/evals.mdx +7 -0
- package/corpus/core/docs/design/durable-agent-runs.md +34 -35
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/engine/builder-engine.ts +81 -9
- package/corpus/core/src/agent/run-manager.ts +8 -7
- package/corpus/core/src/cli/design-connect.ts +175 -0
- package/corpus/core/src/cli/skills.ts +5 -7
- package/corpus/core/src/client/AgentPanel.tsx +8 -0
- package/corpus/core/src/client/MultiTabAssistantChat.tsx +4 -1
- package/corpus/core/src/client/agent-chat-adapter.ts +45 -15
- package/corpus/core/src/client/sse-event-processor.ts +31 -4
- package/corpus/core/src/eval/define-eval.ts +4 -1
- package/corpus/core/src/eval/report.ts +23 -8
- package/corpus/core/src/eval/runner.ts +41 -6
- package/corpus/core/src/eval/types.ts +11 -0
- package/corpus/core/src/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +15 -17
- package/corpus/templates/content/AGENTS.md +9 -2
- package/corpus/templates/content/actions/_database-row-batch.ts +230 -0
- package/corpus/templates/content/actions/_database-utils.ts +17 -9
- package/corpus/templates/content/actions/_notion-action-utils.ts +26 -0
- package/corpus/templates/content/actions/connect-notion-status.ts +13 -4
- package/corpus/templates/content/actions/create-and-link-notion-page.ts +23 -0
- package/corpus/templates/content/actions/delete-database-items.ts +55 -0
- package/corpus/templates/content/actions/delete-document.ts +4 -4
- package/corpus/templates/content/actions/disconnect-notion.ts +16 -0
- package/corpus/templates/content/actions/duplicate-database-item.ts +1 -1
- package/corpus/templates/content/actions/duplicate-database-items.ts +191 -0
- package/corpus/templates/content/actions/link-notion-page.ts +10 -7
- package/corpus/templates/content/actions/list-notion-links.ts +3 -4
- package/corpus/templates/content/actions/pull-notion-page.ts +7 -9
- package/corpus/templates/content/actions/push-notion-page.ts +7 -9
- package/corpus/templates/content/actions/refresh-notion-sync-status.ts +24 -0
- package/corpus/templates/content/actions/resolve-notion-sync-conflict.ts +23 -0
- package/corpus/templates/content/actions/search-notion-pages.ts +67 -0
- package/corpus/templates/content/actions/unlink-notion-page.ts +23 -0
- package/corpus/templates/content/app/components/editor/DocumentDatabase.tsx +27 -34
- package/corpus/templates/content/app/components/editor/DocumentEditor.tsx +16 -13
- package/corpus/templates/content/app/components/editor/DocumentToolbar.tsx +10 -10
- package/corpus/templates/content/app/components/editor/NotionConflictBanner.tsx +1 -1
- package/corpus/templates/content/app/components/editor/NotionSyncBar.tsx +18 -9
- package/corpus/templates/content/app/components/editor/database/DatabaseView.tsx +27 -34
- package/corpus/templates/content/app/components/editor/database/navigation-state.ts +10 -4
- package/corpus/templates/content/app/components/sidebar/NotionButton.tsx +32 -17
- package/corpus/templates/content/app/hooks/use-content-database.ts +35 -0
- package/corpus/templates/content/app/hooks/use-notion.ts +108 -99
- package/corpus/templates/content/package.json +3 -0
- package/corpus/templates/content/parity/README.md +69 -0
- package/corpus/templates/content/parity/eval-scenarios.ts +118 -0
- package/corpus/templates/content/parity/exceptions.allowlist.ts +17 -0
- package/corpus/templates/content/parity/matrix.md +28 -0
- package/corpus/templates/content/parity/matrix.ts +606 -0
- package/corpus/templates/content/parity/matrix.types.ts +42 -0
- package/corpus/templates/content/parity/parity-evals.eval.ts +4 -0
- package/corpus/templates/content/parity/render-matrix.ts +90 -0
- package/corpus/templates/content/parity/scenario-to-eval.ts +57 -0
- package/corpus/templates/content/shared/api.ts +11 -0
- package/corpus/templates/design/actions/apply-motion-edit.ts +50 -18
- package/corpus/templates/design/actions/get-motion-timeline.ts +4 -14
- package/corpus/templates/design/app/components/design/DesignCanvas.tsx +156 -12
- package/corpus/templates/design/app/components/design/MotionDock.tsx +2 -3
- package/corpus/templates/design/app/components/design/MultiScreenCanvas.tsx +217 -75
- package/corpus/templates/design/app/components/design/QuestionFlow.tsx +39 -39
- package/corpus/templates/design/app/components/design/bridge/editor-chrome.bridge.ts +558 -37
- package/corpus/templates/design/app/components/design/bridge/hit-test.bridge.ts +75 -3
- package/corpus/templates/design/app/components/design/types.ts +13 -0
- package/corpus/templates/design/app/components/layout/Layout.tsx +1 -0
- package/corpus/templates/design/app/i18n/zh-TW.ts +17 -0
- package/corpus/templates/design/app/i18n-data.ts +245 -0
- package/corpus/templates/design/app/pages/DesignEditor.tsx +762 -153
- package/corpus/templates/design/changelog/2026-06-30-copying-or-dragging-screen-elements-onto-the-infinite-canvas.md +6 -0
- package/corpus/templates/design/changelog/2026-06-30-design-questions-now-use-tighter-editor-typography-and-contr.md +6 -0
- package/corpus/templates/design/changelog/2026-06-30-element-drags-can-be-cancelled-with-escape-before-they-commi.md +6 -0
- package/corpus/templates/design/changelog/2026-06-30-pending-visual-style-edits-now-warn-before-you-leave-the-edi.md +6 -0
- package/corpus/templates/design/changelog/2026-06-30-visual-style-drags-stay-live-while-pending-edits-can-be-appl.md +6 -0
- package/corpus/templates/design/changelog/2026-07-01-design-chat-no-longer-shows-a-redundant-context-tab-above-the-composer.md +6 -0
- package/corpus/templates/design/changelog/2026-07-01-motion-track-creation-no-longer-fails-in-local-editors.md +6 -0
- package/corpus/templates/design/server/plugins/db.ts +10 -0
- package/dist/agent/engine/builder-engine.d.ts.map +1 -1
- package/dist/agent/engine/builder-engine.js +61 -10
- package/dist/agent/engine/builder-engine.js.map +1 -1
- package/dist/agent/run-manager.d.ts +8 -7
- package/dist/agent/run-manager.d.ts.map +1 -1
- package/dist/agent/run-manager.js +8 -7
- package/dist/agent/run-manager.js.map +1 -1
- package/dist/cli/design-connect.d.ts +2 -0
- package/dist/cli/design-connect.d.ts.map +1 -1
- package/dist/cli/design-connect.js +140 -0
- package/dist/cli/design-connect.js.map +1 -1
- package/dist/cli/skills.d.ts.map +1 -1
- package/dist/cli/skills.js +5 -7
- package/dist/cli/skills.js.map +1 -1
- package/dist/client/AgentPanel.d.ts +5 -1
- package/dist/client/AgentPanel.d.ts.map +1 -1
- package/dist/client/AgentPanel.js +4 -4
- package/dist/client/AgentPanel.js.map +1 -1
- package/dist/client/MultiTabAssistantChat.d.ts +3 -1
- package/dist/client/MultiTabAssistantChat.d.ts.map +1 -1
- package/dist/client/MultiTabAssistantChat.js +2 -2
- package/dist/client/MultiTabAssistantChat.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +37 -14
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/client/sse-event-processor.d.ts +14 -1
- package/dist/client/sse-event-processor.d.ts.map +1 -1
- package/dist/client/sse-event-processor.js +21 -5
- package/dist/client/sse-event-processor.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/eval/define-eval.d.ts.map +1 -1
- package/dist/eval/define-eval.js +2 -1
- package/dist/eval/define-eval.js.map +1 -1
- package/dist/eval/report.d.ts.map +1 -1
- package/dist/eval/report.js +19 -4
- package/dist/eval/report.js.map +1 -1
- package/dist/eval/runner.d.ts.map +1 -1
- package/dist/eval/runner.js +38 -6
- package/dist/eval/runner.js.map +1 -1
- package/dist/eval/types.d.ts +11 -0
- package/dist/eval/types.d.ts.map +1 -1
- package/dist/eval/types.js.map +1 -1
- package/dist/file-upload/actions/upload-image.d.ts +2 -2
- package/dist/observability/routes.d.ts +2 -2
- package/dist/progress/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +3 -3
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/dist/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +15 -17
- package/docs/content/evals.mdx +34 -12
- package/docs/content/locales/ar-SA/evals.mdx +7 -0
- package/docs/content/locales/de-DE/evals.mdx +8 -0
- package/docs/content/locales/es-ES/evals.mdx +8 -0
- package/docs/content/locales/fr-FR/evals.mdx +8 -0
- package/docs/content/locales/hi-IN/evals.mdx +8 -0
- package/docs/content/locales/ja-JP/evals.mdx +7 -0
- package/docs/content/locales/ko-KR/evals.mdx +7 -0
- package/docs/content/locales/pt-BR/evals.mdx +8 -0
- package/docs/content/locales/zh-CN/evals.mdx +7 -0
- package/docs/content/locales/zh-TW/evals.mdx +7 -0
- package/docs/design/durable-agent-runs.md +34 -35
- package/package.json +1 -1
- package/src/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md +15 -17
- package/corpus/templates/content/server/routes/api/notion/disconnect.post.ts +0 -12
package/corpus/README.md
CHANGED
package/corpus/core/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# @agent-native/core
|
|
2
2
|
|
|
3
|
+
## 0.84.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 3ff4f55: Extend Builder gateway timeouts for local and background agent-chat runs, and surface recoverable gateway timeout failures instead of silently retrying activity-only tool preparation loops.
|
|
8
|
+
- 3ff4f55: Allow scoped chat surfaces to hide the composer scope badge while keeping thread context attached.
|
|
9
|
+
|
|
10
|
+
## 0.84.1
|
|
11
|
+
|
|
12
|
+
### Patch Changes
|
|
13
|
+
|
|
14
|
+
- 87806ef: Report intentionally gated eval cases as skipped, keep skipped suites from setting up agent runners, and avoid action/engine setup for eval cases that fully short-circuit with custom run handlers.
|
|
15
|
+
|
|
3
16
|
## 0.84.0
|
|
4
17
|
|
|
5
18
|
### Minor Changes
|
|
@@ -90,13 +90,14 @@ export default defineEval({
|
|
|
90
90
|
|
|
91
91
|
An eval passes only when **every** scorer meets the threshold. Key `defineEval` fields:
|
|
92
92
|
|
|
93
|
-
| Field
|
|
94
|
-
|
|
|
95
|
-
| `name`
|
|
96
|
-
| `input`
|
|
97
|
-
| `scorers`
|
|
98
|
-
| `threshold`
|
|
99
|
-
| `run`
|
|
93
|
+
| Field | Type | Notes |
|
|
94
|
+
| ------------ | --------------------- | --------------------------------------------------------------------------------------------------------------- |
|
|
95
|
+
| `name` | string | Required. Shown in the report. |
|
|
96
|
+
| `input` | `{ prompt, history }` | Required `prompt`; optional prior `{ role, text }` turns. |
|
|
97
|
+
| `scorers` | `Scorer[]` | Required, at least one, unless `skipReason` is set. |
|
|
98
|
+
| `threshold` | number `0..1` | Per-scorer pass bar. Default `0.5`; overridable from the CLI. |
|
|
99
|
+
| `run` | function | Optional override for custom setup (seed data, multi-turn). |
|
|
100
|
+
| `skipReason` | string | Optional reason to skip this eval without running the agent; useful for manual/nightly gates that need secrets. |
|
|
100
101
|
|
|
101
102
|
The agent run handed to scorers is small and transport-agnostic:
|
|
102
103
|
|
|
@@ -225,11 +226,32 @@ The command discovers `**/*.eval.ts` and `evals/*.ts` under the current app, run
|
|
|
225
226
|
|
|
226
227
|
Exit codes:
|
|
227
228
|
|
|
228
|
-
| Code | Meaning
|
|
229
|
-
| ---- |
|
|
230
|
-
| `0` | All evals passed
|
|
231
|
-
| `1` | At least one eval scored below threshold, or the suite errored.
|
|
232
|
-
| `2` | Bad arguments (e.g. `--threshold` outside `[0, 1]`).
|
|
229
|
+
| Code | Meaning |
|
|
230
|
+
| ---- | -------------------------------------------------------------------------------- |
|
|
231
|
+
| `0` | All evals passed, evals were skipped, or no eval files were found (CI-friendly). |
|
|
232
|
+
| `1` | At least one eval scored below threshold, or the suite errored. |
|
|
233
|
+
| `2` | Bad arguments (e.g. `--threshold` outside `[0, 1]`). |
|
|
234
|
+
|
|
235
|
+
### Skipping gated evals {#skipping}
|
|
236
|
+
|
|
237
|
+
Use `skipReason` when an eval suite is intentionally gated behind manual,
|
|
238
|
+
nightly, or secret-backed execution:
|
|
239
|
+
|
|
240
|
+
```ts
|
|
241
|
+
export default defineEval({
|
|
242
|
+
name: "provider-backed source sync",
|
|
243
|
+
input: { prompt: "Refresh the source and summarize the changes." },
|
|
244
|
+
skipReason: process.env.SOURCE_SYNC_EVALS
|
|
245
|
+
? undefined
|
|
246
|
+
: "Skipped because SOURCE_SYNC_EVALS is unset",
|
|
247
|
+
scorers: process.env.SOURCE_SYNC_EVALS ? [contains("updated")] : [],
|
|
248
|
+
});
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Skipped evals do not run the agent or scorers, do not fail the suite, and are
|
|
252
|
+
reported distinctly with `status: "skipped"` and `skipReason` in JSON output.
|
|
253
|
+
Use this for expensive or credential-dependent evals; keep cheap deterministic
|
|
254
|
+
coverage in normal PR CI.
|
|
233
255
|
|
|
234
256
|
### As a CI deploy gate {#ci}
|
|
235
257
|
|
|
@@ -231,6 +231,13 @@ agent-native eval --threshold 0.8 # override every eval's pass threshold (0..
|
|
|
231
231
|
| `1` | تم تسجيل تقييم واحد على الأقل أقل من الحد الأدنى، أو حدث خطأ في المجموعة. |
|
|
232
232
|
| `2` | وسيطات غير صالحة (على سبيل المثال، `--threshold` خارج `[0, 1]`). |
|
|
233
233
|
|
|
234
|
+
### تخطي التقييمات المحكومة {#skipping}
|
|
235
|
+
|
|
236
|
+
استخدم `skipReason` عندما تكون مجموعة التقييمات مخصصة عمدا للتشغيل اليدوي أو
|
|
237
|
+
الليلي أو المعتمد على الأسرار. لا تشغل التقييمات المتخطاة الوكيل أو أدوات
|
|
238
|
+
التسجيل، ولا تفشل المجموعة، وتظهر في JSON مع `status: "skipped"` و
|
|
239
|
+
`skipReason`. أبق التغطية الحتمية الرخيصة في CI العادي لطلبات السحب.
|
|
240
|
+
|
|
234
241
|
### كبوابة نشر CI {#ci}
|
|
235
242
|
|
|
236
243
|
قم بإضافته إلى المسار الذي يتم تشغيله قبل النشر:
|
|
@@ -231,6 +231,14 @@ Exit-Codes:
|
|
|
231
231
|
| `1` | Mindestens eine Bewertung lag unter dem Schwellenwert oder die Suite ist fehlerhaft. |
|
|
232
232
|
| `2` | Ungültige Argumente (z. B. `--threshold` außerhalb von `[0, 1]`). |
|
|
233
233
|
|
|
234
|
+
### Gated Evals überspringen {#skipping}
|
|
235
|
+
|
|
236
|
+
Verwenden Sie `skipReason`, wenn eine Eval-Suite absichtlich nur manuell,
|
|
237
|
+
nachts oder mit Secrets ausgeführt werden soll. Übersprungene Evals führen
|
|
238
|
+
weder Agent noch Scorer aus, lassen die Suite nicht fehlschlagen und erscheinen
|
|
239
|
+
in der JSON-Ausgabe mit `status: "skipped"` und `skipReason`. Günstige
|
|
240
|
+
deterministische Abdeckung bleibt im normalen PR-CI.
|
|
241
|
+
|
|
234
242
|
### Als CI-Deploy-Gate {#ci}
|
|
235
243
|
|
|
236
244
|
Fügen Sie es der Pipeline hinzu, die vor einer Bereitstellung ausgeführt wird:
|
|
@@ -231,6 +231,14 @@ Códigos de salida:
|
|
|
231
231
|
| `1` | Al menos una evaluación obtuvo una puntuación inferior al umbral o la suite tuvo un error. |
|
|
232
232
|
| `2` | Malos argumentos (por ejemplo, `--threshold` fuera de `[0, 1]`). |
|
|
233
233
|
|
|
234
|
+
### Omitir evaluaciones gated {#skipping}
|
|
235
|
+
|
|
236
|
+
Use `skipReason` cuando una suite de evaluaciones esté reservada
|
|
237
|
+
intencionalmente para ejecución manual, nocturna o con secretos. Las
|
|
238
|
+
evaluaciones omitidas no ejecutan el agente ni los scorers, no hacen fallar la
|
|
239
|
+
suite y aparecen en JSON con `status: "skipped"` y `skipReason`. Mantenga la
|
|
240
|
+
cobertura determinista barata en el CI normal de PR.
|
|
241
|
+
|
|
234
242
|
### Como puerta de implementación de CI {#ci}
|
|
235
243
|
|
|
236
244
|
Agréguelo al proceso que se ejecuta antes de una implementación:
|
|
@@ -231,6 +231,14 @@ Codes de sortie :
|
|
|
231
231
|
| `1` | Au moins une évaluation a obtenu un score inférieur au seuil, ou la suite a commis une erreur. |
|
|
232
232
|
| `2` | Mauvais arguments (par exemple `--threshold` en dehors de `[0, 1]`). |
|
|
233
233
|
|
|
234
|
+
### Ignorer les évaluations gated {#skipping}
|
|
235
|
+
|
|
236
|
+
Utilisez `skipReason` lorsqu'une suite d'évaluations est volontairement réservée
|
|
237
|
+
à une exécution manuelle, nocturne ou dépendante de secrets. Les évaluations
|
|
238
|
+
ignorées n'exécutent ni l'agent ni les scorers, ne font pas échouer la suite, et
|
|
239
|
+
apparaissent dans la sortie JSON avec `status: "skipped"` et `skipReason`.
|
|
240
|
+
Gardez la couverture déterministe peu coûteuse dans le CI normal des PR.
|
|
241
|
+
|
|
234
242
|
### En tant que porte de déploiement CI {#ci}
|
|
235
243
|
|
|
236
244
|
Ajoutez-le au pipeline qui s'exécute avant un déploiement :
|
|
@@ -229,6 +229,14 @@ agent-native eval --threshold 0.8 # override every eval's pass threshold (0..
|
|
|
229
229
|
| `1` | कम से कम एक ईवल ने सीमा से नीचे स्कोर किया, या सुइट में त्रुटि हुई। |
|
|
230
230
|
| `2` | खराब तर्क (उदाहरण के लिए `[0, 1]` के बाहर `--threshold`)। |
|
|
231
231
|
|
|
232
|
+
### gated evals छोड़ना {#skipping}
|
|
233
|
+
|
|
234
|
+
जब कोई eval suite केवल manual, nightly, या secret-backed execution के लिए हो,
|
|
235
|
+
तो `skipReason` का उपयोग करें। छोड़े गए evals agent या scorers नहीं चलाते,
|
|
236
|
+
suite को fail नहीं करते, और JSON output में `status: "skipped"` और
|
|
237
|
+
`skipReason` के साथ दिखते हैं। सस्ती deterministic coverage को सामान्य PR CI
|
|
238
|
+
में रखें।
|
|
239
|
+
|
|
232
240
|
### सीआई परिनियोजन गेट के रूप में {#ci}
|
|
233
241
|
|
|
234
242
|
इसे उस पाइपलाइन में जोड़ें जो परिनियोजन से पहले चलती है:
|
|
@@ -231,6 +231,13 @@ agent-native eval --threshold 0.8 # override every eval's pass threshold (0..
|
|
|
231
231
|
| `1` | 少なくとも 1 つの評価スコアがしきい値を下回ったか、スイートでエラーが発生しました。 |
|
|
232
232
|
| `2` | 不正な引数 (例: `[0, 1]` の外の `--threshold`)。 |
|
|
233
233
|
|
|
234
|
+
### gated eval をスキップする {#skipping}
|
|
235
|
+
|
|
236
|
+
手動、夜間、または secret が必要な eval スイートには `skipReason` を使い
|
|
237
|
+
ます。スキップされた eval はエージェントや scorer を実行せず、スイート
|
|
238
|
+
を失敗させず、JSON 出力では `status: "skipped"` と `skipReason` とし
|
|
239
|
+
て表示されます。安価で決定的なカバレッジは通常の PR CI に残します。
|
|
240
|
+
|
|
234
241
|
### CI デプロイゲートとして {#ci}
|
|
235
242
|
|
|
236
243
|
デプロイ前に実行されるパイプラインに追加します。
|
|
@@ -231,6 +231,13 @@ agent-native eval --threshold 0.8 # override every eval's pass threshold (0..
|
|
|
231
231
|
| `1` | 최소 하나의 평가가 임계값 미만의 점수를 얻었거나 제품군에 오류가 발생했습니다. |
|
|
232
232
|
| `2` | 잘못된 인수(예: `[0, 1]` 외부의 `--threshold`). |
|
|
233
233
|
|
|
234
|
+
### 게이트된 평가 건너뛰기 {#skipping}
|
|
235
|
+
|
|
236
|
+
수동, 야간 또는 시크릿이 필요한 평가 스위트에는 `skipReason`를 사용하세요.
|
|
237
|
+
건너뛴 평가는 에이전트나 scorer를 실행하지 않고, 스위트를 실패시키지
|
|
238
|
+
않으며, JSON 출력에 `status: "skipped"` 및 `skipReason`로 표시됩니다.
|
|
239
|
+
저렴한 결정적 커버리지는 일반 PR CI에 유지하세요.
|
|
240
|
+
|
|
234
241
|
### CI 배포 게이트로 {#ci}
|
|
235
242
|
|
|
236
243
|
배포 전에 실행되는 파이프라인에 추가하세요:
|
|
@@ -231,6 +231,14 @@ Códigos de saída:
|
|
|
231
231
|
| `1` | Pelo menos uma avaliação com pontuação abaixo do limite ou o conjunto apresentou erro. |
|
|
232
232
|
| `2` | Argumentos incorretos (por exemplo, `--threshold` fora de `[0, 1]`). |
|
|
233
233
|
|
|
234
|
+
### Ignorar evals gated {#skipping}
|
|
235
|
+
|
|
236
|
+
Use `skipReason` quando uma suíte de evals for intencionalmente reservada para
|
|
237
|
+
execução manual, noturna ou dependente de secrets. Evals ignorados não executam
|
|
238
|
+
o agente nem os scorers, não fazem a suíte falhar e aparecem no JSON com
|
|
239
|
+
`status: "skipped"` e `skipReason`. Mantenha cobertura determinística barata no
|
|
240
|
+
CI normal de PR.
|
|
241
|
+
|
|
234
242
|
### Como porta de implantação de CI {#ci}
|
|
235
243
|
|
|
236
244
|
Adicione-o ao pipeline executado antes da implantação:
|
|
@@ -229,6 +229,13 @@ agent-native eval --threshold 0.8 # override every eval's pass threshold (0..
|
|
|
229
229
|
| `1` | 至少有一项评估得分低于阈值,或者套件出错。 |
|
|
230
230
|
| `2` | 错误的参数(例如 `--threshold` 位于 `[0, 1]` 之外)。 |
|
|
231
231
|
|
|
232
|
+
### 跳过受控评估 {#skipping}
|
|
233
|
+
|
|
234
|
+
当某个评估套件只应在手动、夜间或具备密钥的环境中运行时,请使用
|
|
235
|
+
`skipReason`。被跳过的评估不会运行代理或 scorer,不会让套件失败,并会
|
|
236
|
+
在 JSON 输出中显示 `status: "skipped"` 和 `skipReason`。廉价的确定性覆
|
|
237
|
+
盖应保留在普通 PR CI 中。
|
|
238
|
+
|
|
232
239
|
### 作为 CI 部署门 {#ci}
|
|
233
240
|
|
|
234
241
|
将其添加到部署之前运行的管道:
|
|
@@ -229,6 +229,13 @@ agent-native eval --threshold 0.8 # override every eval's pass threshold (0..
|
|
|
229
229
|
| `1` | 至少有一項評估得分低於閾值,或者套件出錯。 |
|
|
230
230
|
| `2` | 錯誤的參數(例如 `--threshold` 位於 `[0, 1]` 之外)。 |
|
|
231
231
|
|
|
232
|
+
### 跳過受控評估 {#skipping}
|
|
233
|
+
|
|
234
|
+
當某個評估套件只應在手動、夜間或具備密鑰的環境中執行時,請使用
|
|
235
|
+
`skipReason`。被跳過的評估不會執行代理或 scorer,不會讓套件失敗,並會
|
|
236
|
+
在 JSON 輸出中顯示 `status: "skipped"` 和 `skipReason`。廉價的確定性覆
|
|
237
|
+
蓋應保留在普通 PR CI 中。
|
|
238
|
+
|
|
232
239
|
### 作為 CI 部署門 {#ci}
|
|
233
240
|
|
|
234
241
|
將其新增到部署之前執行的管道:
|
|
@@ -83,10 +83,11 @@ terminate with an untruthful "done" state.
|
|
|
83
83
|
not buy more time — it just converts a graceful hand-off into a hard kill. The
|
|
84
84
|
walls, in order:
|
|
85
85
|
|
|
86
|
-
1. **Builder model gateway
|
|
87
|
-
|
|
88
|
-
`packages/core/src/agent/engine/builder-engine.ts`.
|
|
89
|
-
|
|
86
|
+
1. **Builder model gateway foreground cap — ~45s.**
|
|
87
|
+
Hosted foreground calls keep a 45s cap in
|
|
88
|
+
`packages/core/src/agent/engine/builder-engine.ts`. Local/non-hosted and
|
|
89
|
+
proven background-function calls use the longer background-style cap because
|
|
90
|
+
they are not constrained by the synchronous function wall.
|
|
90
91
|
2. **Serverless function kill — ~60–65s.** The hosting function is terminated
|
|
91
92
|
shortly after; the heartbeat then reaps the run row as `stale_run`.
|
|
92
93
|
|
|
@@ -366,8 +367,8 @@ The whole `auto_continue` / 40s soft-timeout dance exists to stay under the
|
|
|
366
367
|
**serverless function wall** (~60–65s synchronous on Netlify), not under any
|
|
367
368
|
model limit. Evidence:
|
|
368
369
|
|
|
369
|
-
- `builder-engine.ts
|
|
370
|
-
|
|
370
|
+
- `builder-engine.ts` allows the long background gateway cap in local/non-hosted
|
|
371
|
+
runs; the run loop has no inherent reason to stop at 40s.
|
|
371
372
|
- `run-manager.ts:58,68` `DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS` /
|
|
372
373
|
`HOSTED_SOFT_TIMEOUT_CEILING_MS = 40_000` are pinned just under the function
|
|
373
374
|
wall, and `templates/brain/netlify.toml` sets `[functions."*"] timeout = 75`.
|
|
@@ -379,10 +380,9 @@ function there is no ~60s wall, so:
|
|
|
379
380
|
|
|
380
381
|
- The agent loop can run for minutes in a single invocation with **few or no
|
|
381
382
|
`auto_continue` continuations**.
|
|
382
|
-
- The per-model-call gateway cap
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
longer chopped into 40s chunks.
|
|
383
|
+
- The foreground hosted per-model-call gateway cap still applies per call — see
|
|
384
|
+
[Per-model-call gateway cap](#per-model-call-gateway-cap) — but the _run_ is
|
|
385
|
+
no longer chopped into 40s chunks.
|
|
386
386
|
|
|
387
387
|
This is exactly the Layer 1 worker with a concrete long-lived host: Netlify is
|
|
388
388
|
the durable worker, reached through the existing self-dispatch primitive — the
|
|
@@ -527,29 +527,27 @@ Minimal client change, because the reconnect machinery already exists:
|
|
|
527
527
|
|
|
528
528
|
## Per-model-call gateway cap
|
|
529
529
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
530
|
+
Builder gateway calls now use a runtime-aware cap. Hosted foreground calls keep
|
|
531
|
+
the 45s cap so the synchronous function can still checkpoint before its hard
|
|
532
|
+
wall. Local/non-hosted runs use a longer local cap, and proven Netlify
|
|
533
|
+
background-function runs may use a background cap below the 15-minute function
|
|
534
|
+
wall. The run-manager's 13-minute background soft-timeout should fire before
|
|
535
|
+
the gateway cap in normal durable background operation.
|
|
535
536
|
|
|
536
537
|
Options, in scope-order:
|
|
537
538
|
|
|
538
|
-
1. **
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
2. **
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
**Recommendation:** ship with option 1. The win is removing the _run-level_
|
|
552
|
-
ceiling; the per-call cap is orthogonal and rarely the binding constraint.
|
|
539
|
+
1. **Foreground hosted:** keep the 45s cap. The synchronous function wall is the
|
|
540
|
+
constraint, and a larger cap would turn graceful checkpointing into a hard
|
|
541
|
+
platform kill.
|
|
542
|
+
2. **Local/non-hosted:** allow longer calls by default. Local development should
|
|
543
|
+
not inherit a serverless wall it does not have.
|
|
544
|
+
3. **Durable background:** allow longer calls only when the runtime proves it is
|
|
545
|
+
inside the emitted background function. Keep the cap below 15 minutes and
|
|
546
|
+
above the 13-minute run soft-timeout so background checkpointing still owns
|
|
547
|
+
logical-turn continuation.
|
|
548
|
+
|
|
549
|
+
**Recommendation:** preserve this split. Do not raise the foreground cap; use
|
|
550
|
+
durable background for long-running tool-input generation.
|
|
553
551
|
|
|
554
552
|
## Idempotency / dedup
|
|
555
553
|
|
|
@@ -626,7 +624,8 @@ only): foreground handler inserts the run row, dispatches, returns the SSE
|
|
|
626
624
|
stream from `subscribeToRun`. The background `_process-run` claims the run and
|
|
627
625
|
calls the same `startRun` + `runAgentLoop` the inline path uses, with
|
|
628
626
|
`softTimeoutMs ≈ 13min` (new `backgroundFunction` option in
|
|
629
|
-
`resolveRunSoftTimeoutMs`).
|
|
627
|
+
`resolveRunSoftTimeoutMs`). Builder gateway calls use the background cap only
|
|
628
|
+
when the worker proves it is inside the emitted background function.
|
|
630
629
|
**Done when:** a long multi-step turn that thrashes today completes in one
|
|
631
630
|
background invocation with zero `auto_continue`, events streaming to the client
|
|
632
631
|
via the SQL-poll path, terminal `done` persisted.
|
|
@@ -645,10 +644,10 @@ dispatches, and background→background `auto_continue` chaining for the rare
|
|
|
645
644
|
> 13-min turn (mirror `agent-teams.ts:1886`). Internal checkpointing (Option A)
|
|
646
645
|
> for monotonic progress across any continuation.
|
|
647
646
|
|
|
648
|
-
**Slice 4
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
647
|
+
**Slice 4 — foreground remains capped.**
|
|
648
|
+
Do not raise the hosted foreground per-call gateway cap. Any future tuning
|
|
649
|
+
should stay in the local/background timeout regime unless the synchronous
|
|
650
|
+
platform wall changes.
|
|
652
651
|
|
|
653
652
|
## Open risks / unknowns
|
|
654
653
|
|
package/corpus/core/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-native/core",
|
|
3
|
-
"version": "0.84.
|
|
3
|
+
"version": "0.84.2",
|
|
4
4
|
"description": "Framework for agent-native application development — where AI agents and UI share SQL state, actions, and context",
|
|
5
5
|
"homepage": "https://github.com/BuilderIO/agent-native#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
normalizeReasoningEffortForModel,
|
|
25
25
|
type ReasoningEffort,
|
|
26
26
|
} from "../../shared/reasoning-effort.js";
|
|
27
|
+
import { isInBackgroundFunctionRuntime } from "../durable-background.js";
|
|
27
28
|
import { BUILDER_MODEL_CONFIG } from "../model-config.js";
|
|
28
29
|
import { getBuilderGatewayRequestHeaders } from "./builder-gateway-headers.js";
|
|
29
30
|
import {
|
|
@@ -53,13 +54,20 @@ export const BUILDER_CAPABILITIES: EngineCapabilities = {
|
|
|
53
54
|
|
|
54
55
|
export const BUILDER_SUPPORTED_MODELS = BUILDER_MODEL_CONFIG.supportedModels;
|
|
55
56
|
|
|
56
|
-
// Keep the
|
|
57
|
-
// can append a continuation and persist terminal state
|
|
58
|
-
//
|
|
57
|
+
// Keep the foreground hosted gateway timeout below the synchronous serverless
|
|
58
|
+
// wall so the agent loop can append a continuation and persist terminal state
|
|
59
|
+
// before the host hard-kills the invocation.
|
|
59
60
|
const DEFAULT_BUILDER_GATEWAY_TIMEOUT_MS = 45_000;
|
|
60
|
-
const
|
|
61
|
-
/**
|
|
62
|
-
|
|
61
|
+
const MAX_HOSTED_FOREGROUND_BUILDER_GATEWAY_TIMEOUT_MS = 45_000;
|
|
62
|
+
/**
|
|
63
|
+
* Netlify background functions have a 15-minute wall. Keep the Builder gateway
|
|
64
|
+
* ceiling below that, but above the run-manager's 13-minute soft-timeout so
|
|
65
|
+
* durable background runs checkpoint before this timer wins.
|
|
66
|
+
*/
|
|
67
|
+
const MAX_BACKGROUND_BUILDER_GATEWAY_TIMEOUT_MS = 14 * 60_000;
|
|
68
|
+
/** Local and non-hosted runtimes have no synchronous serverless wall. */
|
|
69
|
+
const MAX_LOCAL_BUILDER_GATEWAY_TIMEOUT_MS =
|
|
70
|
+
MAX_BACKGROUND_BUILDER_GATEWAY_TIMEOUT_MS;
|
|
63
71
|
const BUILDER_GATEWAY_NETWORK_ERROR_CODE = "builder_gateway_network_error";
|
|
64
72
|
|
|
65
73
|
export const BUILDER_DEFAULT_MODEL = BUILDER_MODEL_CONFIG.defaultModel;
|
|
@@ -315,6 +323,7 @@ class BuilderEngine implements AgentEngine {
|
|
|
315
323
|
}
|
|
316
324
|
|
|
317
325
|
yield* parseJsonlStream(reader, opts.model, {
|
|
326
|
+
abortSignal: gatewayAbort.signal,
|
|
318
327
|
didGatewayTimeout: gatewayAbort.didTimeout,
|
|
319
328
|
gatewayTimeoutMs,
|
|
320
329
|
gatewayUrl,
|
|
@@ -430,11 +439,12 @@ async function* emitHttpError(response: Response): AsyncIterable<EngineEvent> {
|
|
|
430
439
|
// a complete line and the client must still process it.
|
|
431
440
|
async function* readJsonlLines(
|
|
432
441
|
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
442
|
+
abortSignal?: AbortSignal,
|
|
433
443
|
): AsyncIterable<string> {
|
|
434
444
|
const decoder = new TextDecoder();
|
|
435
445
|
let buffer = "";
|
|
436
446
|
while (true) {
|
|
437
|
-
const { done, value } = await reader
|
|
447
|
+
const { done, value } = await readStreamChunk(reader, abortSignal);
|
|
438
448
|
if (done) break;
|
|
439
449
|
buffer += decoder.decode(value, { stream: true });
|
|
440
450
|
let newlineIdx = buffer.indexOf("\n");
|
|
@@ -457,6 +467,7 @@ async function* parseJsonlStream(
|
|
|
457
467
|
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
458
468
|
model: string,
|
|
459
469
|
captureContext: {
|
|
470
|
+
abortSignal?: AbortSignal;
|
|
460
471
|
didGatewayTimeout?: () => boolean;
|
|
461
472
|
gatewayTimeoutMs?: number;
|
|
462
473
|
gatewayUrl?: URL;
|
|
@@ -487,7 +498,10 @@ async function* parseJsonlStream(
|
|
|
487
498
|
};
|
|
488
499
|
|
|
489
500
|
try {
|
|
490
|
-
for await (const line of readJsonlLines(
|
|
501
|
+
for await (const line of readJsonlLines(
|
|
502
|
+
reader,
|
|
503
|
+
captureContext.abortSignal,
|
|
504
|
+
)) {
|
|
491
505
|
let event: any;
|
|
492
506
|
try {
|
|
493
507
|
event = JSON.parse(line);
|
|
@@ -722,6 +736,32 @@ async function* parseJsonlStream(
|
|
|
722
736
|
}
|
|
723
737
|
}
|
|
724
738
|
|
|
739
|
+
function readStreamChunk(
|
|
740
|
+
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
741
|
+
abortSignal?: AbortSignal,
|
|
742
|
+
): Promise<ReadableStreamReadResult<Uint8Array>> {
|
|
743
|
+
if (!abortSignal) return reader.read();
|
|
744
|
+
if (abortSignal.aborted) {
|
|
745
|
+
return Promise.reject(abortSignal.reason ?? new Error("Stream aborted"));
|
|
746
|
+
}
|
|
747
|
+
return new Promise((resolve, reject) => {
|
|
748
|
+
const onAbort = () => {
|
|
749
|
+
reject(abortSignal.reason ?? new Error("Stream aborted"));
|
|
750
|
+
};
|
|
751
|
+
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
752
|
+
reader.read().then(
|
|
753
|
+
(value) => {
|
|
754
|
+
abortSignal.removeEventListener("abort", onAbort);
|
|
755
|
+
resolve(value);
|
|
756
|
+
},
|
|
757
|
+
(err) => {
|
|
758
|
+
abortSignal.removeEventListener("abort", onAbort);
|
|
759
|
+
reject(err);
|
|
760
|
+
},
|
|
761
|
+
);
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
|
|
725
765
|
function normalizeGatewayErrorText(raw: string, status: number): string {
|
|
726
766
|
const text = raw.trim();
|
|
727
767
|
const looksHtml = /<html[\s>]|<body[\s>]|<head[\s>]/i.test(text);
|
|
@@ -761,6 +801,14 @@ export function createBuilderEngine(
|
|
|
761
801
|
}
|
|
762
802
|
|
|
763
803
|
function resolveMaxBuilderGatewayTimeoutMs(): number {
|
|
804
|
+
if (isInBackgroundFunctionRuntime()) {
|
|
805
|
+
return MAX_BACKGROUND_BUILDER_GATEWAY_TIMEOUT_MS;
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
if (!isHostedBuilderRuntime()) {
|
|
809
|
+
return MAX_LOCAL_BUILDER_GATEWAY_TIMEOUT_MS;
|
|
810
|
+
}
|
|
811
|
+
|
|
764
812
|
try {
|
|
765
813
|
const base = getBuilderGatewayBaseUrl();
|
|
766
814
|
if (/^https?:\/\/(localhost|127\.0\.0\.1)([:/]|$)/i.test(base)) {
|
|
@@ -769,7 +817,31 @@ function resolveMaxBuilderGatewayTimeoutMs(): number {
|
|
|
769
817
|
} catch {
|
|
770
818
|
// ignore malformed override
|
|
771
819
|
}
|
|
772
|
-
return
|
|
820
|
+
return MAX_HOSTED_FOREGROUND_BUILDER_GATEWAY_TIMEOUT_MS;
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
function isHostedBuilderRuntime(): boolean {
|
|
824
|
+
if (
|
|
825
|
+
process.env.NETLIFY &&
|
|
826
|
+
process.env.NETLIFY !== "false" &&
|
|
827
|
+
process.env.NETLIFY_LOCAL !== "true"
|
|
828
|
+
) {
|
|
829
|
+
return true;
|
|
830
|
+
}
|
|
831
|
+
if (
|
|
832
|
+
process.env.AWS_LAMBDA_FUNCTION_NAME &&
|
|
833
|
+
process.env.NETLIFY_LOCAL !== "true"
|
|
834
|
+
) {
|
|
835
|
+
return true;
|
|
836
|
+
}
|
|
837
|
+
return Boolean(
|
|
838
|
+
process.env.CF_PAGES ||
|
|
839
|
+
process.env.VERCEL ||
|
|
840
|
+
process.env.VERCEL_ENV ||
|
|
841
|
+
process.env.RENDER ||
|
|
842
|
+
process.env.FLY_APP_NAME ||
|
|
843
|
+
process.env.K_SERVICE,
|
|
844
|
+
);
|
|
773
845
|
}
|
|
774
846
|
|
|
775
847
|
function getBuilderGatewayTimeoutMs(): number {
|
|
@@ -46,8 +46,8 @@ const CLEANUP_DELAY_MS = 5 * 60 * 1000;
|
|
|
46
46
|
*
|
|
47
47
|
* This MUST fire before the two upstream hard walls that otherwise kill a run
|
|
48
48
|
* mid-turn with no chance to hand off:
|
|
49
|
-
* 1. The Builder model gateway
|
|
50
|
-
*
|
|
49
|
+
* 1. The Builder model gateway keeps a 45s cap only for hosted foreground
|
|
50
|
+
* runs; local and proven background-function runs use longer caps.
|
|
51
51
|
* 2. Serverless functions are hard-killed around 60-65s (the heartbeat then
|
|
52
52
|
* reaps the row as a stale_run).
|
|
53
53
|
* Production data showed every cutoff landing in the 44-70s window with ZERO
|
|
@@ -61,11 +61,12 @@ export const DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS = 40_000;
|
|
|
61
61
|
|
|
62
62
|
/**
|
|
63
63
|
* Hard ceiling for the hosted soft timeout. On a hosted runtime the
|
|
64
|
-
* auto_continue soft timeout can never usefully exceed this — the
|
|
65
|
-
*
|
|
66
|
-
* env value just guarantees the cutoff is a hard error instead of a graceful
|
|
67
|
-
* hand-off. Any resolved value above this is clamped down
|
|
68
|
-
* dev (non-hosted) is left alone so long-running local turns aren't
|
|
64
|
+
* foreground auto_continue soft timeout can never usefully exceed this — the
|
|
65
|
+
* synchronous function (~60s) wall kills the run first, so a larger configured
|
|
66
|
+
* or env value just guarantees the cutoff is a hard error instead of a graceful
|
|
67
|
+
* hand-off. Any resolved value above this is clamped down for hosted foreground
|
|
68
|
+
* runs. Local dev (non-hosted) is left alone so long-running local turns aren't
|
|
69
|
+
* chunked.
|
|
69
70
|
*
|
|
70
71
|
* IMPORTANT: this clamp is for the INTERACTIVE / foreground path and must NOT
|
|
71
72
|
* be raised. The foreground POST still rides a synchronous serverless function
|