@iris-eval/mcp-server 0.4.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -44
- package/dist/audit-log-reader.d.ts +0 -2
- package/dist/audit-log-reader.js +5 -5
- package/dist/config/defaults.js +3 -3
- package/dist/config/index.d.ts +1 -0
- package/dist/config/index.js +32 -6
- package/dist/custom-rule-store.js +100 -38
- package/dist/dashboard/assets/index-BZZt8bVh.js +10 -0
- package/dist/dashboard/assets/index-UffZ-aEJ.css +1 -0
- package/dist/dashboard/fonts/jetbrains-mono-cyrillic-ext.woff2 +0 -0
- package/dist/dashboard/fonts/jetbrains-mono-cyrillic.woff2 +0 -0
- package/dist/dashboard/fonts/jetbrains-mono-greek.woff2 +0 -0
- package/dist/dashboard/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- package/dist/dashboard/fonts/jetbrains-mono-latin.woff2 +0 -0
- package/dist/dashboard/fonts/jetbrains-mono-vietnamese.woff2 +0 -0
- package/dist/dashboard/fonts/manrope-cyrillic-ext.woff2 +0 -0
- package/dist/dashboard/fonts/manrope-cyrillic.woff2 +0 -0
- package/dist/dashboard/fonts/manrope-greek.woff2 +0 -0
- package/dist/dashboard/fonts/manrope-latin-ext.woff2 +0 -0
- package/dist/dashboard/fonts/manrope-latin.woff2 +0 -0
- package/dist/dashboard/fonts/manrope-vietnamese.woff2 +0 -0
- package/dist/dashboard/fonts/space-grotesk-latin-ext.woff2 +0 -0
- package/dist/dashboard/fonts/space-grotesk-latin.woff2 +0 -0
- package/dist/dashboard/fonts/space-grotesk-vietnamese.woff2 +0 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/dashboard/routes/failures.d.ts +3 -0
- package/dist/dashboard/routes/failures.js +76 -0
- package/dist/dashboard/routes/index.d.ts +1 -0
- package/dist/dashboard/routes/index.js +1 -0
- package/dist/dashboard/routes/preferences.js +7 -2
- package/dist/dashboard/routes/rules.js +32 -14
- package/dist/dashboard/routes/traces.d.ts +12 -1
- package/dist/dashboard/routes/traces.js +90 -2
- package/dist/dashboard/seed-demo-data.d.ts +49 -0
- package/dist/dashboard/seed-demo-data.js +1080 -0
- package/dist/dashboard/server.js +124 -16
- package/dist/dashboard/validation.d.ts +74 -0
- package/dist/dashboard/validation.js +31 -2
- package/dist/eval/citation-verify/resolve.js +29 -0
- package/dist/eval/citation-verify/verifier.d.ts +1 -0
- package/dist/eval/citation-verify/verifier.js +12 -4
- package/dist/eval/engine.d.ts +15 -1
- package/dist/eval/engine.js +99 -15
- package/dist/eval/failure-rank.d.ts +14 -0
- package/dist/eval/failure-rank.js +44 -0
- package/dist/eval/rules/custom.d.ts +29 -1
- package/dist/eval/rules/custom.js +155 -19
- package/dist/eval/rules/regex-budget.d.ts +5 -0
- package/dist/eval/rules/regex-budget.js +100 -0
- package/dist/eval/rules/regex-sandbox.d.ts +26 -0
- package/dist/eval/rules/regex-sandbox.js +131 -0
- package/dist/eval/rules/relevance.d.ts +0 -2
- package/dist/eval/rules/relevance.js +6 -68
- package/dist/eval/rules/safety.d.ts +10 -0
- package/dist/eval/rules/safety.js +1377 -30
- package/dist/index.js +206 -19
- package/dist/middleware/rebinding-guard.d.ts +21 -0
- package/dist/middleware/rebinding-guard.js +77 -0
- package/dist/preferences.js +5 -10
- package/dist/self-test.d.ts +18 -0
- package/dist/self-test.js +329 -0
- package/dist/storage/migrations/005-normalize-created-at.d.ts +3 -0
- package/dist/storage/migrations/005-normalize-created-at.js +34 -0
- package/dist/storage/migrations/index.js +8 -1
- package/dist/storage/sqlite-adapter.d.ts +2 -0
- package/dist/storage/sqlite-adapter.js +93 -13
- package/dist/tools/delete-rule.d.ts +2 -1
- package/dist/tools/delete-rule.js +13 -4
- package/dist/tools/delete-trace.js +2 -1
- package/dist/tools/deploy-rule.d.ts +2 -1
- package/dist/tools/deploy-rule.js +29 -7
- package/dist/tools/evaluate-output.js +36 -9
- package/dist/tools/evaluate-with-llm-judge.js +2 -1
- package/dist/tools/get-traces.js +6 -2
- package/dist/tools/index.js +2 -2
- package/dist/tools/list-rules.js +2 -1
- package/dist/tools/log-trace.d.ts +51 -0
- package/dist/tools/log-trace.js +14 -2
- package/dist/tools/strict-input.d.ts +2 -0
- package/dist/tools/strict-input.js +35 -0
- package/dist/tools/verify-citations.js +7 -5
- package/dist/transport/http.js +24 -2
- package/dist/types/config.d.ts +7 -0
- package/dist/types/decision-moment.d.ts +12 -0
- package/dist/types/eval.d.ts +32 -0
- package/dist/types/query.d.ts +1 -1
- package/dist/utils/iris-home.d.ts +1 -0
- package/dist/utils/iris-home.js +21 -0
- package/dist/utils/write-atomic.d.ts +3 -0
- package/dist/utils/write-atomic.js +96 -0
- package/package.json +3 -2
- package/server.json +3 -3
- package/dist/dashboard/assets/index-B4Aw6ozt.css +0 -1
- package/dist/dashboard/assets/index-CIKsbEhq.js +0 -10
|
@@ -0,0 +1 @@
|
|
|
1
|
+
@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/jetbrains-mono-cyrillic-ext.woff2)format("woff2");unicode-range:U+460-52F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/jetbrains-mono-cyrillic.woff2)format("woff2");unicode-range:U+301,U+400-45F,U+490-491,U+4B0-4B1,U+2116}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/jetbrains-mono-greek.woff2)format("woff2");unicode-range:U+370-377,U+37A-37F,U+384-38A,U+38C,U+38E-3A1,U+3A3-3FF}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/jetbrains-mono-vietnamese.woff2)format("woff2");unicode-range:U+102-103,U+110-111,U+128-129,U+168-169,U+1A0-1A1,U+1AF-1B0,U+300-301,U+303-304,U+308-309,U+323,U+329,U+1EA0-1EF9,U+20AB}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/jetbrains-mono-latin-ext.woff2)format("woff2");unicode-range:U+100-2BA,U+2BD-2C5,U+2C7-2CC,U+2CE-2D7,U+2DD-2FF,U+304,U+308,U+329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/jetbrains-mono-latin.woff2)format("woff2");unicode-range:U+??,U+131,U+152-153,U+2BB-2BC,U+2C6,U+2DA,U+2DC,U+304,U+308,U+329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:Manrope;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/manrope-cyrillic-ext.woff2)format("woff2");unicode-range:U+460-52F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:Manrope;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/manrope-cyrillic.woff2)format("woff2");unicode-range:U+301,U+400-45F,U+490-491,U+4B0-4B1,U+2116}@font-face{font-family:Manrope;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/manrope-greek.woff2)format("woff2");unicode-range:U+370-377,U+37A-37F,U+384-38A,U+38C,U+38E-3A1,U+3A3-3FF}@font-face{font-family:Manrope;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/manrope-vietnamese.woff2)format("woff2");unicode-range:U+102-103,U+110-111,U+128-129,U+168-169,U+1A0-1A1,U+1AF-1B0,U+300-301,U+303-304,U+308-309,U+323,U+329,U+1EA0-1EF9,U+20AB}@font-face{font-family:Manrope;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/manrope-latin-ext.woff2)format("woff2");unicode-range:U+100-2BA,U+2BD-2C5,U+2C7-2CC,U+2CE-2D7,U+2DD-2FF,U+304,U+308,U+329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:Manrope;font-style:normal;font-weight:400 700;font-display:swap;src:url(/fonts/manrope-latin.woff2)format("woff2");unicode-range:U+??,U+131,U+152-153,U+2BB-2BC,U+2C6,U+2DA,U+2DC,U+304,U+308,U+329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:Space Grotesk;font-style:normal;font-weight:500 700;font-display:swap;src:url(/fonts/space-grotesk-vietnamese.woff2)format("woff2");unicode-range:U+102-103,U+110-111,U+128-129,U+168-169,U+1A0-1A1,U+1AF-1B0,U+300-301,U+303-304,U+308-309,U+323,U+329,U+1EA0-1EF9,U+20AB}@font-face{font-family:Space Grotesk;font-style:normal;font-weight:500 700;font-display:swap;src:url(/fonts/space-grotesk-latin-ext.woff2)format("woff2");unicode-range:U+100-2BA,U+2BD-2C5,U+2C7-2CC,U+2CE-2D7,U+2DD-2FF,U+304,U+308,U+329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:Space Grotesk;font-style:normal;font-weight:500 700;font-display:swap;src:url(/fonts/space-grotesk-latin.woff2)format("woff2");unicode-range:U+??,U+131,U+152-153,U+2BB-2BC,U+2C6,U+2DA,U+2DC,U+304,U+308,U+329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}:root{--lightningcss-light:initial;--lightningcss-dark: ;color-scheme:light dark;--iris-50:#f0fdfa;--iris-100:#ccfbf1;--iris-200:#99f6e4;--iris-300:#5eead4;--iris-400:#2dd4bf;--iris-500:#14b8a6;--iris-600:#0d9488;--iris-700:#0f766e;--iris-800:#115e59;--iris-900:#134e4a;--iris-950:#042f2e;--eval-pass:#22c55e;--eval-warn:#eab308;--eval-fail:#ef4444;--eval-tool:#3b82f6;--eval-llm:#a855f7;--eval-skipped:#71717a}@media (prefers-color-scheme:dark){:root{--lightningcss-light: ;--lightningcss-dark:initial}}:root,[data-theme=dark]{--bg-base:#050508;--bg-raised:#08080e;--bg-surface:#0d0d15;--bg-card:#101018;--bg-card-hover:#16161f;--border-subtle:#ffffff0d;--border-default:#ffffff14;--border-strong:#ffffff24;--border-glow:#14b8a680;--text-primary:#f0f0f5;--text-secondary:#9494a8;--text-muted:#5e5e72;--text-accent:var(--iris-400);--glow-primary:#14b8a61f;--glow-strong:#14b8a640;--shadow-sm:0 1px 2px #0000004d;--shadow-md:0 4px 6px #0006;--shadow-lg:0 10px 15px #00000080}[data-theme=light]{--bg-base:#fafcfc;--bg-raised:#f1f5f5;--bg-surface:#e8eded;--bg-card:#fff;--bg-card-hover:#f4f8f8;--border-subtle:#0000000a;--border-default:#00000014;--border-strong:#00000024;--border-glow:#0d948859;--text-primary:#0a0f0e;--text-secondary:#3d5250;--text-muted:#7a908e;--text-accent:var(--iris-700);--glow-primary:#0d94880f;--glow-strong:#0d94881f;--shadow-sm:0 1px 2px #0000000f;--shadow-md:0 4px 6px #00000014;--shadow-lg:0 10px 15px #0000001a}:root{--font-display:"Space Grotesk", -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;--font-body:"Manrope", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;--font-mono:"JetBrains Mono", "Fira Code", ui-monospace, monospace;--font-sans:var(--font-body);--text-caption-xs:11px;--text-caption:12px;--text-body-sm:13px;--text-body:14px;--text-body-lg:15px;--text-heading-sm:16px;--text-heading:20px;--text-display-sm:28px;--text-display:40px;--font-size-xs:var(--text-caption);--font-size-sm:var(--text-body-sm);--font-size-base:var(--text-body);--font-size-lg:var(--text-body-lg);--font-size-xl:var(--text-heading-sm);--font-size-2xl:var(--text-heading);--font-size-3xl:var(--text-display-sm);--leading-body:1.5;--leading-heading:1.2;--leading-display:1.1;--leading-mono:1.4;--space-0_5:2px;--space-1:4px;--space-1_5:6px;--space-2:8px;--space-2_5:10px;--space-3:12px;--space-4:16px;--space-5:20px;--space-6:24px;--space-8:32px;--space-10:40px;--space-12:48px;--space-16:64px;--space-20:80px;--space-24:96px}:root,[data-density=compact]{--density-row:32px;--density-padding:var(--space-3);--density-body:var(--text-body-sm)}[data-density=comfortable]{--density-row:44px;--density-padding:var(--space-4);--density-body:var(--text-body)}:root{--sidebar-width-expanded:256px;--sidebar-width-collapsed:64px;--header-height:56px;--page-toolbar-height:40px;--radius-xs:4px;--radius-sm:6px;--radius:8px;--radius-lg:12px;--radius-xl:16px;--radius-pill:999px;--border-radius:var(--radius);--border-radius-sm:var(--radius-xs);--border-radius-lg:var(--radius-lg);--transition-instant:.1s ease;--transition-fast:.15s ease;--transition-base:.2s ease;--transition-slow:.3s ease;--ease-iris:cubic-bezier(.25, .4, .25, 1);--bg-primary:var(--bg-base);--bg-secondary:var(--bg-raised);--bg-tertiary:var(--bg-surface);--bg-hover:var(--bg-card-hover);--border-color:var(--border-default);--accent-primary:var(--iris-500);--accent-primary-hover:var(--iris-400);--accent-success:var(--eval-pass);--accent-error:var(--eval-fail);--accent-warning:var(--eval-warn);--accent-tool:var(--eval-tool);--accent-llm:var(--eval-llm)}html{transition:background-color var(--transition-base), color var(--transition-base)}.iris-btn{appearance:none;justify-content:center;align-items:center;gap:var(--space-1_5);border-radius:var(--radius-sm);font-family:inherit;font-size:var(--text-body-sm);font-weight:500;line-height:var(--leading-body);padding:var(--space-1_5) var(--space-3);cursor:pointer;-webkit-user-select:none;user-select:none;transition:background-color var(--transition-instant), border-color var(--transition-instant), color var(--transition-instant), transform var(--transition-instant);border:1px solid #0000;text-decoration:none;display:inline-flex}.iris-btn:disabled,.iris-btn[aria-busy=true]{opacity:.5;cursor:default}.iris-btn:not(:disabled):active{transform:translateY(1px)}.iris-btn--ghost{border-color:var(--border-default);color:var(--text-secondary);background:0 0}.iris-btn--ghost:not(:disabled):hover{background:var(--bg-card-hover);border-color:var(--border-strong);color:var(--text-primary)}.iris-btn--primary{background:var(--iris-600);border-color:var(--iris-600);color:#fff;font-weight:600}.iris-btn--primary:not(:disabled):hover{background:var(--iris-500);border-color:var(--iris-500)}.iris-btn--danger{border-color:var(--border-default);color:var(--eval-fail);background:0 0}.iris-btn--danger:not(:disabled):hover{border-color:var(--eval-fail);background:color-mix(in srgb, var(--eval-fail) 10%, transparent)}.iris-btn--danger-solid{background:var(--eval-fail);border-color:var(--eval-fail);color:#fff;font-weight:600}.iris-btn--danger-solid:not(:disabled):hover{background:#dc2626;border-color:#dc2626}.iris-btn--sm{padding:var(--space-0_5) var(--space-2_5);font-size:var(--text-caption)}.iris-btn--mono{font-family:var(--font-mono)}.iris-kbd{font-size:var(--text-caption);font-family:var(--font-mono);background:var(--bg-surface);border:1px solid var(--border-default);border-radius:var(--radius-xs);padding:2px var(--space-2);color:var(--text-muted)}.iris-card{background:var(--bg-card);border:1px solid var(--border-default);border-radius:var(--radius-lg);transition:background-color var(--transition-fast), border-color var(--transition-fast)}.iris-card--hover:hover,.iris-card--hover:focus-within{border-color:var(--border-strong)}.iris-stack{gap:var(--space-3);flex-direction:column;display:flex}.iris-stack--lg{gap:var(--space-6)}.iris-row{align-items:center;gap:var(--space-3);display:flex}.iris-grid-kpis{gap:var(--space-3);grid-template-columns:repeat(auto-fit,minmax(180px,1fr));display:grid}.iris-grid-split{gap:var(--space-3);grid-template-columns:repeat(auto-fit,minmax(340px,1fr));display:grid}.iris-grid-gauge{gap:var(--space-3);grid-template-columns:minmax(280px,1fr) minmax(0,1.8fr);align-items:stretch;display:grid}@media (width<=960px){.iris-grid-gauge{grid-template-columns:1fr}}.iris-num{font-variant-numeric:tabular-nums}.iris-num--right{text-align:right}.iris-error-box{background:color-mix(in srgb, var(--eval-fail) 12%, transparent);border:1px solid var(--eval-fail);border-radius:var(--radius);padding:var(--space-4);color:var(--eval-fail);gap:var(--space-2);flex-direction:column;display:flex}.iris-backdrop{z-index:110;background:oklch(0% 0 0/.55);justify-content:center;align-items:flex-start;display:flex;position:fixed;inset:0}.iris-modal{background:var(--bg-card);border:1px solid var(--border-default);border-radius:var(--radius-lg);box-shadow:var(--shadow-lg);flex-direction:column;display:flex;overflow:hidden}.moment-card{gap:var(--space-3);padding:var(--space-4);color:var(--text-primary);border-left:3px solid var(--moment-sig-color,transparent);grid-template-columns:24px 48px 1fr auto;align-items:start;display:grid}.moment-card:hover,.moment-card:focus-within{background:var(--bg-card-hover);border-left-color:var(--moment-sig-color,transparent)}.moment-card--archived{opacity:.5}.moment-card--selected{background:var(--bg-card-hover);border-color:var(--iris-500);border-left-color:var(--moment-sig-color,transparent)}.moment-card__checkbox-wrap{padding-top:var(--space-1);justify-content:center;align-items:center;display:flex}.moment-card__checkbox{cursor:pointer;width:16px;height:16px;accent-color:var(--iris-500)}.moment-card__rail{align-items:center;gap:var(--space-1);flex-direction:column;display:flex}.moment-card__glyph{width:32px;height:32px;font-size:var(--text-body);font-family:var(--font-mono);color:var(--bg-base);border-radius:50%;justify-content:center;align-items:center;font-weight:700;display:flex}.moment-card__body-link{color:inherit;gap:var(--space-2);flex-direction:column;min-width:0;text-decoration:none;display:flex}.moment-card__body-link:hover{color:inherit}.moment-card__header{align-items:center;gap:var(--space-2);flex-wrap:wrap;display:flex}.moment-card__agent{font-size:var(--text-body-sm);color:var(--text-primary);font-weight:600}.moment-card__sig{font-size:var(--text-caption);font-family:var(--font-mono);color:var(--text-secondary);text-overflow:ellipsis;white-space:nowrap;font-weight:500;overflow:hidden}.moment-card__verdict{font-size:var(--text-caption);font-family:var(--font-mono);letter-spacing:.05em;font-weight:700}.moment-card__tag{font-size:var(--text-caption);font-family:var(--font-mono);color:var(--text-muted);background:var(--bg-surface);border:1px solid var(--border-default);padding:0 var(--space-2);border-radius:var(--radius-xs)}.moment-card__tag--new{letter-spacing:.05em;color:var(--text-accent);background:var(--glow-primary);border-color:var(--iris-500);font-weight:700}.moment-card__preview{font-size:var(--text-caption);font-family:var(--font-mono);color:var(--text-muted);text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.moment-card__chips{gap:var(--space-1);flex-wrap:wrap;display:flex}.moment-card__chip{font-size:var(--text-caption);font-family:var(--font-mono);padding:1px var(--space-1_5);border-radius:var(--radius-xs);background:color-mix(in srgb, var(--eval-fail) 12%, transparent);color:var(--eval-fail)}.moment-card__meta{align-items:flex-end;gap:var(--space-1);font-size:var(--text-caption);font-family:var(--font-mono);color:var(--text-muted);flex-direction:column;display:flex}.view-tabs{align-items:center;gap:var(--space-1);border-bottom:1px solid var(--border-subtle);margin-bottom:var(--space-5);display:flex}.view-tabs__tab{align-items:center;gap:var(--space-2);padding:var(--space-2) var(--space-3);font-family:var(--font-display);font-size:var(--text-body-sm);letter-spacing:-.01em;color:var(--text-muted);cursor:pointer;transition:color var(--transition-fast), border-color var(--transition-fast);border-bottom:2px solid #0000;margin-bottom:-1px;font-weight:600;text-decoration:none;display:inline-flex}.view-tabs__tab:hover{color:var(--text-primary)}.view-tabs__tab[aria-selected=true]{color:var(--text-primary);border-bottom-color:var(--iris-500)}.view-tabs__hint{font-size:var(--text-caption-xs);color:var(--text-muted);font-family:var(--font-mono);padding-left:var(--space-3);margin-left:auto}.cmdk-backdrop{padding-top:15vh}.cmdk__panel{width:min(640px,100% - 32px);max-height:70vh}.cmdk__input-wrap{padding:var(--space-3) var(--space-4);border-bottom:1px solid var(--border-default);align-items:center;gap:var(--space-2);display:flex}.cmdk__prompt{font-family:var(--font-mono);color:var(--text-muted);font-size:var(--text-body-sm)}.cmdk__input{color:var(--text-primary);font-size:var(--text-body);background:0 0;border:none;outline:none;flex:1;font-family:inherit}.cmdk__list{padding:var(--space-2);flex:1;overflow:auto}.cmdk__section-title{font-size:var(--text-caption);font-family:var(--font-mono);color:var(--text-muted);text-transform:uppercase;letter-spacing:.05em;padding:var(--space-2) var(--space-3)}.cmdk__item{justify-content:space-between;align-items:center;gap:var(--space-3);padding:var(--space-2) var(--space-3);border-radius:var(--radius-sm);cursor:pointer;color:var(--text-secondary);display:flex}.cmdk__item:hover,.cmdk__item[aria-selected=true]{background:var(--bg-card-hover);color:var(--text-primary)}.cmdk__item-body{flex-direction:column;gap:2px;min-width:0;display:flex}.cmdk__item-title{font-size:var(--text-body-sm);font-weight:500}.cmdk__item-subtitle{font-size:var(--text-caption);color:var(--text-muted);font-family:var(--font-mono);text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.cmdk__empty{padding:var(--space-6);text-align:center;color:var(--text-muted);font-size:var(--text-body-sm)}.cmdk__empty-clear{color:var(--text-accent);cursor:pointer;font-size:inherit;background:0 0;border:none;font-family:inherit;text-decoration:underline}.cmdk__footer{gap:var(--space-3);font-size:var(--text-caption);color:var(--text-muted);font-family:var(--font-mono);padding:var(--space-2) var(--space-4);border-top:1px solid var(--border-default);background:var(--bg-surface);display:flex}.cmdk__footer-count{margin-left:auto}.confirm-dialog__panel{width:min(440px,100% - 32px);padding:var(--space-5);gap:var(--space-3);margin-top:20vh}.confirm-dialog__title{font-family:var(--font-display);font-size:var(--text-heading-sm);letter-spacing:-.01em;color:var(--text-primary);margin:0;font-weight:600}.confirm-dialog__body{font-size:var(--text-body-sm);color:var(--text-secondary);line-height:var(--leading-body);margin:0}.confirm-dialog__error{font-size:var(--text-body-sm);color:var(--eval-fail);margin:0}.confirm-dialog__actions{justify-content:flex-end;gap:var(--space-2);margin-top:var(--space-2);display:flex}.stat-tile{padding:var(--space-4) var(--space-5);gap:var(--space-2);flex-direction:column;min-height:110px;display:flex}.stat-tile__header{align-items:center;gap:var(--space-2);font-size:var(--text-caption-xs);font-weight:600;font-family:var(--font-body);text-transform:uppercase;letter-spacing:.08em;color:var(--text-muted);display:flex}.stat-tile__value-row{align-items:baseline;gap:var(--space-2);display:flex}.stat-tile__value{font-family:var(--font-display);font-size:var(--text-display-sm);font-weight:700;line-height:var(--leading-display);letter-spacing:-.02em;font-variant-numeric:tabular-nums}.stat-tile__delta{font-family:var(--font-mono);font-size:var(--text-caption);font-weight:500}.stat-tile__sub{font-size:var(--text-caption);color:var(--text-secondary)}.section-header{padding-top:var(--space-3);margin-bottom:var(--space-1);flex-direction:column;gap:2px;display:flex}.section-header__top{justify-content:space-between;align-items:baseline;gap:var(--space-3);flex-wrap:wrap;display:flex}.section-header__title{font-family:var(--font-display);font-size:var(--text-heading-sm);letter-spacing:-.015em;color:var(--text-primary);margin:0;font-weight:600}.section-header__trailing{font-family:var(--font-mono);font-size:var(--text-caption);color:var(--text-muted)}.section-header__question{font-size:var(--text-caption);color:var(--text-muted);margin:0}.detail-back{color:var(--text-muted);font-size:var(--text-body-sm);align-items:center;gap:var(--space-1);width:fit-content;transition:color var(--transition-instant);text-decoration:none;display:inline-flex}.detail-back:hover{color:var(--text-primary)}.detail-card{padding:var(--space-5)}.detail-card__grid{gap:var(--space-4);font-size:var(--text-body-sm);grid-template-columns:repeat(auto-fit,minmax(200px,1fr));display:grid}.detail-card__label{color:var(--text-muted);font-size:var(--text-caption);font-family:var(--font-mono);text-transform:uppercase;letter-spacing:.05em}.detail-section{gap:var(--space-3);flex-direction:column;display:flex}.detail-section__title{font-family:var(--font-display);font-size:var(--text-heading-sm);color:var(--text-primary);letter-spacing:-.01em;font-weight:600;line-height:var(--leading-heading);margin:0}.eval-card{padding:var(--space-4);gap:var(--space-3);flex-direction:column;display:flex}.eval-card__badges{align-items:center;gap:var(--space-3);display:flex}.eval-card__rules{gap:var(--space-2);flex-direction:column;display:flex}.eval-card__rule{align-items:center;gap:var(--space-2);font-size:var(--text-body-sm);padding:var(--space-1) var(--space-2);background:var(--bg-base);border-radius:var(--radius-xs);display:flex}.eval-card__rule-mark{width:16px}.eval-card__rule-name{color:var(--text-secondary);font-size:var(--text-caption)}.eval-card__rule-message{color:var(--text-muted);font-size:var(--text-caption);flex:1}.eval-card__suggestions{font-size:var(--text-body-sm)}.eval-card__suggestions-label{color:var(--text-muted);margin-bottom:var(--space-1)}.eval-card__suggestions-list{padding-left:var(--space-5);color:var(--text-secondary);margin:0}.rule-card{padding:var(--space-4);gap:var(--space-3);grid-template-columns:1fr auto;align-items:start;display:grid}.rule-card__body{gap:var(--space-2);flex-direction:column;min-width:0;display:flex}.rule-card__name-row{gap:var(--space-2);flex-wrap:wrap;align-items:baseline;display:flex}.rule-card__name{font-size:var(--text-body);font-weight:600;font-family:var(--font-mono);color:var(--text-primary)}.rule-card__description{font-size:var(--text-body-sm);color:var(--text-secondary);margin:0}.rule-card__meta{gap:var(--space-3);font-size:var(--text-caption);color:var(--text-muted);font-family:var(--font-mono);flex-wrap:wrap;display:flex}.rule-card__badge{font-size:var(--text-caption);font-family:var(--font-mono);padding:1px var(--space-1_5);border-radius:var(--radius-xs);background:var(--bg-surface);color:var(--text-secondary)}.rule-card__source-link{color:var(--text-accent);font-size:var(--text-caption);font-family:var(--font-mono);text-decoration:underline}*,:before,:after{box-sizing:border-box;margin:0;padding:0}html,body,#root{width:100%;height:100%}body{font-family:var(--font-body);font-size:var(--text-body);color:var(--text-primary);background-color:var(--bg-base);line-height:var(--leading-body);-webkit-font-smoothing:antialiased;font-feature-settings:"cv11", "ss01";font-variant-numeric:tabular-nums}h1,h2,h3,h4,.display{font-family:var(--font-display);letter-spacing:-.01em;font-weight:600}a{color:var(--text-accent);text-decoration:none}a:hover{color:var(--iris-300)}button{cursor:pointer;font-family:inherit}input,select{font-family:inherit;font-size:inherit}code,pre{font-family:var(--font-mono)}a:focus-visible,button:focus-visible,input:focus-visible,select:focus-visible,[role=button]:focus-visible{outline:2px solid var(--iris-500);outline-offset:2px;border-radius:var(--radius-xs)}.iris-sr-reveal{clip:rect(0, 0, 0, 0);white-space:nowrap;border:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.iris-sr-reveal:focus-within{width:auto;height:auto;margin:var(--space-3) 0 0 0;padding:var(--space-3) var(--space-4);clip:auto;white-space:normal;background:var(--bg-card);color:var(--text-primary);border:1px solid var(--iris-500);border-radius:var(--radius-sm);position:static;overflow:visible}.iris-sr-reveal:focus-within>li{padding:var(--space-1) 0;list-style:none}::selection{background:var(--iris-600);color:#fff}::-webkit-scrollbar{width:8px;height:8px}::-webkit-scrollbar-track{background:var(--bg-base)}::-webkit-scrollbar-thumb{background:var(--border-strong);border-radius:var(--radius-xs)}::-webkit-scrollbar-thumb:hover{background:var(--text-muted)}html{scrollbar-color:var(--border-strong) transparent;scrollbar-width:thin}@keyframes pulse-ring{0%{opacity:.5;transform:scale(1)}to{opacity:0;transform:scale(2.5)}}.pulse-dot{position:relative}.pulse-dot:after{content:"";background:var(--iris-500);border-radius:50%;animation:2s ease-out infinite pulse-ring;position:absolute;inset:-2px}@media (prefers-reduced-motion:reduce){*,:before,:after{scroll-behavior:auto!important;transition-duration:.01ms!important;animation-duration:.01ms!important;animation-iteration-count:1!important}}@media (width<=767px){aside[aria-label=Main\ navigation]{width:160px}main{overflow-x:auto}}@media print{body{color:#000!important;background:#fff!important}aside[aria-label=Main\ navigation],header,[role=region][aria-label=Welcome],[role=region][aria-label=Bulk\ actions],[role=status],[role=dialog]{display:none!important}body,#root,main{height:auto!important;overflow:visible!important}main{padding:0!important}tr,pre,code{page-break-inside:avoid}h1,h2,h3{page-break-after:avoid}[aria-label*=violation],[aria-label*=spike],[aria-label*=collision],[aria-label*=Pass],[aria-label*=Fail]{border:1px solid #000!important}a{color:#000!important;text-decoration:underline!important}a[href^=http]:after{content:" (" attr(href) ")";color:#555;font-size:80%}}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
|
7
7
|
<title>Iris — Agent Eval & Observability</title>
|
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
|
9
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
8
|
+
<script type="module" crossorigin src="/assets/index-BZZt8bVh.js"></script>
|
|
9
|
+
<link rel="stylesheet" crossorigin href="/assets/index-UffZ-aEJ.css">
|
|
10
10
|
</head>
|
|
11
11
|
<body>
|
|
12
12
|
<div id="root"></div>
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { requireTenant } from '../../middleware/tenant.js';
|
|
2
|
+
import { deriveMoment } from '../../eval/decision-moment.js';
|
|
3
|
+
import { isFailureMoment, rankFailureScore } from '../../eval/failure-rank.js';
|
|
4
|
+
import { failuresQuerySchema } from '../validation.js';
|
|
5
|
+
/*
|
|
6
|
+
* How many recent traces to scan when building the failure list. On a
|
|
7
|
+
* mostly-passing fleet failures are sparse, so the scan window must be
|
|
8
|
+
* wider than the returned list — a hard cap of `limit` traces would miss
|
|
9
|
+
* every failure older than the last `limit` runs. 500 is bounded work
|
|
10
|
+
* for local SQLite (same hydration loop the moments route already runs
|
|
11
|
+
* at 200) and reaches far enough back for a single-user install.
|
|
12
|
+
*/
|
|
13
|
+
const FAILURE_SCAN_CAP = 500;
|
|
14
|
+
export function registerFailureRoutes(router, storage) {
|
|
15
|
+
/**
|
|
16
|
+
* GET /failures
|
|
17
|
+
* Ranked failure list — the dashboard's landing surface. Recent
|
|
18
|
+
* failed/flagged moments ranked by severity × recency decay
|
|
19
|
+
* (see src/eval/failure-rank.ts). Unlike /moments this filters and
|
|
20
|
+
* ranks server-side, so a failure buried behind hundreds of passing
|
|
21
|
+
* traces still surfaces.
|
|
22
|
+
*/
|
|
23
|
+
router.get('/failures', async (req, res) => {
|
|
24
|
+
try {
|
|
25
|
+
const tenantId = requireTenant(req);
|
|
26
|
+
const query = failuresQuerySchema.parse(req.query);
|
|
27
|
+
const traceResult = await storage.queryTraces(tenantId, {
|
|
28
|
+
filter: {
|
|
29
|
+
agent_name: query.agent_name,
|
|
30
|
+
since: query.since,
|
|
31
|
+
until: query.until,
|
|
32
|
+
},
|
|
33
|
+
limit: FAILURE_SCAN_CAP,
|
|
34
|
+
offset: 0,
|
|
35
|
+
sort_by: 'timestamp',
|
|
36
|
+
sort_order: 'desc',
|
|
37
|
+
});
|
|
38
|
+
// Hydrate + classify each scanned trace, keep only failures.
|
|
39
|
+
// Sequential per-trace eval fetches match the moments route's
|
|
40
|
+
// approach — acceptable at this cap; batching is a later
|
|
41
|
+
// optimization once we have volume data.
|
|
42
|
+
const nowMs = Date.now();
|
|
43
|
+
const failures = [];
|
|
44
|
+
for (const trace of traceResult.traces) {
|
|
45
|
+
const evals = await storage.getEvalsByTraceId(tenantId, trace.trace_id);
|
|
46
|
+
const moment = deriveMoment(trace, evals);
|
|
47
|
+
if (!isFailureMoment(moment))
|
|
48
|
+
continue;
|
|
49
|
+
failures.push({ ...moment, rankScore: rankFailureScore(moment, nowMs) });
|
|
50
|
+
}
|
|
51
|
+
// Rank: severity × recency blend first, newest first on exact ties.
|
|
52
|
+
failures.sort((a, b) => {
|
|
53
|
+
if (b.rankScore !== a.rankScore)
|
|
54
|
+
return b.rankScore - a.rankScore;
|
|
55
|
+
return new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime();
|
|
56
|
+
});
|
|
57
|
+
const result = {
|
|
58
|
+
failures: failures.slice(0, query.limit),
|
|
59
|
+
scanned: traceResult.traces.length,
|
|
60
|
+
total: traceResult.total,
|
|
61
|
+
limit: query.limit,
|
|
62
|
+
};
|
|
63
|
+
res.json(result);
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
if (err instanceof Error && err.name === 'ZodError') {
|
|
67
|
+
res.status(400).json({
|
|
68
|
+
error: 'Invalid query parameters',
|
|
69
|
+
details: err.issues,
|
|
70
|
+
});
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
throw err;
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
}
|
|
@@ -5,6 +5,7 @@ export { registerEvalStatsRoutes } from './eval-stats.js';
|
|
|
5
5
|
export { registerFilterRoutes } from './filters.js';
|
|
6
6
|
export { registerHealthRoutes } from './health.js';
|
|
7
7
|
export { registerMomentRoutes } from './moments.js';
|
|
8
|
+
export { registerFailureRoutes } from './failures.js';
|
|
8
9
|
export { registerRuleRoutes } from './rules.js';
|
|
9
10
|
export { registerPreferencesRoutes } from './preferences.js';
|
|
10
11
|
export { registerAuditRoutes } from './audit.js';
|
|
@@ -5,6 +5,7 @@ export { registerEvalStatsRoutes } from './eval-stats.js';
|
|
|
5
5
|
export { registerFilterRoutes } from './filters.js';
|
|
6
6
|
export { registerHealthRoutes } from './health.js';
|
|
7
7
|
export { registerMomentRoutes } from './moments.js';
|
|
8
|
+
export { registerFailureRoutes } from './failures.js';
|
|
8
9
|
export { registerRuleRoutes } from './rules.js';
|
|
9
10
|
export { registerPreferencesRoutes } from './preferences.js';
|
|
10
11
|
export { registerAuditRoutes } from './audit.js';
|
|
@@ -32,14 +32,19 @@ const PatchSchema = z
|
|
|
32
32
|
})
|
|
33
33
|
.strict();
|
|
34
34
|
export function registerPreferencesRoutes(router, store) {
|
|
35
|
+
/*
|
|
36
|
+
* Responses deliberately omit store.path: the absolute path embeds the
|
|
37
|
+
* OS username (install-path disclosure, CWE-209 — same class as PR
|
|
38
|
+
* #286) and the frontend never read it.
|
|
39
|
+
*/
|
|
35
40
|
router.get('/preferences', (_req, res) => {
|
|
36
|
-
res.json({ preferences: store.read()
|
|
41
|
+
res.json({ preferences: store.read() });
|
|
37
42
|
});
|
|
38
43
|
router.patch('/preferences', (req, res) => {
|
|
39
44
|
try {
|
|
40
45
|
const patch = PatchSchema.parse(req.body);
|
|
41
46
|
const updated = store.patch(patch);
|
|
42
|
-
res.json({ preferences: updated
|
|
47
|
+
res.json({ preferences: updated });
|
|
43
48
|
}
|
|
44
49
|
catch (err) {
|
|
45
50
|
if (err instanceof z.ZodError) {
|
|
@@ -69,10 +69,12 @@ export function registerRuleRoutes(router, storage, opts) {
|
|
|
69
69
|
sourceMomentId: input.sourceMomentId,
|
|
70
70
|
});
|
|
71
71
|
// Register the new rule with the live engine so it fires on subsequent
|
|
72
|
-
// evaluate_output calls without requiring a server restart.
|
|
72
|
+
// evaluate_output calls without requiring a server restart. Registered
|
|
73
|
+
// under its rule id so the delete paths can hot-remove it. The engine
|
|
73
74
|
// is process-global in v0.4 — Cloud multi-tenant engine wiring is a
|
|
74
|
-
// v0.5 architectural item.
|
|
75
|
-
|
|
75
|
+
// v0.5 architectural item. Severity rides along: high/critical makes
|
|
76
|
+
// the rule hard-failing (same as the MCP deploy path and boot loading).
|
|
77
|
+
opts.evalEngine.registerRule(rule.evalType, createCustomRule(rule.definition, rule.severity), rule.id);
|
|
76
78
|
res.status(201).json({ rule });
|
|
77
79
|
}
|
|
78
80
|
catch (err) {
|
|
@@ -90,10 +92,11 @@ export function registerRuleRoutes(router, storage, opts) {
|
|
|
90
92
|
res.status(404).json({ error: 'Rule not found' });
|
|
91
93
|
return;
|
|
92
94
|
}
|
|
93
|
-
//
|
|
94
|
-
// the
|
|
95
|
-
//
|
|
96
|
-
//
|
|
95
|
+
// Hot-remove from the live engine too, so the deleted rule stops firing
|
|
96
|
+
// on the very next evaluate_output call — no restart needed. No-op when
|
|
97
|
+
// the rule was never registered in this process (e.g. deployed under a
|
|
98
|
+
// different tenant, or the id predates id-tracked registration).
|
|
99
|
+
opts.evalEngine.unregisterRule(req.params.id);
|
|
97
100
|
res.status(204).end();
|
|
98
101
|
});
|
|
99
102
|
router.post('/rules/custom/preview', async (req, res) => {
|
|
@@ -126,14 +129,16 @@ async function previewRule(tenantId, input, storage) {
|
|
|
126
129
|
sort_order: 'desc',
|
|
127
130
|
});
|
|
128
131
|
const rule = createCustomRule(input.definition);
|
|
129
|
-
// Sanity-probe the rule against an empty input.
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
//
|
|
132
|
+
// Sanity-probe the rule against an empty input. A broken DEFINITION —
|
|
133
|
+
// invalid regex, pattern too long, ReDoS rejection, missing required
|
|
134
|
+
// config — comes back marked configInvalid (custom.ts routes every
|
|
135
|
+
// compile/config failure through configError, which also sets skipped,
|
|
136
|
+
// so probing passed/message here would never fire). Config errors depend
|
|
137
|
+
// only on the definition, never the trace, so one probe hit means every
|
|
138
|
+
// trace would "skip" identically. Surface as a 422 so the UI can show
|
|
139
|
+
// the error instead of a misleading "N traces would skip."
|
|
133
140
|
const probe = rule.evaluate({ output: '' });
|
|
134
|
-
if (
|
|
135
|
-
!probe.passed &&
|
|
136
|
-
/^(?:Invalid regex|Regex pattern (?:too long|rejected))/.test(probe.message)) {
|
|
141
|
+
if (probe.configInvalid) {
|
|
137
142
|
const err = new Error(probe.message);
|
|
138
143
|
err.status = 422;
|
|
139
144
|
throw err;
|
|
@@ -142,6 +147,18 @@ async function previewRule(tenantId, input, storage) {
|
|
|
142
147
|
let wouldFail = 0;
|
|
143
148
|
let wouldSkip = 0;
|
|
144
149
|
const examples = [];
|
|
150
|
+
/*
|
|
151
|
+
* ONE regex budget for the whole preview, not one per trace. This loop
|
|
152
|
+
* runs a caller-supplied pattern against up to maxTraces (cap 5000)
|
|
153
|
+
* seedable outputs on the main thread — with no shared breaker, a
|
|
154
|
+
* sandbox-defeating pattern×output pair cost ~142ms per trace, ~12
|
|
155
|
+
* minutes of server freeze at the cap, from one self-serve request
|
|
156
|
+
* (the deploy probe guesses payloads and cannot catch every such
|
|
157
|
+
* pattern). Sharing the breaker means at most 3 traces pay the budget;
|
|
158
|
+
* the rest report wouldSkip instantly — and "this pattern gets
|
|
159
|
+
* defeated" is exactly the answer the rule author needs from a preview.
|
|
160
|
+
*/
|
|
161
|
+
const regexBudget = { breaches: 0 };
|
|
145
162
|
for (const trace of traceResult.traces) {
|
|
146
163
|
if (trace.output === undefined) {
|
|
147
164
|
wouldSkip++;
|
|
@@ -152,6 +169,7 @@ async function previewRule(tenantId, input, storage) {
|
|
|
152
169
|
input: trace.input,
|
|
153
170
|
costUsd: trace.cost_usd,
|
|
154
171
|
tokenUsage: trace.token_usage,
|
|
172
|
+
regexBudget,
|
|
155
173
|
});
|
|
156
174
|
if (result.skipped) {
|
|
157
175
|
wouldSkip++;
|
|
@@ -1,3 +1,14 @@
|
|
|
1
1
|
import { Router } from 'express';
|
|
2
2
|
import type { IStorageAdapter } from '../../types/query.js';
|
|
3
|
-
|
|
3
|
+
import type { EvalEngine } from '../../eval/engine.js';
|
|
4
|
+
export interface TraceRouteOptions {
|
|
5
|
+
/**
|
|
6
|
+
* Live engine for the `evaluate: true` opt-in on POST /traces. When
|
|
7
|
+
* absent (an embedder that wired storage but no engine), an evaluate
|
|
8
|
+
* request is refused with 501 BEFORE the trace is stored — silently
|
|
9
|
+
* storing without the requested eval would be a skipped gate dressed
|
|
10
|
+
* as a success.
|
|
11
|
+
*/
|
|
12
|
+
evalEngine?: EvalEngine;
|
|
13
|
+
}
|
|
14
|
+
export declare function registerTraceRoutes(router: Router, storage: IStorageAdapter, options?: TraceRouteOptions): void;
|
|
@@ -1,6 +1,94 @@
|
|
|
1
1
|
import { requireTenant } from '../../middleware/tenant.js';
|
|
2
|
-
import {
|
|
3
|
-
|
|
2
|
+
import { generateTraceId, generateSpanId } from '../../utils/ids.js';
|
|
3
|
+
import { bestEffortExport } from '../../otel/lazy.js';
|
|
4
|
+
import { traceQuerySchema, ingestTraceSchema } from '../validation.js';
|
|
5
|
+
export function registerTraceRoutes(router, storage, options) {
|
|
6
|
+
/*
|
|
7
|
+
* Deterministic capture over HTTP. MCP tool calls are model-
|
|
8
|
+
* discretionary — a trace lands only if the model chooses to call
|
|
9
|
+
* log_trace — so builders get a path that doesn't depend on the model:
|
|
10
|
+
* POST the same body the log_trace tool accepts (ingestTraceSchema IS
|
|
11
|
+
* that schema) and the row is stored unconditionally. Sits behind the
|
|
12
|
+
* full middleware stack: loopback bind + DNS-rebinding guard + auth +
|
|
13
|
+
* tenant resolution + the shared API rate limiter.
|
|
14
|
+
*/
|
|
15
|
+
router.post('/traces', async (req, res) => {
|
|
16
|
+
try {
|
|
17
|
+
const tenantId = requireTenant(req);
|
|
18
|
+
const body = ingestTraceSchema.parse(req.body);
|
|
19
|
+
if (body.evaluate && !options?.evalEngine) {
|
|
20
|
+
res.status(501).json({
|
|
21
|
+
error: 'Evaluation is not available on this server — trace was NOT stored. Retry without "evaluate", or start the dashboard via iris-mcp so the eval engine is wired.',
|
|
22
|
+
});
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
// Server-minted, exactly like log_trace — a client-supplied
|
|
26
|
+
// trace_id was already stripped by the schema.
|
|
27
|
+
const traceId = generateTraceId();
|
|
28
|
+
const timestamp = body.timestamp ?? new Date().toISOString();
|
|
29
|
+
const trace = {
|
|
30
|
+
trace_id: traceId,
|
|
31
|
+
agent_name: body.agent_name,
|
|
32
|
+
framework: body.framework,
|
|
33
|
+
input: body.input,
|
|
34
|
+
output: body.output,
|
|
35
|
+
tool_calls: body.tool_calls,
|
|
36
|
+
latency_ms: body.latency_ms,
|
|
37
|
+
token_usage: body.token_usage,
|
|
38
|
+
cost_usd: body.cost_usd,
|
|
39
|
+
metadata: body.metadata,
|
|
40
|
+
timestamp,
|
|
41
|
+
spans: body.spans?.map((s) => ({
|
|
42
|
+
...s,
|
|
43
|
+
span_id: s.span_id ?? generateSpanId(),
|
|
44
|
+
trace_id: traceId,
|
|
45
|
+
})),
|
|
46
|
+
};
|
|
47
|
+
await storage.insertTrace(tenantId, trace);
|
|
48
|
+
// Same best-effort OTel fan-out as log_trace: switching capture
|
|
49
|
+
// paths must not silently drop the operator's collector feed.
|
|
50
|
+
bestEffortExport(trace, (err) => {
|
|
51
|
+
// eslint-disable-next-line no-console
|
|
52
|
+
console.warn(`[iris.otel] ${err.message}`);
|
|
53
|
+
});
|
|
54
|
+
if (!body.evaluate || !options?.evalEngine) {
|
|
55
|
+
res.status(201).json({ trace_id: traceId, status: 'stored' });
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
// Deterministic engine, same context evaluate_output builds. The
|
|
59
|
+
// superRefine on ingestTraceSchema guarantees output is present.
|
|
60
|
+
const evaluation = options.evalEngine.evaluate(body.eval_type, {
|
|
61
|
+
output: body.output,
|
|
62
|
+
input: body.input,
|
|
63
|
+
costUsd: body.cost_usd,
|
|
64
|
+
tokenUsage: body.token_usage,
|
|
65
|
+
});
|
|
66
|
+
evaluation.trace_id = traceId;
|
|
67
|
+
await storage.insertEvalResult(tenantId, evaluation);
|
|
68
|
+
res.status(201).json({
|
|
69
|
+
trace_id: traceId,
|
|
70
|
+
status: 'stored',
|
|
71
|
+
evaluation: {
|
|
72
|
+
id: evaluation.id,
|
|
73
|
+
eval_type: evaluation.eval_type,
|
|
74
|
+
score: evaluation.score,
|
|
75
|
+
passed: evaluation.passed,
|
|
76
|
+
rule_results: evaluation.rule_results,
|
|
77
|
+
suggestions: evaluation.suggestions,
|
|
78
|
+
rules_evaluated: evaluation.rules_evaluated,
|
|
79
|
+
rules_skipped: evaluation.rules_skipped,
|
|
80
|
+
insufficient_data: evaluation.insufficient_data,
|
|
81
|
+
},
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
if (err instanceof Error && err.name === 'ZodError') {
|
|
86
|
+
res.status(400).json({ error: 'Invalid trace payload', details: err.issues });
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
throw err;
|
|
90
|
+
}
|
|
91
|
+
});
|
|
4
92
|
router.get('/traces', async (req, res) => {
|
|
5
93
|
try {
|
|
6
94
|
const tenantId = requireTenant(req);
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export declare const DEFAULT_DEMO_TRACE_COUNT = 250;
|
|
2
|
+
/** The demo trace database. Never the same file as the real iris.db. */
|
|
3
|
+
export declare function demoDbPath(): string;
|
|
4
|
+
/** Demo-scoped dashboard preferences — keeps demo mode out of the real preferences.json. */
|
|
5
|
+
export declare function demoPreferencesPath(): string;
|
|
6
|
+
/** Demo-scoped custom rules — a rule deployed while exploring the demo never lands in custom-rules.json. */
|
|
7
|
+
export declare function demoCustomRulesPath(): string;
|
|
8
|
+
/** Demo-scoped audit log — rule deploy/delete audit entries from demo mode stay out of audit.log. */
|
|
9
|
+
export declare function demoAuditLogPath(): string;
|
|
10
|
+
export interface SeedDemoDataOptions {
|
|
11
|
+
/** Database file to seed. Defaults to demoDbPath() (demo.db under irisHome()). */
|
|
12
|
+
dbPath?: string;
|
|
13
|
+
/** Approximate number of traces to generate. */
|
|
14
|
+
count?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface SeedDemoDataSummary {
|
|
17
|
+
dbPath: string;
|
|
18
|
+
/** True when the database already held traces and was left untouched. */
|
|
19
|
+
alreadySeeded: boolean;
|
|
20
|
+
traceCount: number;
|
|
21
|
+
spanCount: number;
|
|
22
|
+
evalCount: number;
|
|
23
|
+
passedEvalCount: number;
|
|
24
|
+
failedEvalCount: number;
|
|
25
|
+
totalCostUsd: number;
|
|
26
|
+
piiDetectionCount: number;
|
|
27
|
+
injectionDetectionCount: number;
|
|
28
|
+
hallucinationDetectionCount: number;
|
|
29
|
+
costViolationCount: number;
|
|
30
|
+
judgeFailureCount: number;
|
|
31
|
+
agents: Array<{
|
|
32
|
+
name: string;
|
|
33
|
+
traceCount: number;
|
|
34
|
+
evalPassRatePct: number | null;
|
|
35
|
+
}>;
|
|
36
|
+
/** Trace count per day, index 0 = 6 days ago … index 6 = today. */
|
|
37
|
+
dailyTraceCounts: number[];
|
|
38
|
+
}
|
|
39
|
+
/** Delete the entire demo surface. Returns the paths actually removed. */
|
|
40
|
+
export declare function clearDemoData(): {
|
|
41
|
+
removed: string[];
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Seed the demo database. Idempotent: when the database already holds
|
|
45
|
+
* traces, nothing is written and the summary reports alreadySeeded. The
|
|
46
|
+
* demo database is a separate file from the real store — this function
|
|
47
|
+
* never opens iris.db (or whatever IRIS_DB_PATH points at).
|
|
48
|
+
*/
|
|
49
|
+
export declare function seedDemoData(options?: SeedDemoDataOptions): Promise<SeedDemoDataSummary>;
|