modelrig 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/dist/adapters/anthropic.d.ts +4 -4
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +8 -1
- package/dist/adapters/deepseek.d.ts +2 -4
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +9 -5
- package/dist/adapters/gemini.d.ts +4 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +16 -1
- package/dist/adapters/grok.d.ts +2 -4
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +8 -4
- package/dist/adapters/history-mapping.d.ts +126 -0
- package/dist/adapters/history-mapping.d.ts.map +1 -0
- package/dist/adapters/history-mapping.js +230 -0
- package/dist/adapters/openai-compat.d.ts +2 -4
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +9 -5
- package/dist/adapters/openai.d.ts +2 -4
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +8 -4
- package/dist/adapters/responses-api.d.ts.map +1 -1
- package/dist/adapters/responses-api.js +9 -4
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +79 -0
- package/dist/artifacts/conversation.d.ts +128 -0
- package/dist/artifacts/conversation.d.ts.map +1 -0
- package/dist/artifacts/conversation.js +172 -0
- package/dist/artifacts/index.d.ts +1 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/media.d.ts +46 -0
- package/dist/artifacts/media.d.ts.map +1 -0
- package/dist/artifacts/media.js +88 -0
- package/dist/artifacts/run-context.d.ts +61 -2
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +183 -12
- package/dist/artifacts/scrub.d.ts +3 -0
- package/dist/artifacts/scrub.d.ts.map +1 -1
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +34 -7
- package/dist/bakeoff/eval-cases.d.ts +5 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +10 -1
- package/dist/bakeoff/replay.d.ts +15 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +19 -1
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +9 -2
- package/dist/bakeoff/turn-replay.d.ts +93 -0
- package/dist/bakeoff/turn-replay.d.ts.map +1 -0
- package/dist/bakeoff/turn-replay.js +150 -0
- package/dist/bakeoff/types.d.ts +12 -0
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1 -0
- package/dist/config.d.ts +18 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +23 -3
- package/dist/doctor-preflight.d.ts.map +1 -1
- package/dist/doctor-preflight.js +6 -1
- package/dist/feedback/grade.d.ts +7 -2
- package/dist/feedback/grade.d.ts.map +1 -1
- package/dist/feedback/grade.js +14 -0
- package/dist/feedback/outcome.d.ts +103 -0
- package/dist/feedback/outcome.d.ts.map +1 -0
- package/dist/feedback/outcome.js +175 -0
- package/dist/gateway/history-bridge.d.ts +32 -0
- package/dist/gateway/history-bridge.d.ts.map +1 -0
- package/dist/gateway/history-bridge.js +64 -0
- package/dist/gateway/serve-client.d.ts.map +1 -1
- package/dist/gateway/serve-client.js +6 -0
- package/dist/index.d.ts +44 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +110 -9
- package/dist/init/init-cli.d.ts.map +1 -1
- package/dist/init/init-cli.js +21 -4
- package/dist/init/observe-cli.js +1 -1
- package/dist/init/observe.d.ts +7 -1
- package/dist/init/observe.d.ts.map +1 -1
- package/dist/init/observe.js +74 -22
- package/dist/init/starter.d.ts +36 -0
- package/dist/init/starter.d.ts.map +1 -0
- package/dist/init/starter.js +85 -0
- package/dist/registry/build.d.ts +4 -0
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/generation-models.d.ts +68 -0
- package/dist/registry/generation-models.d.ts.map +1 -0
- package/dist/registry/generation-models.js +168 -0
- package/dist/registry/model-kind.d.ts +34 -0
- package/dist/registry/model-kind.d.ts.map +1 -0
- package/dist/registry/model-kind.js +40 -0
- package/dist/registry/pricing-shape.d.ts +29 -1
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +9 -0
- package/dist/registry/pricing.d.ts +26 -0
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +29 -0
- package/dist/registry/rows.d.ts.map +1 -1
- package/dist/registry/rows.js +3 -1
- package/dist/registry/unit-pricing.d.ts +32 -0
- package/dist/registry/unit-pricing.d.ts.map +1 -0
- package/dist/registry/unit-pricing.js +62 -0
- package/dist/routes/input-loader.d.ts +38 -0
- package/dist/routes/input-loader.d.ts.map +1 -0
- package/dist/routes/input-loader.js +80 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +3 -0
- package/dist/routes/resolved-bundle.d.ts +2 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +3 -0
- package/dist/run/cost.d.ts +28 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +74 -0
- package/dist/run/dispatch.d.ts +4 -1
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +3 -0
- package/dist/run/limits.d.ts +88 -0
- package/dist/run/limits.d.ts.map +1 -0
- package/dist/run/limits.js +155 -0
- package/dist/run/messages.d.ts +199 -0
- package/dist/run/messages.d.ts.map +1 -0
- package/dist/run/messages.js +430 -0
- package/dist/run/raw-finalize.d.ts +5 -2
- package/dist/run/raw-finalize.d.ts.map +1 -1
- package/dist/run/raw-finalize.js +8 -5
- package/dist/run/raw-request.d.ts.map +1 -1
- package/dist/run/raw-request.js +2 -1
- package/dist/run/raw.d.ts +10 -4
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +11 -3
- package/dist/run/rawonly.d.ts +29 -5
- package/dist/run/rawonly.d.ts.map +1 -1
- package/dist/run/rawonly.js +54 -19
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +16 -6
- package/dist/run/record-call.d.ts +13 -2
- package/dist/run/record-call.d.ts.map +1 -1
- package/dist/run/record-call.js +50 -7
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +12 -8
- package/dist/telemetry/attempt-meta.d.ts +49 -1
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +45 -1
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +2 -1
- package/dist/telemetry/grade-mappers.d.ts +1 -1
- package/dist/telemetry/grade-mappers.d.ts.map +1 -1
- package/dist/telemetry/grade-mappers.js +7 -1
- package/dist/telemetry/grade-task.d.ts +49 -0
- package/dist/telemetry/grade-task.d.ts.map +1 -0
- package/dist/telemetry/grade-task.js +85 -0
- package/dist/telemetry/grades-sql.d.ts +16 -1
- package/dist/telemetry/grades-sql.d.ts.map +1 -1
- package/dist/telemetry/grades-sql.js +64 -4
- package/dist/telemetry/ingest-protocol.d.ts +24 -1
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +72 -43
- package/dist/telemetry/otel-content.d.ts +31 -0
- package/dist/telemetry/otel-content.d.ts.map +1 -0
- package/dist/telemetry/otel-content.js +106 -0
- package/dist/telemetry/otel-hierarchy.d.ts +57 -0
- package/dist/telemetry/otel-hierarchy.d.ts.map +1 -0
- package/dist/telemetry/otel-hierarchy.js +155 -0
- package/dist/telemetry/otel-normalize.d.ts +15 -1
- package/dist/telemetry/otel-normalize.d.ts.map +1 -1
- package/dist/telemetry/otel-normalize.js +24 -39
- package/dist/telemetry/sinks.d.ts +4 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +53 -34
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +8 -0
- package/dist/telemetry/store.d.ts +9 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +10 -0
- package/dist/telemetry/uniform-rows.d.ts +43 -0
- package/dist/telemetry/uniform-rows.d.ts.map +1 -0
- package/dist/telemetry/uniform-rows.js +139 -0
- package/dist/types/failures-teaching.d.ts.map +1 -1
- package/dist/types/failures-teaching.js +11 -0
- package/dist/types/failures.d.ts +1 -1
- package/dist/types/failures.d.ts.map +1 -1
- package/dist/types/failures.js +2 -0
- package/dist/types/provider.d.ts +12 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/route.d.ts +13 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +14 -1
- package/dist/types/telemetry.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +122 -12
- package/docs/how-it-fits.md +4 -4
- package/docs/instrumentation.md +148 -44
- package/docs/llms.txt +143 -13
- package/docs/mcp-oracle.md +15 -4
- package/docs/quickstart.md +129 -16
- package/package.json +4 -2
- package/registry/registry.json +456 -0
- package/templates/creative-approval.json +81 -0
- package/templates/support-conversation.json +82 -0
package/registry/registry.json
CHANGED
|
@@ -788,6 +788,82 @@
|
|
|
788
788
|
"callNotes": [],
|
|
789
789
|
"facts": null
|
|
790
790
|
},
|
|
791
|
+
{
|
|
792
|
+
"model_key": "black_forest_labs/flux-pro-1.1",
|
|
793
|
+
"kind": "image_gen",
|
|
794
|
+
"declared": {
|
|
795
|
+
"as_of": "2026-07-30",
|
|
796
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"black_forest_labs/flux-pro-1.1\" mode image_generation field output_cost_per_image (upstream: https://bfl.ai/pricing)",
|
|
797
|
+
"pricing": null,
|
|
798
|
+
"pricing_v2": {
|
|
799
|
+
"base": null,
|
|
800
|
+
"tiers": null,
|
|
801
|
+
"context_bands": null,
|
|
802
|
+
"batch_discount": null,
|
|
803
|
+
"modality": {
|
|
804
|
+
"image_per_unit": 0.04,
|
|
805
|
+
"provenance": {
|
|
806
|
+
"source": "litellm-pricing-map@de706a35a6 key \"black_forest_labs/flux-pro-1.1\" mode image_generation field output_cost_per_image (upstream: https://bfl.ai/pricing)",
|
|
807
|
+
"as_of": "2026-07-30"
|
|
808
|
+
}
|
|
809
|
+
},
|
|
810
|
+
"source": "litellm-pricing-map@de706a35a6 key \"black_forest_labs/flux-pro-1.1\" mode image_generation field output_cost_per_image (upstream: https://bfl.ai/pricing)",
|
|
811
|
+
"as_of": "2026-07-30",
|
|
812
|
+
"basis": "per generated image · as filed under the snapshot key · declared-only (unprobed)"
|
|
813
|
+
},
|
|
814
|
+
"flags": {
|
|
815
|
+
"supports_response_schema": null,
|
|
816
|
+
"supports_web_search": null,
|
|
817
|
+
"supports_prompt_caching": null,
|
|
818
|
+
"adapter_structured_native": null,
|
|
819
|
+
"adapter_grounded_native": null,
|
|
820
|
+
"supports_image_input": null
|
|
821
|
+
}
|
|
822
|
+
},
|
|
823
|
+
"probed": null,
|
|
824
|
+
"observed": null,
|
|
825
|
+
"discrepancies": [],
|
|
826
|
+
"callNotes": [],
|
|
827
|
+
"facts": null
|
|
828
|
+
},
|
|
829
|
+
{
|
|
830
|
+
"model_key": "deepgram/nova-3",
|
|
831
|
+
"kind": "stt",
|
|
832
|
+
"declared": {
|
|
833
|
+
"as_of": "2026-07-30",
|
|
834
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"deepgram/nova-3\" mode audio_transcription field input_cost_per_second (upstream: https://deepgram.com/pricing)",
|
|
835
|
+
"pricing": null,
|
|
836
|
+
"pricing_v2": {
|
|
837
|
+
"base": null,
|
|
838
|
+
"tiers": null,
|
|
839
|
+
"context_bands": null,
|
|
840
|
+
"batch_discount": null,
|
|
841
|
+
"modality": {
|
|
842
|
+
"audio_per_second": 0.00007167,
|
|
843
|
+
"provenance": {
|
|
844
|
+
"source": "litellm-pricing-map@de706a35a6 key \"deepgram/nova-3\" mode audio_transcription field input_cost_per_second (upstream: https://deepgram.com/pricing)",
|
|
845
|
+
"as_of": "2026-07-30"
|
|
846
|
+
}
|
|
847
|
+
},
|
|
848
|
+
"source": "litellm-pricing-map@de706a35a6 key \"deepgram/nova-3\" mode audio_transcription field input_cost_per_second (upstream: https://deepgram.com/pricing)",
|
|
849
|
+
"as_of": "2026-07-30",
|
|
850
|
+
"basis": "per second of input audio · as filed under the snapshot key · declared-only (unprobed)"
|
|
851
|
+
},
|
|
852
|
+
"flags": {
|
|
853
|
+
"supports_response_schema": null,
|
|
854
|
+
"supports_web_search": null,
|
|
855
|
+
"supports_prompt_caching": null,
|
|
856
|
+
"adapter_structured_native": null,
|
|
857
|
+
"adapter_grounded_native": null,
|
|
858
|
+
"supports_image_input": null
|
|
859
|
+
}
|
|
860
|
+
},
|
|
861
|
+
"probed": null,
|
|
862
|
+
"observed": null,
|
|
863
|
+
"discrepancies": [],
|
|
864
|
+
"callNotes": [],
|
|
865
|
+
"facts": null
|
|
866
|
+
},
|
|
791
867
|
{
|
|
792
868
|
"model_key": "deepinfra/MiniMaxAI/MiniMax-M3",
|
|
793
869
|
"declared": {
|
|
@@ -3996,6 +4072,44 @@
|
|
|
3996
4072
|
],
|
|
3997
4073
|
"facts": null
|
|
3998
4074
|
},
|
|
4075
|
+
{
|
|
4076
|
+
"model_key": "elevenlabs/eleven_multilingual_v2",
|
|
4077
|
+
"kind": "tts",
|
|
4078
|
+
"declared": {
|
|
4079
|
+
"as_of": "2026-07-30",
|
|
4080
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"elevenlabs/eleven_multilingual_v2\" mode audio_speech field input_cost_per_character (upstream: https://elevenlabs.io/pricing)",
|
|
4081
|
+
"pricing": null,
|
|
4082
|
+
"pricing_v2": {
|
|
4083
|
+
"base": null,
|
|
4084
|
+
"tiers": null,
|
|
4085
|
+
"context_bands": null,
|
|
4086
|
+
"batch_discount": null,
|
|
4087
|
+
"modality": {
|
|
4088
|
+
"characters_per_million": 180,
|
|
4089
|
+
"provenance": {
|
|
4090
|
+
"source": "litellm-pricing-map@de706a35a6 key \"elevenlabs/eleven_multilingual_v2\" mode audio_speech field input_cost_per_character (upstream: https://elevenlabs.io/pricing)",
|
|
4091
|
+
"as_of": "2026-07-30"
|
|
4092
|
+
}
|
|
4093
|
+
},
|
|
4094
|
+
"source": "litellm-pricing-map@de706a35a6 key \"elevenlabs/eleven_multilingual_v2\" mode audio_speech field input_cost_per_character (upstream: https://elevenlabs.io/pricing)",
|
|
4095
|
+
"as_of": "2026-07-30",
|
|
4096
|
+
"basis": "per input character (shown per 1M) · as filed under the snapshot key · declared-only (unprobed)"
|
|
4097
|
+
},
|
|
4098
|
+
"flags": {
|
|
4099
|
+
"supports_response_schema": null,
|
|
4100
|
+
"supports_web_search": null,
|
|
4101
|
+
"supports_prompt_caching": null,
|
|
4102
|
+
"adapter_structured_native": null,
|
|
4103
|
+
"adapter_grounded_native": null,
|
|
4104
|
+
"supports_image_input": null
|
|
4105
|
+
}
|
|
4106
|
+
},
|
|
4107
|
+
"probed": null,
|
|
4108
|
+
"observed": null,
|
|
4109
|
+
"discrepancies": [],
|
|
4110
|
+
"callNotes": [],
|
|
4111
|
+
"facts": null
|
|
4112
|
+
},
|
|
3999
4113
|
{
|
|
4000
4114
|
"model_key": "fireworks/glm-5p2",
|
|
4001
4115
|
"declared": {
|
|
@@ -5433,6 +5547,158 @@
|
|
|
5433
5547
|
]
|
|
5434
5548
|
}
|
|
5435
5549
|
},
|
|
5550
|
+
{
|
|
5551
|
+
"model_key": "gemini/imagen-4.0-fast-generate-001",
|
|
5552
|
+
"kind": "image_gen",
|
|
5553
|
+
"declared": {
|
|
5554
|
+
"as_of": "2026-07-30",
|
|
5555
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"gemini/imagen-4.0-fast-generate-001\" mode image_generation field output_cost_per_image (upstream: https://cloud.google.com/vertex-ai/generative-ai/pricing)",
|
|
5556
|
+
"pricing": null,
|
|
5557
|
+
"pricing_v2": {
|
|
5558
|
+
"base": null,
|
|
5559
|
+
"tiers": null,
|
|
5560
|
+
"context_bands": null,
|
|
5561
|
+
"batch_discount": null,
|
|
5562
|
+
"modality": {
|
|
5563
|
+
"image_per_unit": 0.02,
|
|
5564
|
+
"provenance": {
|
|
5565
|
+
"source": "litellm-pricing-map@de706a35a6 key \"gemini/imagen-4.0-fast-generate-001\" mode image_generation field output_cost_per_image (upstream: https://cloud.google.com/vertex-ai/generative-ai/pricing)",
|
|
5566
|
+
"as_of": "2026-07-30"
|
|
5567
|
+
}
|
|
5568
|
+
},
|
|
5569
|
+
"source": "litellm-pricing-map@de706a35a6 key \"gemini/imagen-4.0-fast-generate-001\" mode image_generation field output_cost_per_image (upstream: https://cloud.google.com/vertex-ai/generative-ai/pricing)",
|
|
5570
|
+
"as_of": "2026-07-30",
|
|
5571
|
+
"basis": "per generated image · as filed under the snapshot key · declared-only (unprobed)"
|
|
5572
|
+
},
|
|
5573
|
+
"flags": {
|
|
5574
|
+
"supports_response_schema": null,
|
|
5575
|
+
"supports_web_search": null,
|
|
5576
|
+
"supports_prompt_caching": null,
|
|
5577
|
+
"adapter_structured_native": null,
|
|
5578
|
+
"adapter_grounded_native": null,
|
|
5579
|
+
"supports_image_input": null
|
|
5580
|
+
}
|
|
5581
|
+
},
|
|
5582
|
+
"probed": null,
|
|
5583
|
+
"observed": null,
|
|
5584
|
+
"discrepancies": [],
|
|
5585
|
+
"callNotes": [],
|
|
5586
|
+
"facts": null
|
|
5587
|
+
},
|
|
5588
|
+
{
|
|
5589
|
+
"model_key": "gemini/imagen-4.0-generate-001",
|
|
5590
|
+
"kind": "image_gen",
|
|
5591
|
+
"declared": {
|
|
5592
|
+
"as_of": "2026-07-30",
|
|
5593
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"gemini/imagen-4.0-generate-001\" mode image_generation field output_cost_per_image (upstream: https://cloud.google.com/vertex-ai/generative-ai/pricing)",
|
|
5594
|
+
"pricing": null,
|
|
5595
|
+
"pricing_v2": {
|
|
5596
|
+
"base": null,
|
|
5597
|
+
"tiers": null,
|
|
5598
|
+
"context_bands": null,
|
|
5599
|
+
"batch_discount": null,
|
|
5600
|
+
"modality": {
|
|
5601
|
+
"image_per_unit": 0.04,
|
|
5602
|
+
"provenance": {
|
|
5603
|
+
"source": "litellm-pricing-map@de706a35a6 key \"gemini/imagen-4.0-generate-001\" mode image_generation field output_cost_per_image (upstream: https://cloud.google.com/vertex-ai/generative-ai/pricing)",
|
|
5604
|
+
"as_of": "2026-07-30"
|
|
5605
|
+
}
|
|
5606
|
+
},
|
|
5607
|
+
"source": "litellm-pricing-map@de706a35a6 key \"gemini/imagen-4.0-generate-001\" mode image_generation field output_cost_per_image (upstream: https://cloud.google.com/vertex-ai/generative-ai/pricing)",
|
|
5608
|
+
"as_of": "2026-07-30",
|
|
5609
|
+
"basis": "per generated image · as filed under the snapshot key · declared-only (unprobed)"
|
|
5610
|
+
},
|
|
5611
|
+
"flags": {
|
|
5612
|
+
"supports_response_schema": null,
|
|
5613
|
+
"supports_web_search": null,
|
|
5614
|
+
"supports_prompt_caching": null,
|
|
5615
|
+
"adapter_structured_native": null,
|
|
5616
|
+
"adapter_grounded_native": null,
|
|
5617
|
+
"supports_image_input": null
|
|
5618
|
+
}
|
|
5619
|
+
},
|
|
5620
|
+
"probed": null,
|
|
5621
|
+
"observed": null,
|
|
5622
|
+
"discrepancies": [],
|
|
5623
|
+
"callNotes": [],
|
|
5624
|
+
"facts": null
|
|
5625
|
+
},
|
|
5626
|
+
{
|
|
5627
|
+
"model_key": "gemini/veo-3.1-fast-generate-preview",
|
|
5628
|
+
"kind": "video_gen",
|
|
5629
|
+
"declared": {
|
|
5630
|
+
"as_of": "2026-07-30",
|
|
5631
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"gemini/veo-3.1-fast-generate-preview\" mode video_generation field output_cost_per_second (upstream: https://ai.google.dev/gemini-api/docs/video)",
|
|
5632
|
+
"pricing": null,
|
|
5633
|
+
"pricing_v2": {
|
|
5634
|
+
"base": null,
|
|
5635
|
+
"tiers": null,
|
|
5636
|
+
"context_bands": null,
|
|
5637
|
+
"batch_discount": null,
|
|
5638
|
+
"modality": {
|
|
5639
|
+
"video_per_second": 0.15,
|
|
5640
|
+
"provenance": {
|
|
5641
|
+
"source": "litellm-pricing-map@de706a35a6 key \"gemini/veo-3.1-fast-generate-preview\" mode video_generation field output_cost_per_second (upstream: https://ai.google.dev/gemini-api/docs/video)",
|
|
5642
|
+
"as_of": "2026-07-30"
|
|
5643
|
+
}
|
|
5644
|
+
},
|
|
5645
|
+
"source": "litellm-pricing-map@de706a35a6 key \"gemini/veo-3.1-fast-generate-preview\" mode video_generation field output_cost_per_second (upstream: https://ai.google.dev/gemini-api/docs/video)",
|
|
5646
|
+
"as_of": "2026-07-30",
|
|
5647
|
+
"basis": "per second of generated video · as filed under the snapshot key · declared-only (unprobed)"
|
|
5648
|
+
},
|
|
5649
|
+
"flags": {
|
|
5650
|
+
"supports_response_schema": null,
|
|
5651
|
+
"supports_web_search": null,
|
|
5652
|
+
"supports_prompt_caching": null,
|
|
5653
|
+
"adapter_structured_native": null,
|
|
5654
|
+
"adapter_grounded_native": null,
|
|
5655
|
+
"supports_image_input": null
|
|
5656
|
+
}
|
|
5657
|
+
},
|
|
5658
|
+
"probed": null,
|
|
5659
|
+
"observed": null,
|
|
5660
|
+
"discrepancies": [],
|
|
5661
|
+
"callNotes": [],
|
|
5662
|
+
"facts": null
|
|
5663
|
+
},
|
|
5664
|
+
{
|
|
5665
|
+
"model_key": "gemini/veo-3.1-generate-preview",
|
|
5666
|
+
"kind": "video_gen",
|
|
5667
|
+
"declared": {
|
|
5668
|
+
"as_of": "2026-07-30",
|
|
5669
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"gemini/veo-3.1-generate-preview\" mode video_generation field output_cost_per_second (upstream: https://ai.google.dev/gemini-api/docs/video)",
|
|
5670
|
+
"pricing": null,
|
|
5671
|
+
"pricing_v2": {
|
|
5672
|
+
"base": null,
|
|
5673
|
+
"tiers": null,
|
|
5674
|
+
"context_bands": null,
|
|
5675
|
+
"batch_discount": null,
|
|
5676
|
+
"modality": {
|
|
5677
|
+
"video_per_second": 0.4,
|
|
5678
|
+
"provenance": {
|
|
5679
|
+
"source": "litellm-pricing-map@de706a35a6 key \"gemini/veo-3.1-generate-preview\" mode video_generation field output_cost_per_second (upstream: https://ai.google.dev/gemini-api/docs/video)",
|
|
5680
|
+
"as_of": "2026-07-30"
|
|
5681
|
+
}
|
|
5682
|
+
},
|
|
5683
|
+
"source": "litellm-pricing-map@de706a35a6 key \"gemini/veo-3.1-generate-preview\" mode video_generation field output_cost_per_second (upstream: https://ai.google.dev/gemini-api/docs/video)",
|
|
5684
|
+
"as_of": "2026-07-30",
|
|
5685
|
+
"basis": "per second of generated video · as filed under the snapshot key · declared-only (unprobed)"
|
|
5686
|
+
},
|
|
5687
|
+
"flags": {
|
|
5688
|
+
"supports_response_schema": null,
|
|
5689
|
+
"supports_web_search": null,
|
|
5690
|
+
"supports_prompt_caching": null,
|
|
5691
|
+
"adapter_structured_native": null,
|
|
5692
|
+
"adapter_grounded_native": null,
|
|
5693
|
+
"supports_image_input": null
|
|
5694
|
+
}
|
|
5695
|
+
},
|
|
5696
|
+
"probed": null,
|
|
5697
|
+
"observed": null,
|
|
5698
|
+
"discrepancies": [],
|
|
5699
|
+
"callNotes": [],
|
|
5700
|
+
"facts": null
|
|
5701
|
+
},
|
|
5436
5702
|
{
|
|
5437
5703
|
"model_key": "grok/grok-4-1-fast",
|
|
5438
5704
|
"declared": {
|
|
@@ -5859,6 +6125,44 @@
|
|
|
5859
6125
|
"callNotes": [],
|
|
5860
6126
|
"facts": null
|
|
5861
6127
|
},
|
|
6128
|
+
{
|
|
6129
|
+
"model_key": "openai/dall-e-3",
|
|
6130
|
+
"kind": "image_gen",
|
|
6131
|
+
"declared": {
|
|
6132
|
+
"as_of": "2026-07-30",
|
|
6133
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"dall-e-3\" mode image_generation field input_cost_per_image",
|
|
6134
|
+
"pricing": null,
|
|
6135
|
+
"pricing_v2": {
|
|
6136
|
+
"base": null,
|
|
6137
|
+
"tiers": null,
|
|
6138
|
+
"context_bands": null,
|
|
6139
|
+
"batch_discount": null,
|
|
6140
|
+
"modality": {
|
|
6141
|
+
"image_per_unit": 0.04,
|
|
6142
|
+
"provenance": {
|
|
6143
|
+
"source": "litellm-pricing-map@de706a35a6 key \"dall-e-3\" mode image_generation field input_cost_per_image",
|
|
6144
|
+
"as_of": "2026-07-30"
|
|
6145
|
+
}
|
|
6146
|
+
},
|
|
6147
|
+
"source": "litellm-pricing-map@de706a35a6 key \"dall-e-3\" mode image_generation field input_cost_per_image",
|
|
6148
|
+
"as_of": "2026-07-30",
|
|
6149
|
+
"basis": "per generated image · as filed under the snapshot key · declared-only (unprobed)"
|
|
6150
|
+
},
|
|
6151
|
+
"flags": {
|
|
6152
|
+
"supports_response_schema": null,
|
|
6153
|
+
"supports_web_search": null,
|
|
6154
|
+
"supports_prompt_caching": null,
|
|
6155
|
+
"adapter_structured_native": null,
|
|
6156
|
+
"adapter_grounded_native": null,
|
|
6157
|
+
"supports_image_input": null
|
|
6158
|
+
}
|
|
6159
|
+
},
|
|
6160
|
+
"probed": null,
|
|
6161
|
+
"observed": null,
|
|
6162
|
+
"discrepancies": [],
|
|
6163
|
+
"callNotes": [],
|
|
6164
|
+
"facts": null
|
|
6165
|
+
},
|
|
5862
6166
|
{
|
|
5863
6167
|
"model_key": "openai/gpt-5-mini",
|
|
5864
6168
|
"declared": {
|
|
@@ -7297,6 +7601,158 @@
|
|
|
7297
7601
|
],
|
|
7298
7602
|
"facts": null
|
|
7299
7603
|
},
|
|
7604
|
+
{
|
|
7605
|
+
"model_key": "openai/sora-2",
|
|
7606
|
+
"kind": "video_gen",
|
|
7607
|
+
"declared": {
|
|
7608
|
+
"as_of": "2026-07-30",
|
|
7609
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"sora-2\" mode video_generation field output_cost_per_video_per_second (upstream: https://platform.openai.com/docs/api-reference/videos)",
|
|
7610
|
+
"pricing": null,
|
|
7611
|
+
"pricing_v2": {
|
|
7612
|
+
"base": null,
|
|
7613
|
+
"tiers": null,
|
|
7614
|
+
"context_bands": null,
|
|
7615
|
+
"batch_discount": null,
|
|
7616
|
+
"modality": {
|
|
7617
|
+
"video_per_second": 0.1,
|
|
7618
|
+
"provenance": {
|
|
7619
|
+
"source": "litellm-pricing-map@de706a35a6 key \"sora-2\" mode video_generation field output_cost_per_video_per_second (upstream: https://platform.openai.com/docs/api-reference/videos)",
|
|
7620
|
+
"as_of": "2026-07-30"
|
|
7621
|
+
}
|
|
7622
|
+
},
|
|
7623
|
+
"source": "litellm-pricing-map@de706a35a6 key \"sora-2\" mode video_generation field output_cost_per_video_per_second (upstream: https://platform.openai.com/docs/api-reference/videos)",
|
|
7624
|
+
"as_of": "2026-07-30",
|
|
7625
|
+
"basis": "per second of generated video · as filed under the snapshot key · declared-only (unprobed)"
|
|
7626
|
+
},
|
|
7627
|
+
"flags": {
|
|
7628
|
+
"supports_response_schema": null,
|
|
7629
|
+
"supports_web_search": null,
|
|
7630
|
+
"supports_prompt_caching": null,
|
|
7631
|
+
"adapter_structured_native": null,
|
|
7632
|
+
"adapter_grounded_native": null,
|
|
7633
|
+
"supports_image_input": null
|
|
7634
|
+
}
|
|
7635
|
+
},
|
|
7636
|
+
"probed": null,
|
|
7637
|
+
"observed": null,
|
|
7638
|
+
"discrepancies": [],
|
|
7639
|
+
"callNotes": [],
|
|
7640
|
+
"facts": null
|
|
7641
|
+
},
|
|
7642
|
+
{
|
|
7643
|
+
"model_key": "openai/tts-1",
|
|
7644
|
+
"kind": "tts",
|
|
7645
|
+
"declared": {
|
|
7646
|
+
"as_of": "2026-07-30",
|
|
7647
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"tts-1\" mode audio_speech field input_cost_per_character",
|
|
7648
|
+
"pricing": null,
|
|
7649
|
+
"pricing_v2": {
|
|
7650
|
+
"base": null,
|
|
7651
|
+
"tiers": null,
|
|
7652
|
+
"context_bands": null,
|
|
7653
|
+
"batch_discount": null,
|
|
7654
|
+
"modality": {
|
|
7655
|
+
"characters_per_million": 15,
|
|
7656
|
+
"provenance": {
|
|
7657
|
+
"source": "litellm-pricing-map@de706a35a6 key \"tts-1\" mode audio_speech field input_cost_per_character",
|
|
7658
|
+
"as_of": "2026-07-30"
|
|
7659
|
+
}
|
|
7660
|
+
},
|
|
7661
|
+
"source": "litellm-pricing-map@de706a35a6 key \"tts-1\" mode audio_speech field input_cost_per_character",
|
|
7662
|
+
"as_of": "2026-07-30",
|
|
7663
|
+
"basis": "per input character (shown per 1M) · as filed under the snapshot key · declared-only (unprobed)"
|
|
7664
|
+
},
|
|
7665
|
+
"flags": {
|
|
7666
|
+
"supports_response_schema": null,
|
|
7667
|
+
"supports_web_search": null,
|
|
7668
|
+
"supports_prompt_caching": null,
|
|
7669
|
+
"adapter_structured_native": null,
|
|
7670
|
+
"adapter_grounded_native": null,
|
|
7671
|
+
"supports_image_input": null
|
|
7672
|
+
}
|
|
7673
|
+
},
|
|
7674
|
+
"probed": null,
|
|
7675
|
+
"observed": null,
|
|
7676
|
+
"discrepancies": [],
|
|
7677
|
+
"callNotes": [],
|
|
7678
|
+
"facts": null
|
|
7679
|
+
},
|
|
7680
|
+
{
|
|
7681
|
+
"model_key": "openai/whisper-1",
|
|
7682
|
+
"kind": "stt",
|
|
7683
|
+
"declared": {
|
|
7684
|
+
"as_of": "2026-07-30",
|
|
7685
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"whisper-1\" mode audio_transcription field input_cost_per_second",
|
|
7686
|
+
"pricing": null,
|
|
7687
|
+
"pricing_v2": {
|
|
7688
|
+
"base": null,
|
|
7689
|
+
"tiers": null,
|
|
7690
|
+
"context_bands": null,
|
|
7691
|
+
"batch_discount": null,
|
|
7692
|
+
"modality": {
|
|
7693
|
+
"audio_per_second": 0.0001,
|
|
7694
|
+
"provenance": {
|
|
7695
|
+
"source": "litellm-pricing-map@de706a35a6 key \"whisper-1\" mode audio_transcription field input_cost_per_second",
|
|
7696
|
+
"as_of": "2026-07-30"
|
|
7697
|
+
}
|
|
7698
|
+
},
|
|
7699
|
+
"source": "litellm-pricing-map@de706a35a6 key \"whisper-1\" mode audio_transcription field input_cost_per_second",
|
|
7700
|
+
"as_of": "2026-07-30",
|
|
7701
|
+
"basis": "per second of input audio · as filed under the snapshot key · declared-only (unprobed)"
|
|
7702
|
+
},
|
|
7703
|
+
"flags": {
|
|
7704
|
+
"supports_response_schema": null,
|
|
7705
|
+
"supports_web_search": null,
|
|
7706
|
+
"supports_prompt_caching": null,
|
|
7707
|
+
"adapter_structured_native": null,
|
|
7708
|
+
"adapter_grounded_native": null,
|
|
7709
|
+
"supports_image_input": null
|
|
7710
|
+
}
|
|
7711
|
+
},
|
|
7712
|
+
"probed": null,
|
|
7713
|
+
"observed": null,
|
|
7714
|
+
"discrepancies": [],
|
|
7715
|
+
"callNotes": [],
|
|
7716
|
+
"facts": null
|
|
7717
|
+
},
|
|
7718
|
+
{
|
|
7719
|
+
"model_key": "stability/sd3.5-large",
|
|
7720
|
+
"kind": "image_gen",
|
|
7721
|
+
"declared": {
|
|
7722
|
+
"as_of": "2026-07-30",
|
|
7723
|
+
"source": "declared-only generation model (unprobed, never ranked) · litellm-pricing-map@de706a35a6 key \"stability/sd3.5-large\" mode image_generation field output_cost_per_image",
|
|
7724
|
+
"pricing": null,
|
|
7725
|
+
"pricing_v2": {
|
|
7726
|
+
"base": null,
|
|
7727
|
+
"tiers": null,
|
|
7728
|
+
"context_bands": null,
|
|
7729
|
+
"batch_discount": null,
|
|
7730
|
+
"modality": {
|
|
7731
|
+
"image_per_unit": 0.065,
|
|
7732
|
+
"provenance": {
|
|
7733
|
+
"source": "litellm-pricing-map@de706a35a6 key \"stability/sd3.5-large\" mode image_generation field output_cost_per_image",
|
|
7734
|
+
"as_of": "2026-07-30"
|
|
7735
|
+
}
|
|
7736
|
+
},
|
|
7737
|
+
"source": "litellm-pricing-map@de706a35a6 key \"stability/sd3.5-large\" mode image_generation field output_cost_per_image",
|
|
7738
|
+
"as_of": "2026-07-30",
|
|
7739
|
+
"basis": "per generated image · as filed under the snapshot key · declared-only (unprobed)"
|
|
7740
|
+
},
|
|
7741
|
+
"flags": {
|
|
7742
|
+
"supports_response_schema": null,
|
|
7743
|
+
"supports_web_search": null,
|
|
7744
|
+
"supports_prompt_caching": null,
|
|
7745
|
+
"adapter_structured_native": null,
|
|
7746
|
+
"adapter_grounded_native": null,
|
|
7747
|
+
"supports_image_input": null
|
|
7748
|
+
}
|
|
7749
|
+
},
|
|
7750
|
+
"probed": null,
|
|
7751
|
+
"observed": null,
|
|
7752
|
+
"discrepancies": [],
|
|
7753
|
+
"callNotes": [],
|
|
7754
|
+
"facts": null
|
|
7755
|
+
},
|
|
7300
7756
|
{
|
|
7301
7757
|
"model_key": "zai/glm-5p2",
|
|
7302
7758
|
"declared": {
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
{
|
|
2
|
+
"use_case": "creative-approval",
|
|
3
|
+
"title": "Creative approval",
|
|
4
|
+
"summary": "Approve or reject generated images, video and audio the way a creative lead reviews a batch: on brief, safe for your brand, any text legible, no visible generation defects, and usable without another round.",
|
|
5
|
+
"fit_note": "Creative approval fits the shipped loop when four conditions hold. (1) Each generation call is recorded with rig.recordCall({ ..., units }) — images, seconds or characters — so it is priced per unit from a sourced rate (or flagged unpriced, never guessed), and each asset it produced is saved as an artifact of type image/*, video/* or audio/* in the same step. (2) Someone on your creative team can write down what on brief and on brand mean for your work, which is the grading guide below. (3) That person approves or rejects assets in the Grade queue's Media approval tab, so the approval rate per dollar on Costs is measured from real verdicts; it is withheld until a model has ten approvals. (4) You accept that generated media is stored UNSCREENED: ModelRig's PII/PHI scrub reads text only, so do not generate or store images of real people's documents or other regulated content through it. There is no automated image scoring in this template; the verdicts are human. It is a starting point, not a claim that any model produces usable creative — fill in your brief and brand rules before you rely on it.",
|
|
6
|
+
"starter_schema": {
|
|
7
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
8
|
+
"$comment": "STARTER SCHEMA for the RECORD of one generated asset (the brief and what was produced), not for the media bytes themselves — the asset is saved as its own image/*, video/* or audio/* artifact. Replace every TODO with your own values before use. The template runs as-is: free-text fields validate immediately.",
|
|
9
|
+
"title": "CreativeAsset",
|
|
10
|
+
"type": "object",
|
|
11
|
+
"additionalProperties": false,
|
|
12
|
+
"required": ["brief", "format", "asset_artifact_id"],
|
|
13
|
+
"properties": {
|
|
14
|
+
"brief": {
|
|
15
|
+
"type": "string",
|
|
16
|
+
"description": "The creative brief or prompt the asset was generated from — what the reviewer checks on-brief against."
|
|
17
|
+
},
|
|
18
|
+
"format": {
|
|
19
|
+
"type": "string",
|
|
20
|
+
"description": "The placement the asset is for (a hero banner, a social square, a voice-over). TODO(enum): constrain this to YOUR own placement names by adding an \"enum\": [ ... ] here. It runs as free text immediately.",
|
|
21
|
+
"$comment": "TODO(enum): add your placement names as the enum for this field."
|
|
22
|
+
},
|
|
23
|
+
"asset_artifact_id": {
|
|
24
|
+
"type": "string",
|
|
25
|
+
"description": "The id of the image/video/audio artifact this record describes (the handle rig.artifacts.save returned)."
|
|
26
|
+
},
|
|
27
|
+
"required_text": {
|
|
28
|
+
"type": "string",
|
|
29
|
+
"description": "Optional: any words that must appear in the asset exactly (a headline, a product name), so legibility can be checked against them."
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"starter_guide": {
|
|
34
|
+
"criteria": [
|
|
35
|
+
{
|
|
36
|
+
"id": "on_brief",
|
|
37
|
+
"title": "Matches the brief",
|
|
38
|
+
"pass": "the asset shows what the brief asked for — subject, setting, mood and placement — without the reviewer having to squint or explain it away.",
|
|
39
|
+
"fail": "The asset misses or contradicts the brief: the wrong subject, a missing element the brief named, or a mood the brief ruled out."
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"id": "brand_safe",
|
|
43
|
+
"title": "Safe for your brand",
|
|
44
|
+
"pass": "nothing in the asset conflicts with your brand rules: colours, logo use, imagery you avoid, and anything that could embarrass the brand or offend its audience.",
|
|
45
|
+
"fail": "The asset breaks a brand rule, misuses a logo or trademark, or includes imagery your brand would not publish."
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"id": "legible_text",
|
|
49
|
+
"title": "Any text is legible and correct",
|
|
50
|
+
"pass": "every word that appears in the asset is spelled correctly, readable at the placement's size, and matches the required text exactly when there is one.",
|
|
51
|
+
"fail": "Text in the asset is garbled, misspelled, unreadable at size, or differs from the required text."
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"id": "no_artifacts",
|
|
55
|
+
"title": "No visible generation defects",
|
|
56
|
+
"pass": "the asset has no obvious generation defects: anatomy, hands, faces and objects look right, edges are clean, and audio or video has no glitches.",
|
|
57
|
+
"fail": "The asset shows a visible defect — extra or fused fingers, warped faces or objects, smeared edges, audio clicks, or video flicker — that a viewer would notice."
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"id": "usable_as_is",
|
|
61
|
+
"title": "Usable without another round",
|
|
62
|
+
"pass": "the asset could ship in its placement as it is, or with a trim or crop at most.",
|
|
63
|
+
"fail": "The asset needs a regeneration or real retouching before it could be used."
|
|
64
|
+
}
|
|
65
|
+
],
|
|
66
|
+
"teach_grade_ids": [],
|
|
67
|
+
"escalation": {
|
|
68
|
+
"epsilon": 0.1
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
"golden_checklist": [
|
|
72
|
+
"One clearly approved asset per placement you generate for — the ordinary case your team signs off every week.",
|
|
73
|
+
"An asset that is beautiful but off-brief, so the guide learns that quality does not excuse a missed brief.",
|
|
74
|
+
"An asset with text in it: one where the words are exactly right, and one with a single misspelled or garbled word.",
|
|
75
|
+
"An asset with a subtle generation defect (a hand, an edge, a reflection) that a quick glance misses.",
|
|
76
|
+
"An asset that breaks one of your brand rules while looking fine otherwise.",
|
|
77
|
+
"An asset that needed only a crop (usable as is) next to one that needed a regeneration.",
|
|
78
|
+
"Aim for roughly twenty goldens total, weighted toward the borderline cases above rather than the obvious approvals."
|
|
79
|
+
],
|
|
80
|
+
"escalation_starter": "Start by sending to a second reviewer every asset destined for a paid or high-visibility placement, every asset with text in it, and every asset a reviewer approved with a comment. The machine-side band lives in starter_guide.escalation.epsilon: a grade within that distance of your task's pass threshold is treated as grader-uncertain and sent to the human queue. The 0.1 here is a starting band — widen it while you build trust, narrow it as your grader calibrates."
|
|
81
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
{
|
|
2
|
+
"use_case": "support-conversation",
|
|
3
|
+
"title": "Support conversation",
|
|
4
|
+
"summary": "Grade a whole support conversation the way a QA lead reviews a ticket: was the customer's problem really solved, inside your policies, with the right tools and tone, and handed to a human when it should have been.",
|
|
5
|
+
"fit_note": "Support conversations fit the shipped teach → calibrate → grade → optimize loop when four conditions hold. (1) Each conversation is wrapped in rig.conversation(id, { agent }), so its turns land as one run and the conversation is graded once, whole, when it closes — not turn by turn. (2) Your support lead can write down what a resolved conversation means for your product (not just one that went quiet), which is the grading guide below. (3) Someone who knows the queue can grade a batch of past conversations so the grader for the conversation:<agent> task can be calibrated against them; outcomes you post later (resolved, reopened) show beside it as a receipt, never as the calibration itself. (4) Content capture is on for the agent, so the grader and your reviewers can read the transcript; without it only metadata is graded. This template is a starting point for each of those, not a claim that support is solved — fill in your policies and your actions before you rely on it.",
|
|
6
|
+
"starter_schema": {
|
|
7
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
8
|
+
"$comment": "STARTER SCHEMA for ONE assistant turn in the conversation. Replace every TODO with your own actions before use. The template runs as-is: free-text fields validate immediately. TODO markers show where adding your own enum values tightens grading to your playbook; do not copy example words from the docs.",
|
|
9
|
+
"title": "SupportTurn",
|
|
10
|
+
"type": "object",
|
|
11
|
+
"additionalProperties": false,
|
|
12
|
+
"required": ["reply", "action", "escalate"],
|
|
13
|
+
"properties": {
|
|
14
|
+
"reply": {
|
|
15
|
+
"type": "string",
|
|
16
|
+
"description": "What the assistant says to the customer on this turn."
|
|
17
|
+
},
|
|
18
|
+
"action": {
|
|
19
|
+
"type": "string",
|
|
20
|
+
"description": "The action the assistant takes alongside the reply (answer only, look something up, change something, hand off). TODO(enum): constrain this to YOUR own action names by adding an \"enum\": [ ... ] here. It runs as free text immediately.",
|
|
21
|
+
"$comment": "TODO(enum): add your action names as the enum for this field."
|
|
22
|
+
},
|
|
23
|
+
"escalate": {
|
|
24
|
+
"type": "boolean",
|
|
25
|
+
"description": "True when this turn hands the conversation to a human. Pairs with the escalated-when-needed criterion in the grading guide."
|
|
26
|
+
},
|
|
27
|
+
"policy_refs": {
|
|
28
|
+
"type": "array",
|
|
29
|
+
"items": { "type": "string" },
|
|
30
|
+
"description": "Optional: the ids of your own policy articles the reply relies on, so the grader can check policy adherence against them."
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"starter_guide": {
|
|
35
|
+
"criteria": [
|
|
36
|
+
{
|
|
37
|
+
"id": "resolved_for_real",
|
|
38
|
+
"title": "The problem was actually resolved",
|
|
39
|
+
"pass": "the customer's stated problem is solved by the end of the conversation, by your own definition of solved — not merely that the customer stopped replying.",
|
|
40
|
+
"fail": "The conversation ends with the problem unsolved, deflected, or only acknowledged, or it reads as closed only because the customer went quiet."
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"id": "policy_adherence",
|
|
44
|
+
"title": "Stayed inside your policies",
|
|
45
|
+
"pass": "every commitment, refund, exception, or claim the assistant makes is allowed by your written policies.",
|
|
46
|
+
"fail": "The assistant promises, grants, or states something your policies do not allow, or invents a policy."
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"id": "correct_tool_use",
|
|
50
|
+
"title": "Used the right tools with the right inputs",
|
|
51
|
+
"pass": "the assistant looks up or changes things only through the tools meant for them, with inputs grounded in what the customer said, and does not repeat a call that already answered.",
|
|
52
|
+
"fail": "The assistant calls the wrong tool, calls one with invented or mismatched inputs, loops on the same call, or answers without looking up what it needed to."
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "tone",
|
|
56
|
+
"title": "Tone fits your brand and the customer's state",
|
|
57
|
+
"pass": "the replies are clear, respectful, and match the voice your team uses, including when the customer is frustrated.",
|
|
58
|
+
"fail": "A reply is curt, condescending, robotic, or off-brand, or ignores that the customer is upset."
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"id": "escalated_when_needed",
|
|
62
|
+
"title": "Handed off to a human when it should have",
|
|
63
|
+
"pass": "a conversation that needs a person (a sensitive case, a request outside the assistant's authority, or a customer who asks for one) is escalated promptly.",
|
|
64
|
+
"fail": "The assistant keeps going when it should have handed off, or escalates a routine case it could have resolved."
|
|
65
|
+
}
|
|
66
|
+
],
|
|
67
|
+
"teach_grade_ids": [],
|
|
68
|
+
"escalation": {
|
|
69
|
+
"epsilon": 0.1
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
"golden_checklist": [
|
|
73
|
+
"One clearly resolved conversation per common contact reason — the ordinary case your team handles every day.",
|
|
74
|
+
"A conversation that went quiet without being solved, so the guide learns that silence is not resolution.",
|
|
75
|
+
"A conversation where the customer asked for something your policy forbids, handled correctly — and one handled wrongly.",
|
|
76
|
+
"A tool-heavy conversation (an order or account lookup) with correct inputs, and one where the assistant looped on the same lookup.",
|
|
77
|
+
"A frustrated customer handled well, so tone is judged on the hard cases rather than the easy ones.",
|
|
78
|
+
"A sensitive case (legal, safety, billing dispute, or churn risk) that should have been escalated, and a routine case that should not.",
|
|
79
|
+
"Aim for roughly twenty goldens total, weighted toward the boundaries above rather than the easy centre of each contact reason."
|
|
80
|
+
],
|
|
81
|
+
"escalation_starter": "Start by sending to a human every conversation the assistant escalated, every conversation a customer reopened (post it as an outcome named reopened), and every sensitive conversation regardless of how it ended. The machine-side band lives in starter_guide.escalation.epsilon: a grade within that distance of your conversation task's pass threshold is treated as grader-uncertain and sent to the human queue. The 0.1 here is a starting band — widen it while you build trust, narrow it as your grader calibrates."
|
|
82
|
+
}
|