completion-kit 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +0 -3
- data/app/assets/stylesheets/completion_kit/application.css +353 -37
- data/app/controllers/completion_kit/runs_controller.rb +3 -1
- data/app/models/completion_kit/provider_credential.rb +11 -0
- data/app/models/completion_kit/run.rb +25 -8
- data/app/services/completion_kit/model_discovery_service.rb +99 -22
- data/app/views/completion_kit/provider_credentials/_discovery_status.html.erb +2 -2
- data/app/views/completion_kit/provider_credentials/_form.html.erb +1 -32
- data/app/views/completion_kit/provider_credentials/_models_card.html.erb +70 -0
- data/app/views/completion_kit/responses/show.html.erb +27 -6
- data/app/views/completion_kit/runs/_response_row.html.erb +32 -25
- data/app/views/completion_kit/runs/_status_header.html.erb +4 -29
- data/app/views/completion_kit/runs/_status_panel.html.erb +46 -0
- data/app/views/completion_kit/runs/show.html.erb +1 -1
- data/lib/completion_kit/version.rb +1 -1
- metadata +4 -3
- data/app/views/completion_kit/runs/_progress.html.erb +0 -18
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8246be5476d4ae32ad0f916710ef465f6e4b55733bc939c2325821a1763282a9
|
|
4
|
+
data.tar.gz: 68d5cdc9b13ede743551e2f8a911136a25e1ca4a87c4c932eca6e3242c1897a4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f17b84d559dd1177ba22653666b7db8d88db109f4bc29ce20445e3610f65b9a4b439ef050a74631a58be36f2ae5baa767354af55e874e58ce134aae7b88bc938
|
|
7
|
+
data.tar.gz: ab7d1cac9c158344f54a8808a62bfccae113cafed8073f4c19b8862143f1717f9833d0550954490654f7e0c0aa6da7ff8039ab37ed1ec377a95cb7e443358973
|
data/Rakefile
CHANGED
|
@@ -724,25 +724,200 @@ tr:hover .ck-chip--publish {
|
|
|
724
724
|
color: var(--ck-text);
|
|
725
725
|
}
|
|
726
726
|
|
|
727
|
-
.ck-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
727
|
+
.ck-status-badge {
|
|
728
|
+
display: inline-flex;
|
|
729
|
+
align-items: center;
|
|
730
|
+
gap: 0.55rem;
|
|
731
|
+
padding: 0.35rem 0.7rem 0.35rem 0.65rem;
|
|
732
|
+
margin: 0 0 0.85rem;
|
|
733
|
+
border-radius: 999px;
|
|
734
|
+
border: 1px solid var(--ck-line);
|
|
735
|
+
background: var(--ck-surface-soft);
|
|
731
736
|
font-family: var(--ck-mono);
|
|
737
|
+
font-size: 0.72rem;
|
|
738
|
+
font-weight: 600;
|
|
739
|
+
letter-spacing: 0.16em;
|
|
740
|
+
text-transform: uppercase;
|
|
741
|
+
line-height: 1;
|
|
742
|
+
color: var(--ck-text);
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
.ck-status-badge__dot {
|
|
746
|
+
width: 8px;
|
|
747
|
+
height: 8px;
|
|
748
|
+
border-radius: 50%;
|
|
749
|
+
flex-shrink: 0;
|
|
750
|
+
background: var(--ck-dim);
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
.ck-status-badge--pending {
|
|
754
|
+
border-color: rgba(139, 154, 181, 0.25);
|
|
732
755
|
color: var(--ck-muted);
|
|
756
|
+
}
|
|
757
|
+
.ck-status-badge--pending .ck-status-badge__dot {
|
|
758
|
+
background: var(--ck-dim);
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
.ck-status-badge--running {
|
|
762
|
+
border-color: rgba(6, 182, 212, 0.4);
|
|
763
|
+
background: rgba(6, 182, 212, 0.08);
|
|
764
|
+
color: var(--ck-accent);
|
|
765
|
+
}
|
|
766
|
+
.ck-status-badge--running .ck-status-badge__dot {
|
|
767
|
+
background: var(--ck-accent);
|
|
768
|
+
box-shadow: 0 0 10px rgba(6, 182, 212, 0.7);
|
|
769
|
+
animation: ck-pulse 1.5s ease-in-out infinite;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
.ck-status-badge--completed {
|
|
773
|
+
border-color: rgba(45, 212, 168, 0.35);
|
|
774
|
+
background: rgba(45, 212, 168, 0.08);
|
|
775
|
+
color: var(--ck-success);
|
|
776
|
+
}
|
|
777
|
+
.ck-status-badge--completed .ck-status-badge__dot {
|
|
778
|
+
background: var(--ck-success);
|
|
779
|
+
box-shadow: 0 0 8px rgba(45, 212, 168, 0.6);
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
.ck-status-badge--failed {
|
|
783
|
+
border-color: rgba(248, 113, 113, 0.4);
|
|
784
|
+
background: rgba(248, 113, 113, 0.08);
|
|
785
|
+
color: var(--ck-danger);
|
|
786
|
+
}
|
|
787
|
+
.ck-status-badge--failed .ck-status-badge__dot {
|
|
788
|
+
background: var(--ck-danger);
|
|
789
|
+
box-shadow: 0 0 8px rgba(248, 113, 113, 0.6);
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
.ck-run-status {
|
|
733
793
|
display: flex;
|
|
734
|
-
flex-
|
|
735
|
-
|
|
794
|
+
flex-wrap: wrap;
|
|
795
|
+
align-items: center;
|
|
796
|
+
justify-content: space-between;
|
|
797
|
+
gap: 1.5rem;
|
|
798
|
+
padding: 1.4rem 1.6rem;
|
|
799
|
+
margin: 2rem 0 1.25rem;
|
|
800
|
+
border: 1px solid var(--ck-line);
|
|
801
|
+
border-radius: var(--ck-radius-lg);
|
|
802
|
+
background: var(--ck-surface);
|
|
736
803
|
}
|
|
737
804
|
|
|
738
|
-
.ck-
|
|
805
|
+
.ck-run-status--running {
|
|
806
|
+
border-color: rgba(6, 182, 212, 0.25);
|
|
807
|
+
background: linear-gradient(180deg, rgba(6, 182, 212, 0.05), var(--ck-surface) 70%);
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
.ck-run-status--completed {
|
|
811
|
+
border-color: rgba(45, 212, 168, 0.2);
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
.ck-run-status__metrics {
|
|
815
|
+
display: flex;
|
|
816
|
+
flex-wrap: wrap;
|
|
817
|
+
gap: 2.5rem;
|
|
818
|
+
align-items: flex-start;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
.ck-run-status__metric-label {
|
|
822
|
+
display: block;
|
|
823
|
+
margin: 0 0 0.4rem;
|
|
824
|
+
font-family: var(--ck-mono);
|
|
825
|
+
font-size: 0.7rem;
|
|
826
|
+
font-weight: 500;
|
|
827
|
+
letter-spacing: 0.14em;
|
|
828
|
+
text-transform: uppercase;
|
|
829
|
+
color: var(--ck-dim);
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
.ck-run-status__metric-value {
|
|
739
833
|
display: flex;
|
|
740
|
-
gap: 0.4rem;
|
|
741
834
|
align-items: baseline;
|
|
835
|
+
gap: 0.4rem;
|
|
836
|
+
margin: 0;
|
|
837
|
+
font-family: var(--ck-mono);
|
|
838
|
+
line-height: 1;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
.ck-run-status__metric-num {
|
|
842
|
+
font-size: 1.6rem;
|
|
843
|
+
font-weight: 700;
|
|
844
|
+
letter-spacing: -0.02em;
|
|
845
|
+
color: var(--ck-text);
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
.ck-run-status__metric-total {
|
|
849
|
+
font-size: 1rem;
|
|
850
|
+
color: var(--ck-muted);
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
.ck-run-status__metric-fail {
|
|
854
|
+
margin-left: 0.35rem;
|
|
855
|
+
padding: 0.18rem 0.45rem;
|
|
856
|
+
border: 1px solid rgba(248, 113, 113, 0.35);
|
|
857
|
+
background: rgba(248, 113, 113, 0.08);
|
|
858
|
+
border-radius: 3px;
|
|
859
|
+
font-size: 0.66rem;
|
|
860
|
+
font-weight: 600;
|
|
861
|
+
letter-spacing: 0.12em;
|
|
862
|
+
text-transform: uppercase;
|
|
863
|
+
color: var(--ck-danger);
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
.ck-run-status__action {
|
|
867
|
+
margin: 0;
|
|
868
|
+
display: inline-block;
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
.ck-run-status__retry {
|
|
872
|
+
appearance: none;
|
|
873
|
+
font-family: var(--ck-mono);
|
|
874
|
+
font-size: 0.74rem;
|
|
875
|
+
font-weight: 600;
|
|
876
|
+
letter-spacing: 0.12em;
|
|
877
|
+
text-transform: uppercase;
|
|
878
|
+
padding: 0.7rem 1.1rem;
|
|
879
|
+
border: 1px solid rgba(248, 113, 113, 0.5);
|
|
880
|
+
background: rgba(248, 113, 113, 0.08);
|
|
881
|
+
color: var(--ck-danger);
|
|
882
|
+
border-radius: var(--ck-radius);
|
|
883
|
+
cursor: pointer;
|
|
884
|
+
transition: background 0.15s, border-color 0.15s;
|
|
742
885
|
}
|
|
743
886
|
|
|
744
|
-
.ck-
|
|
887
|
+
.ck-run-status__retry:hover {
|
|
888
|
+
background: rgba(248, 113, 113, 0.16);
|
|
889
|
+
border-color: var(--ck-danger);
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
.ck-chip--retry {
|
|
893
|
+
appearance: none;
|
|
894
|
+
font-family: var(--ck-mono);
|
|
895
|
+
font-size: 0.7rem;
|
|
896
|
+
font-weight: 600;
|
|
897
|
+
letter-spacing: 0.12em;
|
|
898
|
+
text-transform: uppercase;
|
|
899
|
+
padding: 0.25rem 0.55rem;
|
|
900
|
+
border: 1px solid rgba(248, 113, 113, 0.5);
|
|
901
|
+
background: rgba(248, 113, 113, 0.08);
|
|
745
902
|
color: var(--ck-danger);
|
|
903
|
+
border-radius: 4px;
|
|
904
|
+
cursor: pointer;
|
|
905
|
+
transition: background 0.15s;
|
|
906
|
+
text-decoration: none;
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
.ck-chip--retry:hover {
|
|
910
|
+
background: rgba(248, 113, 113, 0.18);
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
@media (max-width: 720px) {
|
|
914
|
+
.ck-run-status {
|
|
915
|
+
flex-direction: column;
|
|
916
|
+
align-items: stretch;
|
|
917
|
+
}
|
|
918
|
+
.ck-run-status__metrics {
|
|
919
|
+
gap: 1.5rem;
|
|
920
|
+
}
|
|
746
921
|
}
|
|
747
922
|
|
|
748
923
|
.ck-model-list-details summary {
|
|
@@ -758,33 +933,157 @@ tr:hover .ck-chip--publish {
|
|
|
758
933
|
content: '▾ ';
|
|
759
934
|
}
|
|
760
935
|
|
|
761
|
-
.ck-model-
|
|
762
|
-
|
|
763
|
-
flex-wrap: wrap;
|
|
764
|
-
gap: 0.3rem;
|
|
765
|
-
margin-top: 0.5rem;
|
|
766
|
-
max-height: 240px;
|
|
767
|
-
overflow-y: auto;
|
|
936
|
+
.ck-model-list__progress {
|
|
937
|
+
margin-top: 0.65rem;
|
|
768
938
|
}
|
|
769
939
|
|
|
770
|
-
.ck-model-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
940
|
+
.ck-model-list__progress .ck-discovery-bar {
|
|
941
|
+
padding: 0;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
.ck-model-table-wrap {
|
|
945
|
+
margin-top: 0.85rem;
|
|
946
|
+
border: 1px solid var(--ck-line);
|
|
947
|
+
border-radius: var(--ck-radius);
|
|
948
|
+
background: var(--ck-surface);
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
.ck-model-table {
|
|
952
|
+
width: 100%;
|
|
953
|
+
border-collapse: collapse;
|
|
775
954
|
font-family: var(--ck-mono);
|
|
955
|
+
font-size: 0.8rem;
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
.ck-model-table thead th {
|
|
959
|
+
position: sticky;
|
|
960
|
+
top: 0;
|
|
961
|
+
z-index: 1;
|
|
962
|
+
padding: 0.5rem 0.85rem;
|
|
963
|
+
background: var(--ck-surface-soft);
|
|
964
|
+
border-bottom: 1px solid var(--ck-line);
|
|
965
|
+
font-size: 0.66rem;
|
|
966
|
+
font-weight: 500;
|
|
967
|
+
letter-spacing: 0.14em;
|
|
968
|
+
text-transform: uppercase;
|
|
969
|
+
color: var(--ck-dim);
|
|
970
|
+
text-align: left;
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
.ck-model-table thead th.ck-model-table__cap {
|
|
974
|
+
text-align: center;
|
|
975
|
+
width: 6.5rem;
|
|
976
|
+
white-space: nowrap;
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
.ck-model-table thead th .ck-info-toggle {
|
|
980
|
+
width: 16px;
|
|
981
|
+
height: 16px;
|
|
776
982
|
font-size: 0.7rem;
|
|
777
|
-
|
|
983
|
+
font-family: var(--ck-sans);
|
|
984
|
+
font-weight: 600;
|
|
985
|
+
letter-spacing: 0;
|
|
986
|
+
text-indent: 0;
|
|
987
|
+
line-height: 1;
|
|
988
|
+
margin-left: 0.4rem;
|
|
989
|
+
vertical-align: middle;
|
|
990
|
+
position: relative;
|
|
991
|
+
top: -1px;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
.ck-model-table tbody tr {
|
|
995
|
+
border-top: 1px solid var(--ck-line);
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
.ck-model-table tbody tr:first-child {
|
|
999
|
+
border-top: 0;
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
.ck-model-table tbody tr:hover {
|
|
1003
|
+
background: var(--ck-surface-hover);
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
.ck-model-table td {
|
|
1007
|
+
padding: 0.55rem 0.85rem;
|
|
1008
|
+
vertical-align: middle;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
.ck-model-table td.ck-model-table__name {
|
|
1012
|
+
color: var(--ck-text);
|
|
1013
|
+
letter-spacing: -0.01em;
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
.ck-model-table td.ck-model-table__cap {
|
|
1017
|
+
text-align: center;
|
|
1018
|
+
width: 6.5rem;
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
.ck-model-table__tick {
|
|
1022
|
+
display: inline-flex;
|
|
1023
|
+
align-items: center;
|
|
1024
|
+
justify-content: center;
|
|
1025
|
+
width: 1.25rem;
|
|
1026
|
+
height: 1.25rem;
|
|
1027
|
+
border-radius: 50%;
|
|
1028
|
+
background: rgba(45, 212, 168, 0.12);
|
|
1029
|
+
color: var(--ck-success);
|
|
1030
|
+
font-size: 0.78rem;
|
|
1031
|
+
font-weight: 700;
|
|
1032
|
+
line-height: 1;
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
.ck-model-table__dash {
|
|
1036
|
+
color: var(--ck-dim);
|
|
1037
|
+
font-size: 0.85rem;
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
.ck-model-list__summary {
|
|
1041
|
+
display: flex;
|
|
1042
|
+
align-items: center;
|
|
1043
|
+
justify-content: space-between;
|
|
1044
|
+
gap: 0.75rem;
|
|
1045
|
+
cursor: pointer;
|
|
1046
|
+
user-select: none;
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
.ck-model-list__summary-label {
|
|
1050
|
+
flex: 1;
|
|
1051
|
+
font-family: var(--ck-mono);
|
|
1052
|
+
font-size: 0.78rem;
|
|
1053
|
+
font-weight: 500;
|
|
1054
|
+
letter-spacing: 0.04em;
|
|
1055
|
+
color: var(--ck-text);
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
.ck-model-list__summary-count {
|
|
1059
|
+
display: inline-block;
|
|
1060
|
+
margin-left: 0.35rem;
|
|
1061
|
+
padding: 0.05rem 0.4rem;
|
|
1062
|
+
border-radius: 999px;
|
|
778
1063
|
background: var(--ck-surface-soft);
|
|
779
1064
|
border: 1px solid var(--ck-line);
|
|
780
|
-
|
|
1065
|
+
font-size: 0.7rem;
|
|
1066
|
+
color: var(--ck-muted);
|
|
1067
|
+
font-weight: 500;
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
.ck-model-list__summary-meta {
|
|
1071
|
+
display: inline-flex;
|
|
1072
|
+
align-items: center;
|
|
1073
|
+
gap: 0.6rem;
|
|
781
1074
|
}
|
|
782
1075
|
|
|
783
|
-
.ck-model-
|
|
784
|
-
font-
|
|
1076
|
+
.ck-model-list__summary-stamp {
|
|
1077
|
+
font-family: var(--ck-mono);
|
|
1078
|
+
font-size: 0.7rem;
|
|
785
1079
|
color: var(--ck-dim);
|
|
786
|
-
|
|
787
|
-
|
|
1080
|
+
letter-spacing: 0.04em;
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
.ck-model-list__refresh {
|
|
1084
|
+
width: 1.6rem;
|
|
1085
|
+
height: 1.6rem;
|
|
1086
|
+
padding: 0;
|
|
788
1087
|
}
|
|
789
1088
|
|
|
790
1089
|
.ck-button--warning {
|
|
@@ -902,12 +1201,17 @@ tr:hover .ck-chip--publish {
|
|
|
902
1201
|
display: grid;
|
|
903
1202
|
gap: 0;
|
|
904
1203
|
max-width: 32rem;
|
|
1204
|
+
margin-bottom: 1.5rem;
|
|
905
1205
|
border: 1px solid var(--ck-line);
|
|
906
1206
|
border-radius: var(--ck-radius-lg);
|
|
907
1207
|
background: var(--ck-surface);
|
|
908
1208
|
overflow: hidden;
|
|
909
1209
|
}
|
|
910
1210
|
|
|
1211
|
+
.ck-run-config + .ck-prompt-preview {
|
|
1212
|
+
margin-top: 0.5rem;
|
|
1213
|
+
}
|
|
1214
|
+
|
|
911
1215
|
.ck-run-config__row {
|
|
912
1216
|
display: flex;
|
|
913
1217
|
align-items: center;
|
|
@@ -2035,20 +2339,32 @@ select.ck-input {
|
|
|
2035
2339
|
.ck-info-popup {
|
|
2036
2340
|
display: none;
|
|
2037
2341
|
position: absolute;
|
|
2038
|
-
top: 100
|
|
2039
|
-
left:
|
|
2040
|
-
|
|
2041
|
-
padding: 0.
|
|
2342
|
+
top: calc(100% + 8px);
|
|
2343
|
+
left: 50%;
|
|
2344
|
+
transform: translateX(-50%);
|
|
2345
|
+
padding: 0.65rem 0.85rem;
|
|
2042
2346
|
background: var(--ck-surface);
|
|
2043
2347
|
border: 1px solid var(--ck-line-strong);
|
|
2044
2348
|
border-radius: var(--ck-radius);
|
|
2045
|
-
font-
|
|
2046
|
-
|
|
2047
|
-
color: var(--ck-text);
|
|
2048
|
-
width: 320px;
|
|
2049
|
-
z-index: 10;
|
|
2349
|
+
font-family: var(--ck-sans);
|
|
2350
|
+
font-size: 0.75rem;
|
|
2050
2351
|
font-weight: 400;
|
|
2051
|
-
|
|
2352
|
+
line-height: 1.45;
|
|
2353
|
+
letter-spacing: normal;
|
|
2354
|
+
text-transform: none;
|
|
2355
|
+
text-align: left;
|
|
2356
|
+
color: var(--ck-text);
|
|
2357
|
+
width: max-content;
|
|
2358
|
+
max-width: 240px;
|
|
2359
|
+
z-index: 50;
|
|
2360
|
+
box-shadow: 0 6px 20px rgba(0, 0, 0, 0.4);
|
|
2361
|
+
pointer-events: auto;
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2364
|
+
.ck-info-popup--right {
|
|
2365
|
+
left: auto;
|
|
2366
|
+
right: 0;
|
|
2367
|
+
transform: none;
|
|
2052
2368
|
}
|
|
2053
2369
|
|
|
2054
2370
|
.ck-suggest-reasoning {
|
|
@@ -45,7 +45,9 @@ module CompletionKit
|
|
|
45
45
|
|
|
46
46
|
def update
|
|
47
47
|
if @run.responses.any?
|
|
48
|
-
|
|
48
|
+
attrs = run_params.except(:metric_ids).to_h
|
|
49
|
+
attrs.delete("name") if attrs["name"].to_s == @run.name.to_s
|
|
50
|
+
new_run = Run.create!(attrs.merge(status: "pending"))
|
|
49
51
|
new_run.replace_metrics!(params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
|
|
50
52
|
redirect_to run_path(new_run), notice: "Saved as a new run. The previous run and its results are preserved."
|
|
51
53
|
elsif @run.update(run_params.except(:metric_ids))
|
|
@@ -75,6 +75,7 @@ module CompletionKit
|
|
|
75
75
|
target: "discovery_status_#{id}",
|
|
76
76
|
html: render_partial("completion_kit/provider_credentials/discovery_status", provider_credential: self)
|
|
77
77
|
)
|
|
78
|
+
broadcast_provider_models
|
|
78
79
|
end
|
|
79
80
|
|
|
80
81
|
def broadcast_discovery_complete
|
|
@@ -82,6 +83,16 @@ module CompletionKit
|
|
|
82
83
|
broadcast_model_dropdowns
|
|
83
84
|
end
|
|
84
85
|
|
|
86
|
+
def broadcast_provider_models
|
|
87
|
+
Turbo::StreamsChannel.broadcast_action_to(
|
|
88
|
+
"completion_kit_provider_#{id}",
|
|
89
|
+
action: "replace",
|
|
90
|
+
target: "provider_models_#{id}",
|
|
91
|
+
method: "morph",
|
|
92
|
+
html: render_partial("completion_kit/provider_credentials/models_card", provider_credential: self)
|
|
93
|
+
)
|
|
94
|
+
end
|
|
95
|
+
|
|
85
96
|
private
|
|
86
97
|
|
|
87
98
|
def enqueue_discovery
|
|
@@ -119,12 +119,28 @@ module CompletionKit
|
|
|
119
119
|
generated_total = progress_total
|
|
120
120
|
|
|
121
121
|
metric_count = metrics.count
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
.where(
|
|
122
|
+
judged_total = metric_count > 0 ? generated_done : 0
|
|
123
|
+
judged_done = 0
|
|
124
|
+
judged_failed = 0
|
|
125
|
+
|
|
126
|
+
if metric_count > 0 && judged_total > 0
|
|
127
|
+
succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
|
|
128
|
+
metric_ids = metrics.pluck(:id)
|
|
129
|
+
review_counts = Review
|
|
130
|
+
.where(response_id: succeeded_response_ids, metric_id: metric_ids)
|
|
131
|
+
.group(:response_id, :status)
|
|
132
|
+
.count
|
|
133
|
+
succeeded_response_ids.each do |rid|
|
|
134
|
+
ok = review_counts[[rid, "succeeded"]] || 0
|
|
135
|
+
bad = review_counts[[rid, "failed"]] || 0
|
|
136
|
+
next unless ok + bad == metric_count
|
|
137
|
+
if bad > 0
|
|
138
|
+
judged_failed += 1
|
|
139
|
+
else
|
|
140
|
+
judged_done += 1
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
128
144
|
|
|
129
145
|
{
|
|
130
146
|
generated_done: generated_done,
|
|
@@ -175,6 +191,7 @@ module CompletionKit
|
|
|
175
191
|
end
|
|
176
192
|
|
|
177
193
|
def render_engine_partial(partial, locals)
|
|
194
|
+
CompletionKit::Engine.routes.url_helpers
|
|
178
195
|
CompletionKit::ApplicationController.render(
|
|
179
196
|
partial: partial,
|
|
180
197
|
locals: locals
|
|
@@ -185,8 +202,8 @@ module CompletionKit
|
|
|
185
202
|
reload
|
|
186
203
|
broadcast_replace_to(
|
|
187
204
|
"completion_kit_run_#{id}",
|
|
188
|
-
target: "
|
|
189
|
-
html: render_engine_partial("completion_kit/runs/
|
|
205
|
+
target: "run_status_panel",
|
|
206
|
+
html: render_engine_partial("completion_kit/runs/status_panel", run: self)
|
|
190
207
|
)
|
|
191
208
|
broadcast_status_header
|
|
192
209
|
end
|
|
@@ -13,7 +13,6 @@ module CompletionKit
|
|
|
13
13
|
def refresh!(&on_progress)
|
|
14
14
|
models_with_names = fetch_models
|
|
15
15
|
reconcile(models_with_names)
|
|
16
|
-
return if %w[openrouter ollama].include?(@provider)
|
|
17
16
|
probe_new_models(&on_progress)
|
|
18
17
|
end
|
|
19
18
|
|
|
@@ -111,22 +110,44 @@ module CompletionKit
|
|
|
111
110
|
end
|
|
112
111
|
|
|
113
112
|
def probe_new_models(&on_progress)
|
|
114
|
-
|
|
115
|
-
|
|
113
|
+
candidates = Model.where(provider: @provider, status: %w[active failed])
|
|
114
|
+
.where("supports_generation IS NULL OR supports_judging IS NULL OR (generation_error IS NOT NULL AND #{retryable_error_sql('generation_error')}) OR (judging_error IS NOT NULL AND #{retryable_error_sql('judging_error')})")
|
|
115
|
+
total = candidates.count
|
|
116
116
|
current = 0
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
117
|
+
candidates.find_each do |model|
|
|
118
|
+
probed = false
|
|
119
|
+
if model.supports_generation.nil? || retryable_error?(model.generation_error)
|
|
120
|
+
model.generation_error = nil
|
|
121
|
+
probe_generation(model)
|
|
122
|
+
probed = true
|
|
123
|
+
end
|
|
124
|
+
if model.supports_generation && (model.supports_judging.nil? || retryable_error?(model.judging_error))
|
|
125
|
+
model.judging_error = nil
|
|
126
|
+
probe_judging(model)
|
|
127
|
+
probed = true
|
|
128
|
+
end
|
|
129
|
+
if probed
|
|
130
|
+
model.probed_at = Time.current
|
|
131
|
+
model.status = (model.supports_generation == false ? "failed" : "active")
|
|
132
|
+
model.save!
|
|
133
|
+
end
|
|
123
134
|
current += 1
|
|
124
135
|
on_progress&.call(current, total)
|
|
125
136
|
end
|
|
126
137
|
end
|
|
127
138
|
|
|
139
|
+
def retryable_error?(error_string)
|
|
140
|
+
return false if error_string.blank?
|
|
141
|
+
return true if error_string.start_with?("429 ")
|
|
142
|
+
!error_string.match?(/\A4\d\d\s/)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def retryable_error_sql(column)
|
|
146
|
+
"(#{column} LIKE '429 -%' OR #{column} NOT LIKE '4__ -%')"
|
|
147
|
+
end
|
|
148
|
+
|
|
128
149
|
def probe_generation(model)
|
|
129
|
-
response = send_probe(model.model_id, "Say hello",
|
|
150
|
+
response = send_probe(model.model_id, "Say hello", 65536)
|
|
130
151
|
if response.success?
|
|
131
152
|
text = extract_text(response)
|
|
132
153
|
if text.present?
|
|
@@ -154,7 +175,7 @@ module CompletionKit
|
|
|
154
175
|
AI output to evaluate: The sky is blue.
|
|
155
176
|
PROMPT
|
|
156
177
|
|
|
157
|
-
response = send_probe(model.model_id, judge_input,
|
|
178
|
+
response = send_probe(model.model_id, judge_input, 65536)
|
|
158
179
|
if response.success?
|
|
159
180
|
text = extract_text(response).to_s
|
|
160
181
|
if text.match?(/Score:\s*\d/i)
|
|
@@ -173,37 +194,60 @@ module CompletionKit
|
|
|
173
194
|
end
|
|
174
195
|
|
|
175
196
|
def send_probe(model_id, input, max_tokens)
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
197
|
+
case @provider
|
|
198
|
+
when "openai" then openai_probe(model_id, input, max_tokens)
|
|
199
|
+
when "anthropic" then anthropic_probe(model_id, input, max_tokens)
|
|
200
|
+
when "openrouter" then openrouter_probe(model_id, input, max_tokens)
|
|
201
|
+
when "ollama" then ollama_probe(model_id, input, max_tokens)
|
|
202
|
+
else raise ArgumentError, "Unsupported probe provider: #{@provider}"
|
|
180
203
|
end
|
|
181
204
|
end
|
|
182
205
|
|
|
183
206
|
def extract_text(response)
|
|
184
207
|
data = JSON.parse(response.body)
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
208
|
+
case @provider
|
|
209
|
+
when "openai"
|
|
210
|
+
message = Array(data["output"]).find { |o| o["type"] == "message" }
|
|
211
|
+
message&.dig("content", 0, "text")
|
|
212
|
+
when "anthropic" then data.dig("content", 0, "text")
|
|
213
|
+
else data.dig("choices", 0, "message", "content")
|
|
189
214
|
end
|
|
190
215
|
end
|
|
191
216
|
|
|
192
217
|
def openai_probe(model_id, input, max_tokens)
|
|
193
|
-
conn =
|
|
194
|
-
|
|
218
|
+
conn = openai_probe_connection
|
|
219
|
+
response = openai_probe_post(conn, model_id, input, max_tokens, openai_reasoning_effort_for(model_id))
|
|
220
|
+
if response.status == 400 && response.body.to_s.include?("is not supported with the")
|
|
221
|
+
response = openai_probe_post(conn, model_id, input, max_tokens, nil)
|
|
222
|
+
end
|
|
223
|
+
response
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def openai_probe_connection
|
|
227
|
+
Faraday.new(url: "https://api.openai.com") do |f|
|
|
228
|
+
f.options.timeout = 180
|
|
195
229
|
f.options.open_timeout = 5
|
|
196
230
|
f.request :retry, max: 1, interval: 0.5
|
|
197
231
|
f.adapter Faraday.default_adapter
|
|
198
232
|
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def openai_probe_post(conn, model_id, input, max_tokens, effort)
|
|
236
|
+
body = { model: model_id, input: input, max_output_tokens: max_tokens, store: false }
|
|
237
|
+
body[:reasoning] = { effort: effort } if effort
|
|
199
238
|
conn.post do |req|
|
|
200
239
|
req.url "/v1/responses"
|
|
201
240
|
req.headers["Content-Type"] = "application/json"
|
|
202
241
|
req.headers["Authorization"] = "Bearer #{@api_key}"
|
|
203
|
-
req.body =
|
|
242
|
+
req.body = body.to_json
|
|
204
243
|
end
|
|
205
244
|
end
|
|
206
245
|
|
|
246
|
+
def openai_reasoning_effort_for(model_id)
|
|
247
|
+
return nil unless model_id.to_s.match?(/\A(gpt-5|o1|o3)/)
|
|
248
|
+
"low"
|
|
249
|
+
end
|
|
250
|
+
|
|
207
251
|
def anthropic_probe(model_id, input, max_tokens)
|
|
208
252
|
conn = Faraday.new(url: "https://api.anthropic.com") do |f|
|
|
209
253
|
f.options.timeout = 15
|
|
@@ -219,5 +263,38 @@ module CompletionKit
|
|
|
219
263
|
req.body = { model: model_id, messages: [{ role: "user", content: input }], max_tokens: max_tokens }.to_json
|
|
220
264
|
end
|
|
221
265
|
end
|
|
266
|
+
|
|
267
|
+
def openrouter_probe(model_id, input, max_tokens)
|
|
268
|
+
conn = Faraday.new(url: "https://openrouter.ai") do |f|
|
|
269
|
+
f.options.timeout = 30
|
|
270
|
+
f.options.open_timeout = 5
|
|
271
|
+
f.request :retry, max: 1, interval: 0.5
|
|
272
|
+
f.adapter Faraday.default_adapter
|
|
273
|
+
end
|
|
274
|
+
conn.post do |req|
|
|
275
|
+
req.url "/api/v1/chat/completions"
|
|
276
|
+
req.headers["Content-Type"] = "application/json"
|
|
277
|
+
req.headers["Authorization"] = "Bearer #{@api_key}"
|
|
278
|
+
req.headers["HTTP-Referer"] = "https://completionkit.com"
|
|
279
|
+
req.headers["X-Title"] = "CompletionKit"
|
|
280
|
+
req.body = { model: model_id, messages: [{ role: "user", content: input }], max_tokens: max_tokens }.to_json
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def ollama_probe(model_id, input, max_tokens)
|
|
285
|
+
base_url = @api_endpoint.to_s.delete_suffix("/")
|
|
286
|
+
conn = Faraday.new(url: base_url) do |f|
|
|
287
|
+
f.options.timeout = 60
|
|
288
|
+
f.options.open_timeout = 5
|
|
289
|
+
f.request :retry, max: 1, interval: 0.5
|
|
290
|
+
f.adapter Faraday.default_adapter
|
|
291
|
+
end
|
|
292
|
+
conn.post do |req|
|
|
293
|
+
req.url "/chat/completions"
|
|
294
|
+
req.headers["Content-Type"] = "application/json"
|
|
295
|
+
req.headers["Authorization"] = "Bearer #{@api_key}" if @api_key.present?
|
|
296
|
+
req.body = { model: model_id, messages: [{ role: "user", content: input }], max_tokens: max_tokens }.to_json
|
|
297
|
+
end
|
|
298
|
+
end
|
|
222
299
|
end
|
|
223
300
|
end
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
<div class="ck-discovery-bar">
|
|
4
4
|
<div class="ck-discovery-bar__label">
|
|
5
5
|
<% if provider_credential.discovery_total > 0 %>
|
|
6
|
-
|
|
6
|
+
Checking models… <%= provider_credential.discovery_current %>/<%= provider_credential.discovery_total %>
|
|
7
7
|
<% else %>
|
|
8
|
-
|
|
8
|
+
Looking up models…
|
|
9
9
|
<% end %>
|
|
10
10
|
</div>
|
|
11
11
|
<% if provider_credential.discovery_total > 0 %>
|
|
@@ -36,36 +36,5 @@
|
|
|
36
36
|
|
|
37
37
|
<% if provider_credential.persisted? %>
|
|
38
38
|
<%= turbo_stream_from "completion_kit_provider_#{provider_credential.id}" %>
|
|
39
|
-
|
|
40
|
-
<% models = CompletionKit::Model.where(provider: provider_credential.provider).active.order(:model_id) %>
|
|
41
|
-
<% if models.any? || provider_credential.discovery_status.present? %>
|
|
42
|
-
<div class="ck-card ck-form-card__models">
|
|
43
|
-
<div class="ck-form-card__footer-header">
|
|
44
|
-
<%= render "discovery_status", provider_credential: provider_credential %>
|
|
45
|
-
<button type="button" class="ck-icon-btn" title="Refresh models" aria-label="Refresh available models" onclick="fetch('<%= refresh_provider_credential_path(provider_credential) %>', {method:'POST',headers:{'X-CSRF-Token':document.querySelector('meta[name=csrf-token]').content}})">
|
|
46
|
-
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="14" height="14" aria-hidden="true"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg>
|
|
47
|
-
</button>
|
|
48
|
-
</div>
|
|
49
|
-
|
|
50
|
-
<% if models.any? %>
|
|
51
|
-
<details class="ck-model-list-details">
|
|
52
|
-
<summary class="ck-label" style="cursor: pointer; user-select: none;">Available models (<%= models.count %>)</summary>
|
|
53
|
-
<div class="ck-model-list">
|
|
54
|
-
<% models.each do |m| %>
|
|
55
|
-
<span class="ck-model-list__item">
|
|
56
|
-
<%= m.display_name || m.model_id %>
|
|
57
|
-
<% if m.supports_generation && m.supports_judging %>
|
|
58
|
-
<span class="ck-model-list__badge">gen + judge</span>
|
|
59
|
-
<% elsif m.supports_generation %>
|
|
60
|
-
<span class="ck-model-list__badge">gen</span>
|
|
61
|
-
<% elsif m.supports_judging %>
|
|
62
|
-
<span class="ck-model-list__badge">judge</span>
|
|
63
|
-
<% end %>
|
|
64
|
-
</span>
|
|
65
|
-
<% end %>
|
|
66
|
-
</div>
|
|
67
|
-
</details>
|
|
68
|
-
<% end %>
|
|
69
|
-
</div>
|
|
70
|
-
<% end %>
|
|
39
|
+
<%= render "models_card", provider_credential: provider_credential %>
|
|
71
40
|
<% end %>
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
<div id="provider_models_<%= provider_credential.id %>">
|
|
2
|
+
<% models = CompletionKit::Model.where(provider: provider_credential.provider).active.order(:model_id) %>
|
|
3
|
+
<% if models.any? || provider_credential.discovery_status.present? %>
|
|
4
|
+
<div class="ck-card ck-form-card__models">
|
|
5
|
+
<% if models.none? && provider_credential.discovery_status.in?(%w[discovering failed]) %>
|
|
6
|
+
<%= render "completion_kit/provider_credentials/discovery_status", provider_credential: provider_credential, show_completed: false %>
|
|
7
|
+
<% end %>
|
|
8
|
+
|
|
9
|
+
<% if models.any? %>
|
|
10
|
+
<% discovering = provider_credential.discovery_status == "discovering" %>
|
|
11
|
+
<% recently_completed = provider_credential.discovery_status == "completed" && provider_credential.updated_at > 1.minute.ago %>
|
|
12
|
+
<% expanded = discovering || recently_completed %>
|
|
13
|
+
<details class="ck-model-list-details"<%= " open" if expanded %>>
|
|
14
|
+
<summary class="ck-model-list__summary">
|
|
15
|
+
<span class="ck-model-list__summary-label">Available models <span class="ck-model-list__summary-count"><%= models.count %></span></span>
|
|
16
|
+
<span class="ck-model-list__summary-meta">
|
|
17
|
+
<% if provider_credential.discovery_status == "completed" %>
|
|
18
|
+
<span class="ck-model-list__summary-stamp">updated <%= time_ago_in_words(provider_credential.updated_at) %> ago</span>
|
|
19
|
+
<% end %>
|
|
20
|
+
<button type="button" class="ck-icon-btn ck-model-list__refresh<%= ' ck-icon-btn--spinning' if discovering %>" title="Refresh models" aria-label="Refresh available models" <%= 'disabled' if discovering %> onclick="event.preventDefault();event.stopPropagation();fetch('<%= refresh_provider_credential_path(provider_credential) %>', {method:'POST',headers:{'X-CSRF-Token':document.querySelector('meta[name=csrf-token]').content}})">
|
|
21
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="13" height="13" aria-hidden="true"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg>
|
|
22
|
+
</button>
|
|
23
|
+
</span>
|
|
24
|
+
</summary>
|
|
25
|
+
<% if provider_credential.discovery_status.in?(%w[discovering failed]) %>
|
|
26
|
+
<div class="ck-model-list__progress">
|
|
27
|
+
<%= render "completion_kit/provider_credentials/discovery_status", provider_credential: provider_credential, show_completed: false %>
|
|
28
|
+
</div>
|
|
29
|
+
<% end %>
|
|
30
|
+
<div class="ck-model-table-wrap">
|
|
31
|
+
<table class="ck-model-table">
|
|
32
|
+
<thead>
|
|
33
|
+
<tr>
|
|
34
|
+
<th scope="col" class="ck-model-table__name">Model</th>
|
|
35
|
+
<th scope="col" class="ck-model-table__cap">
|
|
36
|
+
Gen<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup ck-info-popup--right">Generation models produce the responses your prompts ask for. Pick one when creating a prompt.</span>
|
|
37
|
+
</th>
|
|
38
|
+
<th scope="col" class="ck-model-table__cap">
|
|
39
|
+
Judge<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup ck-info-popup--right">Judge models score generated responses against your metrics. Pick one when configuring a run.</span>
|
|
40
|
+
</th>
|
|
41
|
+
</tr>
|
|
42
|
+
</thead>
|
|
43
|
+
<tbody>
|
|
44
|
+
<% models.each do |m| %>
|
|
45
|
+
<tr>
|
|
46
|
+
<td class="ck-model-table__name"><%= m.display_name || m.model_id %></td>
|
|
47
|
+
<td class="ck-model-table__cap">
|
|
48
|
+
<% if m.supports_generation %>
|
|
49
|
+
<span class="ck-model-table__tick" aria-label="Supports generation">✓</span>
|
|
50
|
+
<% else %>
|
|
51
|
+
<span class="ck-model-table__dash" aria-label="No generation support">—</span>
|
|
52
|
+
<% end %>
|
|
53
|
+
</td>
|
|
54
|
+
<td class="ck-model-table__cap">
|
|
55
|
+
<% if m.supports_judging %>
|
|
56
|
+
<span class="ck-model-table__tick" aria-label="Supports judging">✓</span>
|
|
57
|
+
<% else %>
|
|
58
|
+
<span class="ck-model-table__dash" aria-label="No judging support">—</span>
|
|
59
|
+
<% end %>
|
|
60
|
+
</td>
|
|
61
|
+
</tr>
|
|
62
|
+
<% end %>
|
|
63
|
+
</tbody>
|
|
64
|
+
</table>
|
|
65
|
+
</div>
|
|
66
|
+
</details>
|
|
67
|
+
<% end %>
|
|
68
|
+
</div>
|
|
69
|
+
<% end %>
|
|
70
|
+
</div>
|
|
@@ -14,12 +14,6 @@
|
|
|
14
14
|
<span class="<%= ck_badge_classes(ck_score_kind(score)) %>"><%= score %></span>
|
|
15
15
|
<% end %>
|
|
16
16
|
</div>
|
|
17
|
-
<p class="ck-meta-copy">
|
|
18
|
-
<span class="ck-run-config__key">Prompt</span> <%= link_to @run.prompt.display_name, prompt_path(@run.prompt), class: "ck-link" %>
|
|
19
|
-
<% if @run.dataset %>
|
|
20
|
-
 · <span class="ck-run-config__key">Dataset</span> <%= link_to @run.dataset.name, dataset_path(@run.dataset), class: "ck-link" %>
|
|
21
|
-
<% end %>
|
|
22
|
-
</p>
|
|
23
17
|
</div>
|
|
24
18
|
<div class="ck-actions">
|
|
25
19
|
<% if @prev_response %>
|
|
@@ -31,6 +25,33 @@
|
|
|
31
25
|
</div>
|
|
32
26
|
</section>
|
|
33
27
|
|
|
28
|
+
<div class="ck-run-config">
|
|
29
|
+
<div class="ck-run-config__row">
|
|
30
|
+
<span class="ck-run-config__key">Run</span>
|
|
31
|
+
<%= link_to @run.name, run_path(@run), class: "ck-link" %>
|
|
32
|
+
</div>
|
|
33
|
+
<div class="ck-run-config__row">
|
|
34
|
+
<span class="ck-run-config__key">Prompt</span>
|
|
35
|
+
<%= link_to @run.prompt.display_name, prompt_path(@run.prompt), class: "ck-link" %>
|
|
36
|
+
</div>
|
|
37
|
+
<% if @run.dataset %>
|
|
38
|
+
<div class="ck-run-config__row">
|
|
39
|
+
<span class="ck-run-config__key">Dataset</span>
|
|
40
|
+
<%= link_to @run.dataset.name, dataset_path(@run.dataset), class: "ck-link" %>
|
|
41
|
+
</div>
|
|
42
|
+
<% end %>
|
|
43
|
+
<div class="ck-run-config__row">
|
|
44
|
+
<span class="ck-run-config__key">Model</span>
|
|
45
|
+
<span style="text-transform: none;"><%= @run.prompt.llm_model %></span>
|
|
46
|
+
</div>
|
|
47
|
+
<% if @run.judge_model.present? %>
|
|
48
|
+
<div class="ck-run-config__row">
|
|
49
|
+
<span class="ck-run-config__key">Judge</span>
|
|
50
|
+
<span style="text-transform: none;"><%= @run.judge_model %></span>
|
|
51
|
+
</div>
|
|
52
|
+
<% end %>
|
|
53
|
+
</div>
|
|
54
|
+
|
|
34
55
|
<section>
|
|
35
56
|
<p class="ck-kicker">Input</p>
|
|
36
57
|
<pre class="ck-code ck-code--dark"><%= begin; JSON.pretty_generate(JSON.parse(@response.input_data)); rescue; @response.input_data; end %></pre>
|
|
@@ -1,31 +1,38 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
<% elsif response.status == "failed" %>
|
|
7
|
-
<% err = response.error_payload %>
|
|
8
|
-
<span class="ck-response-row__error">
|
|
9
|
-
<%= err && err[:provider]&.titleize %><%= " #{err[:status]}" if err && err[:status] %> — <%= truncate(err && err[:message].to_s, length: 120) %>
|
|
10
|
-
</span>
|
|
11
|
-
<% end %>
|
|
12
|
-
</span>
|
|
13
|
-
<span class="ck-response-row__score">
|
|
14
|
-
<% case response.status
|
|
15
|
-
when "succeeded" %>
|
|
1
|
+
<% if response.succeeded? %>
|
|
2
|
+
<%= link_to run_response_path(run, response, sort: params[:sort]), class: "ck-response-row ck-response-row--succeeded", id: "response_#{response.id}" do %>
|
|
3
|
+
<span class="ck-response-row__index">#<%= index %></span>
|
|
4
|
+
<span class="ck-response-row__text"><%= truncate(response.response_text.to_s, length: 160) %></span>
|
|
5
|
+
<span class="ck-response-row__score">
|
|
16
6
|
<% if response.reviewed? %>
|
|
17
7
|
<span class="ck-score"><span class="ck-score__star">★</span> <%= response.score %></span>
|
|
18
8
|
<% elsif run.status == "running" %>
|
|
19
9
|
<span class="ck-chip">Judging</span>
|
|
20
10
|
<% end %>
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
11
|
+
</span>
|
|
12
|
+
<% end %>
|
|
13
|
+
<% else %>
|
|
14
|
+
<div class="ck-response-row ck-response-row--<%= response.status %>" id="response_<%= response.id %>">
|
|
15
|
+
<span class="ck-response-row__index">#<%= index %></span>
|
|
16
|
+
<span class="ck-response-row__text">
|
|
17
|
+
<% if response.status == "failed" %>
|
|
18
|
+
<% err = response.error_payload %>
|
|
19
|
+
<span class="ck-response-row__error">
|
|
20
|
+
<%= err && err[:provider]&.titleize %><%= " #{err[:status]}" if err && err[:status] %> — <%= truncate(err && err[:message].to_s, length: 120) %>
|
|
21
|
+
</span>
|
|
22
|
+
<% end %>
|
|
23
|
+
</span>
|
|
24
|
+
<span class="ck-response-row__score">
|
|
25
|
+
<% case response.status
|
|
26
|
+
when "pending" %>
|
|
27
|
+
<span class="ck-chip">Queued</span>
|
|
28
|
+
<% when "retrying" %>
|
|
29
|
+
<span class="ck-chip ck-chip--warning">Retrying <%= response.attempts %>/5</span>
|
|
30
|
+
<% when "failed" %>
|
|
31
|
+
<%= button_to "Retry", retry_failures_run_path(run, only: response.id),
|
|
32
|
+
method: :post,
|
|
33
|
+
class: "ck-chip ck-chip--retry",
|
|
34
|
+
form_class: "inline-block" %>
|
|
35
|
+
<% end %>
|
|
36
|
+
</span>
|
|
37
|
+
</div>
|
|
31
38
|
<% end %>
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
<% snap = run.progress_snapshot %>
|
|
2
1
|
<div id="run_status_header">
|
|
3
2
|
<% if run.status == "failed" %>
|
|
4
3
|
<div class="ck-flash ck-flash--alert">
|
|
@@ -14,37 +13,13 @@
|
|
|
14
13
|
|
|
15
14
|
<section class="ck-page-header">
|
|
16
15
|
<div>
|
|
17
|
-
<
|
|
16
|
+
<span class="ck-status-badge ck-status-badge--<%= run.status %>">
|
|
17
|
+
<span class="ck-status-badge__dot" aria-hidden="true"></span>
|
|
18
|
+
<span class="ck-status-badge__label"><%= run.status.upcase %></span>
|
|
19
|
+
</span>
|
|
18
20
|
<h1 class="ck-title"><%= run.name %></h1>
|
|
19
21
|
<p class="ck-meta-copy"><%= link_to run.prompt.display_name, prompt_path(run.prompt), class: "ck-link" %> <span class="ck-chip" style="text-transform: none;"><%= run.prompt.llm_model %></span></p>
|
|
20
22
|
</div>
|
|
21
23
|
<%= render "completion_kit/runs/actions", run: run %>
|
|
22
24
|
</section>
|
|
23
|
-
|
|
24
|
-
<% if run.status.in?(%w[running completed]) && snap[:generated_total] > 0 %>
|
|
25
|
-
<div class="ck-progress-block">
|
|
26
|
-
<div class="ck-progress-line">
|
|
27
|
-
Generated <%= snap[:generated_done] %>/<%= snap[:generated_total] %>
|
|
28
|
-
<% if snap[:generated_failed] > 0 %>
|
|
29
|
-
<span class="ck-progress-failed">(<%= snap[:generated_failed] %> failed)</span>
|
|
30
|
-
<% end %>
|
|
31
|
-
</div>
|
|
32
|
-
<% if snap[:judged_total] > 0 %>
|
|
33
|
-
<div class="ck-progress-line">
|
|
34
|
-
Judged <%= snap[:judged_done] %>/<%= snap[:judged_total] %>
|
|
35
|
-
<% if snap[:judged_failed] > 0 %>
|
|
36
|
-
<span class="ck-progress-failed">(<%= snap[:judged_failed] %> failed)</span>
|
|
37
|
-
<% end %>
|
|
38
|
-
</div>
|
|
39
|
-
<% end %>
|
|
40
|
-
<% failed_count = snap[:generated_failed] + snap[:judged_failed] %>
|
|
41
|
-
<% if failed_count > 0 %>
|
|
42
|
-
<%= button_to "Retry #{failed_count} failed #{"row".pluralize(failed_count)}",
|
|
43
|
-
retry_failures_run_path(run),
|
|
44
|
-
method: :post,
|
|
45
|
-
class: ck_button_classes(:light, variant: :outline),
|
|
46
|
-
form_class: "inline-block" %>
|
|
47
|
-
<% end %>
|
|
48
|
-
</div>
|
|
49
|
-
<% end %>
|
|
50
25
|
</div>
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
<% snap = run.progress_snapshot %>
|
|
2
|
+
<div id="run_status_panel">
|
|
3
|
+
<% if run.status.in?(%w[running completed]) && snap[:generated_total] > 0 %>
|
|
4
|
+
<% failed_count = snap[:generated_failed] + snap[:judged_failed] %>
|
|
5
|
+
<section class="ck-run-status ck-run-status--<%= run.status %>">
|
|
6
|
+
<div class="ck-run-status__metrics">
|
|
7
|
+
<div class="ck-run-status__metric">
|
|
8
|
+
<p class="ck-run-status__metric-label">Generated</p>
|
|
9
|
+
<p class="ck-run-status__metric-value">
|
|
10
|
+
<span class="ck-run-status__metric-num"><%= snap[:generated_done] %></span><span class="ck-run-status__metric-total">/<%= snap[:generated_total] %></span>
|
|
11
|
+
<% if snap[:generated_failed] > 0 %>
|
|
12
|
+
<span class="ck-run-status__metric-fail"><%= snap[:generated_failed] %> failed</span>
|
|
13
|
+
<% end %>
|
|
14
|
+
</p>
|
|
15
|
+
</div>
|
|
16
|
+
|
|
17
|
+
<% if snap[:judged_total] > 0 || run.judge_configured? %>
|
|
18
|
+
<div class="ck-run-status__metric">
|
|
19
|
+
<p class="ck-run-status__metric-label">Judged</p>
|
|
20
|
+
<p class="ck-run-status__metric-value">
|
|
21
|
+
<span class="ck-run-status__metric-num"><%= snap[:judged_done] %></span><span class="ck-run-status__metric-total">/<%= snap[:judged_total] %></span>
|
|
22
|
+
<% if snap[:judged_failed] > 0 %>
|
|
23
|
+
<span class="ck-run-status__metric-fail"><%= snap[:judged_failed] %> failed</span>
|
|
24
|
+
<% end %>
|
|
25
|
+
</p>
|
|
26
|
+
</div>
|
|
27
|
+
<% end %>
|
|
28
|
+
|
|
29
|
+
<% if run.status == "completed" && run.avg_score %>
|
|
30
|
+
<div class="ck-run-status__metric">
|
|
31
|
+
<p class="ck-run-status__metric-label">Avg score</p>
|
|
32
|
+
<p class="ck-run-status__metric-value">
|
|
33
|
+
<span class="ck-run-status__metric-num"><%= run.avg_score %></span><span class="ck-run-status__metric-total">/5</span>
|
|
34
|
+
</p>
|
|
35
|
+
</div>
|
|
36
|
+
<% end %>
|
|
37
|
+
</div>
|
|
38
|
+
|
|
39
|
+
<% if failed_count > 0 %>
|
|
40
|
+
<%= button_to retry_failures_run_path(run), method: :post, class: "ck-run-status__retry", form_class: "ck-run-status__action" do %>
|
|
41
|
+
Retry <%= failed_count %> failed <%= "row".pluralize(failed_count) %>
|
|
42
|
+
<% end %>
|
|
43
|
+
<% end %>
|
|
44
|
+
</section>
|
|
45
|
+
<% end %>
|
|
46
|
+
</div>
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: completion-kit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Damien Bastin
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-05-
|
|
11
|
+
date: 2026-05-07 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rails
|
|
@@ -313,16 +313,17 @@ files:
|
|
|
313
313
|
- app/views/completion_kit/prompts/show.html.erb
|
|
314
314
|
- app/views/completion_kit/provider_credentials/_discovery_status.html.erb
|
|
315
315
|
- app/views/completion_kit/provider_credentials/_form.html.erb
|
|
316
|
+
- app/views/completion_kit/provider_credentials/_models_card.html.erb
|
|
316
317
|
- app/views/completion_kit/provider_credentials/edit.html.erb
|
|
317
318
|
- app/views/completion_kit/provider_credentials/index.html.erb
|
|
318
319
|
- app/views/completion_kit/provider_credentials/new.html.erb
|
|
319
320
|
- app/views/completion_kit/responses/show.html.erb
|
|
320
321
|
- app/views/completion_kit/runs/_actions.html.erb
|
|
321
322
|
- app/views/completion_kit/runs/_form.html.erb
|
|
322
|
-
- app/views/completion_kit/runs/_progress.html.erb
|
|
323
323
|
- app/views/completion_kit/runs/_response_row.html.erb
|
|
324
324
|
- app/views/completion_kit/runs/_sort_toolbar.html.erb
|
|
325
325
|
- app/views/completion_kit/runs/_status_header.html.erb
|
|
326
|
+
- app/views/completion_kit/runs/_status_panel.html.erb
|
|
326
327
|
- app/views/completion_kit/runs/edit.html.erb
|
|
327
328
|
- app/views/completion_kit/runs/index.html.erb
|
|
328
329
|
- app/views/completion_kit/runs/new.html.erb
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
<div id="run_progress">
|
|
2
|
-
<% if run.status == "running" %>
|
|
3
|
-
<div class="ck-discovery-bar">
|
|
4
|
-
<div class="ck-discovery-bar__label">
|
|
5
|
-
<%= ck_run_status_label(run) %>
|
|
6
|
-
</div>
|
|
7
|
-
<% if run.progress_total.to_i > 0 %>
|
|
8
|
-
<div class="ck-discovery-bar__track">
|
|
9
|
-
<div class="ck-discovery-bar__fill" style="width: <%= (run.progress_current.to_f / run.progress_total * 100).round %>%"></div>
|
|
10
|
-
</div>
|
|
11
|
-
<% else %>
|
|
12
|
-
<div class="ck-discovery-bar__track">
|
|
13
|
-
<div class="ck-discovery-bar__fill ck-discovery-bar__fill--indeterminate"></div>
|
|
14
|
-
</div>
|
|
15
|
-
<% end %>
|
|
16
|
-
</div>
|
|
17
|
-
<% end %>
|
|
18
|
-
</div>
|