@terreno/admin-frontend 58.0.5 → 58.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dist/AdminNestedArrayField.d.ts.map +1 -1
- package/dist/AdminNestedArrayField.js +6 -5
- package/dist/AdminNestedArrayField.js.map +1 -1
- package/dist/AdminRolesList.d.ts +34 -1
- package/dist/AdminRolesList.d.ts.map +1 -1
- package/dist/AdminRolesList.js +53 -8
- package/dist/AdminRolesList.js.map +1 -1
- package/dist/AdminShell.d.ts.map +1 -1
- package/dist/AdminShell.js +7 -2
- package/dist/AdminShell.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/useAdminRoles.d.ts +8 -0
- package/dist/useAdminRoles.d.ts.map +1 -1
- package/dist/useAdminRoles.js +11 -0
- package/dist/useAdminRoles.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetDetailScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetDetailScreen.js +10 -36
- package/dist/widgets/aiObservability/datasets/AiDatasetDetailScreen.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetDetailView.d.ts +4 -6
- package/dist/widgets/aiObservability/datasets/AiDatasetDetailView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetDetailView.js +11 -64
- package/dist/widgets/aiObservability/datasets/AiDatasetDetailView.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetItemScreen.d.ts +4 -0
- package/dist/widgets/aiObservability/datasets/AiDatasetItemScreen.d.ts.map +1 -0
- package/dist/widgets/aiObservability/datasets/AiDatasetItemScreen.js +204 -0
- package/dist/widgets/aiObservability/datasets/AiDatasetItemScreen.js.map +1 -0
- package/dist/widgets/aiObservability/datasets/AiDatasetItemView.d.ts +23 -0
- package/dist/widgets/aiObservability/datasets/AiDatasetItemView.d.ts.map +1 -0
- package/dist/widgets/aiObservability/datasets/AiDatasetItemView.js +31 -0
- package/dist/widgets/aiObservability/datasets/AiDatasetItemView.js.map +1 -0
- package/dist/widgets/aiObservability/datasets/ExpectedOutputForm.d.ts +12 -0
- package/dist/widgets/aiObservability/datasets/ExpectedOutputForm.d.ts.map +1 -0
- package/dist/widgets/aiObservability/datasets/ExpectedOutputForm.js +101 -0
- package/dist/widgets/aiObservability/datasets/ExpectedOutputForm.js.map +1 -0
- package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts +5 -0
- package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.js +9 -0
- package/dist/widgets/aiObservability/datasets/datasetTypes.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/expectedOutputModel.d.ts +44 -0
- package/dist/widgets/aiObservability/datasets/expectedOutputModel.d.ts.map +1 -0
- package/dist/widgets/aiObservability/datasets/expectedOutputModel.js +265 -0
- package/dist/widgets/aiObservability/datasets/expectedOutputModel.js.map +1 -0
- package/dist/widgets/aiObservability/datasets/useAiObservabilityDatasetsApi.d.ts +6 -0
- package/dist/widgets/aiObservability/datasets/useAiObservabilityDatasetsApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/useAiObservabilityDatasetsApi.js +8 -0
- package/dist/widgets/aiObservability/datasets/useAiObservabilityDatasetsApi.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js +5 -2
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.js +4 -3
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js +2 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.d.ts +2 -0
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.js +92 -26
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/promptTypes.d.ts +29 -5
- package/dist/widgets/aiObservability/prompts/promptTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/promptTypes.js +114 -11
- package/dist/widgets/aiObservability/prompts/promptTypes.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js +2 -0
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js.map +1 -1
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.js +1 -0
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.js.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTraceDetailView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTraceDetailView.js +15 -1
- package/dist/widgets/aiObservability/traces/AiTraceDetailView.js.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.js +20 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.js.map +1 -1
- package/dist/widgets/aiObservability/traces/traceTypes.d.ts +12 -0
- package/dist/widgets/aiObservability/traces/traceTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/traceTypes.js +33 -0
- package/dist/widgets/aiObservability/traces/traceTypes.js.map +1 -1
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js +9 -0
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js.map +1 -1
- package/package.json +4 -4
- package/src/AdminNestedArrayField.tsx +6 -4
- package/src/AdminRolesList.test.tsx +119 -2
- package/src/AdminRolesList.tsx +171 -13
- package/src/AdminShell.test.tsx +14 -0
- package/src/AdminShell.tsx +7 -1
- package/src/index.tsx +13 -1
- package/src/isolated/AdminNestedArrayField.isolated.tsx +50 -8
- package/src/isolated/aiObservabilityDatasets.isolated.tsx +417 -67
- package/src/isolated/aiObservabilityPrompts.isolated.tsx +42 -24
- package/src/isolated/aiObservabilityTraces.isolated.tsx +2 -1
- package/src/isolated/aiTracesTable.isolated.tsx +17 -1
- package/src/useAdminRoles.test.ts +15 -1
- package/src/useAdminRoles.ts +25 -1
- package/src/widgets/aiObservability/datasets/AiDatasetDetailScreen.tsx +17 -48
- package/src/widgets/aiObservability/datasets/AiDatasetDetailView.test.tsx +40 -77
- package/src/widgets/aiObservability/datasets/AiDatasetDetailView.tsx +29 -153
- package/src/widgets/aiObservability/datasets/AiDatasetItemScreen.tsx +279 -0
- package/src/widgets/aiObservability/datasets/AiDatasetItemView.test.tsx +110 -0
- package/src/widgets/aiObservability/datasets/AiDatasetItemView.tsx +174 -0
- package/src/widgets/aiObservability/datasets/ExpectedOutputForm.test.tsx +234 -0
- package/src/widgets/aiObservability/datasets/ExpectedOutputForm.tsx +313 -0
- package/src/widgets/aiObservability/datasets/datasetTypes.test.ts +31 -0
- package/src/widgets/aiObservability/datasets/datasetTypes.ts +13 -0
- package/src/widgets/aiObservability/datasets/expectedOutputModel.test.ts +285 -0
- package/src/widgets/aiObservability/datasets/expectedOutputModel.ts +338 -0
- package/src/widgets/aiObservability/datasets/useAiObservabilityDatasetsApi.test.ts +11 -0
- package/src/widgets/aiObservability/datasets/useAiObservabilityDatasetsApi.ts +22 -0
- package/src/widgets/aiObservability/prompts/AiPromptEditorView.tsx +7 -2
- package/src/widgets/aiObservability/prompts/AiPromptHubView.tsx +10 -3
- package/src/widgets/aiObservability/prompts/AiPromptsListScreen.tsx +3 -0
- package/src/widgets/aiObservability/prompts/AiPromptsListView.test.tsx +149 -3
- package/src/widgets/aiObservability/prompts/AiPromptsListView.tsx +176 -42
- package/src/widgets/aiObservability/prompts/promptTypes.test.ts +87 -7
- package/src/widgets/aiObservability/prompts/promptTypes.ts +140 -15
- package/src/widgets/aiObservability/shell/AiObservabilityScreenWidgets.tsx +2 -0
- package/src/widgets/aiObservability/shell/aiObservabilityNav.ts +1 -0
- package/src/widgets/aiObservability/traces/AiTraceDetailView.test.tsx +107 -0
- package/src/widgets/aiObservability/traces/AiTraceDetailView.tsx +42 -7
- package/src/widgets/aiObservability/traces/AiTracesListView.tsx +20 -0
- package/src/widgets/aiObservability/traces/traceTypes.test.ts +42 -0
- package/src/widgets/aiObservability/traces/traceTypes.ts +45 -0
- package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.test.ts +38 -0
- package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.ts +9 -0
|
@@ -1,16 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
Box,
|
|
4
|
-
Button,
|
|
5
|
-
Heading,
|
|
6
|
-
Modal,
|
|
7
|
-
SegmentedControl,
|
|
8
|
-
Spinner,
|
|
9
|
-
Text,
|
|
10
|
-
TextArea,
|
|
11
|
-
} from "@terreno/ui";
|
|
12
|
-
import {DateTime} from "luxon";
|
|
13
|
-
import React, {useCallback, useMemo, useState} from "react";
|
|
1
|
+
import {Badge, Box, Button, SegmentedControl, Spinner, Text} from "@terreno/ui";
|
|
2
|
+
import React, {useMemo, useState} from "react";
|
|
14
3
|
import {
|
|
15
4
|
ObservabilityTable,
|
|
16
5
|
type ObservabilityTableColumn,
|
|
@@ -28,12 +17,13 @@ export interface AiDatasetDetailViewProps {
|
|
|
28
17
|
canAddItem?: boolean;
|
|
29
18
|
canRunExperiment?: boolean;
|
|
30
19
|
dataset: DatasetRecord;
|
|
20
|
+
initialTab?: DatasetItemTab;
|
|
31
21
|
isItemsLoading?: boolean;
|
|
32
22
|
items: DatasetItemRecord[];
|
|
33
23
|
itemsLoadError?: string;
|
|
34
|
-
|
|
24
|
+
onCreateItem: () => void;
|
|
35
25
|
onOpenExperiment: () => void;
|
|
36
|
-
|
|
26
|
+
onOpenItem: (itemId: string) => void;
|
|
37
27
|
onRetryItems?: () => void;
|
|
38
28
|
routeBase: string;
|
|
39
29
|
}
|
|
@@ -63,47 +53,20 @@ const COLUMNS: ObservabilityTableColumn[] = [
|
|
|
63
53
|
{grow: 0.7, minWidth: 90, title: "Trace"},
|
|
64
54
|
];
|
|
65
55
|
|
|
66
|
-
const formatFullJson = (value: unknown): string => {
|
|
67
|
-
if (value === undefined || value === null) {
|
|
68
|
-
return "—";
|
|
69
|
-
}
|
|
70
|
-
if (typeof value === "string") {
|
|
71
|
-
return value;
|
|
72
|
-
}
|
|
73
|
-
try {
|
|
74
|
-
return JSON.stringify(value, null, 2);
|
|
75
|
-
} catch {
|
|
76
|
-
return String(value);
|
|
77
|
-
}
|
|
78
|
-
};
|
|
79
|
-
|
|
80
|
-
const formatTimestamp = (value: string): string => {
|
|
81
|
-
const timestamp = DateTime.fromISO(value);
|
|
82
|
-
if (!timestamp.isValid) {
|
|
83
|
-
return value;
|
|
84
|
-
}
|
|
85
|
-
return timestamp.toLocal().toLocaleString(DateTime.DATETIME_MED);
|
|
86
|
-
};
|
|
87
|
-
|
|
88
56
|
export const AiDatasetDetailView: React.FC<AiDatasetDetailViewProps> = ({
|
|
89
57
|
canAddItem = true,
|
|
90
58
|
canRunExperiment = true,
|
|
91
59
|
dataset,
|
|
60
|
+
initialTab = "all",
|
|
92
61
|
isItemsLoading,
|
|
93
62
|
items,
|
|
94
63
|
itemsLoadError,
|
|
95
|
-
|
|
64
|
+
onCreateItem,
|
|
96
65
|
onOpenExperiment,
|
|
97
|
-
|
|
66
|
+
onOpenItem,
|
|
98
67
|
onRetryItems,
|
|
99
68
|
}) => {
|
|
100
|
-
const [tab, setTab] = useState<DatasetItemTab>(
|
|
101
|
-
const [addOpen, setAddOpen] = useState(false);
|
|
102
|
-
const [addError, setAddError] = useState("");
|
|
103
|
-
const [isAdding, setIsAdding] = useState(false);
|
|
104
|
-
const [inputText, setInputText] = useState("{}");
|
|
105
|
-
const [expectedText, setExpectedText] = useState("{}");
|
|
106
|
-
const [selectedItem, setSelectedItem] = useState<DatasetItemRecord | undefined>();
|
|
69
|
+
const [tab, setTab] = useState<DatasetItemTab>(initialTab);
|
|
107
70
|
|
|
108
71
|
const filtered = useMemo(() => {
|
|
109
72
|
return filterDatasetItemsByTab(items, tab);
|
|
@@ -116,7 +79,7 @@ export const AiDatasetDetailView: React.FC<AiDatasetDetailViewProps> = ({
|
|
|
116
79
|
const attribution = item.annotatedBy?.label ?? (item.proofread ? "Human" : "Needs review");
|
|
117
80
|
const traceId = item.sourceTraceId;
|
|
118
81
|
return {
|
|
119
|
-
accessibilityLabel: `
|
|
82
|
+
accessibilityLabel: `Edit expected output for dataset item ${item.id}`,
|
|
120
83
|
cells: [
|
|
121
84
|
summarizeJson(item.input),
|
|
122
85
|
summarizeJson(item.expectedOutput),
|
|
@@ -125,37 +88,13 @@ export const AiDatasetDetailView: React.FC<AiDatasetDetailViewProps> = ({
|
|
|
125
88
|
],
|
|
126
89
|
key: item.id,
|
|
127
90
|
onClick: () => {
|
|
128
|
-
|
|
91
|
+
onOpenItem(item.id);
|
|
129
92
|
},
|
|
130
93
|
};
|
|
131
94
|
});
|
|
132
|
-
}, [filtered]);
|
|
95
|
+
}, [filtered, onOpenItem]);
|
|
133
96
|
|
|
134
97
|
const selectedIndex = TAB_OPTIONS.indexOf(tab);
|
|
135
|
-
const handleAddItem = useCallback(async (): Promise<void> => {
|
|
136
|
-
setAddError("");
|
|
137
|
-
setIsAdding(true);
|
|
138
|
-
const error = await onAddItem({expectedOutput: expectedText, input: inputText});
|
|
139
|
-
setIsAdding(false);
|
|
140
|
-
if (error) {
|
|
141
|
-
setAddError(error);
|
|
142
|
-
return;
|
|
143
|
-
}
|
|
144
|
-
setAddOpen(false);
|
|
145
|
-
}, [expectedText, inputText, onAddItem]);
|
|
146
|
-
|
|
147
|
-
const handleDismissItem = useCallback((): void => {
|
|
148
|
-
setSelectedItem(undefined);
|
|
149
|
-
}, []);
|
|
150
|
-
|
|
151
|
-
const handleOpenSelectedTrace = useCallback((): void => {
|
|
152
|
-
if (!selectedItem?.sourceTraceId || !onOpenTrace) {
|
|
153
|
-
return;
|
|
154
|
-
}
|
|
155
|
-
const traceId = selectedItem.sourceTraceId;
|
|
156
|
-
setSelectedItem(undefined);
|
|
157
|
-
onOpenTrace(traceId);
|
|
158
|
-
}, [onOpenTrace, selectedItem]);
|
|
159
98
|
|
|
160
99
|
return (
|
|
161
100
|
<Box gap={4} testID="ai-dataset-detail">
|
|
@@ -171,10 +110,7 @@ export const AiDatasetDetailView: React.FC<AiDatasetDetailViewProps> = ({
|
|
|
171
110
|
<Box direction="row" gap={2}>
|
|
172
111
|
{canAddItem ? (
|
|
173
112
|
<Button
|
|
174
|
-
onClick={
|
|
175
|
-
setAddError("");
|
|
176
|
-
setAddOpen(true);
|
|
177
|
-
}}
|
|
113
|
+
onClick={onCreateItem}
|
|
178
114
|
testID="ai-dataset-add-item"
|
|
179
115
|
text="Add item"
|
|
180
116
|
variant="secondary"
|
|
@@ -189,6 +125,10 @@ export const AiDatasetDetailView: React.FC<AiDatasetDetailViewProps> = ({
|
|
|
189
125
|
) : undefined}
|
|
190
126
|
</Box>
|
|
191
127
|
</Box>
|
|
128
|
+
<Text color="secondaryDark" size="sm" testID="ai-dataset-golden-guidance">
|
|
129
|
+
Each item's expected output is its golden label: the answer this feature must give for
|
|
130
|
+
that input. Experiments score the feature against proofread items only.
|
|
131
|
+
</Text>
|
|
192
132
|
<SegmentedControl
|
|
193
133
|
items={TAB_OPTIONS.map((option) => tabLabel(option, needsReviewCount))}
|
|
194
134
|
onChange={(index) => {
|
|
@@ -200,6 +140,12 @@ export const AiDatasetDetailView: React.FC<AiDatasetDetailViewProps> = ({
|
|
|
200
140
|
selectedIndex={selectedIndex}
|
|
201
141
|
testID="ai-dataset-tabs"
|
|
202
142
|
/>
|
|
143
|
+
{tab === "needsReview" ? (
|
|
144
|
+
<Text color="secondaryDark" size="sm" testID="ai-dataset-review-guidance">
|
|
145
|
+
Needs review items are not proofread yet. Open one, correct its expected output, tick
|
|
146
|
+
Proofread, then Save and next to move down this list.
|
|
147
|
+
</Text>
|
|
148
|
+
) : undefined}
|
|
203
149
|
{tab === "needsReview" && needsReviewCount > 0 ? (
|
|
204
150
|
<Badge
|
|
205
151
|
status="error"
|
|
@@ -218,90 +164,20 @@ export const AiDatasetDetailView: React.FC<AiDatasetDetailViewProps> = ({
|
|
|
218
164
|
</Box>
|
|
219
165
|
) : filtered.length === 0 ? (
|
|
220
166
|
<Box padding={4} testID="ai-dataset-items-empty">
|
|
221
|
-
<Text color="secondaryDark">
|
|
167
|
+
<Text color="secondaryDark">
|
|
168
|
+
{tab === "needsReview"
|
|
169
|
+
? "Nothing needs review. Every item is proofread."
|
|
170
|
+
: "No items in this tab."}
|
|
171
|
+
</Text>
|
|
222
172
|
</Box>
|
|
223
173
|
) : (
|
|
224
174
|
<Box gap={2}>
|
|
225
175
|
<Text color="secondaryDark" size="sm">
|
|
226
|
-
Select an item to
|
|
176
|
+
Select an item to edit its expected output.
|
|
227
177
|
</Text>
|
|
228
178
|
<ObservabilityTable columns={COLUMNS} rows={rows} testID="ai-dataset-items-table" />
|
|
229
179
|
</Box>
|
|
230
180
|
)}
|
|
231
|
-
<Modal
|
|
232
|
-
onDismiss={handleDismissItem}
|
|
233
|
-
title="Dataset item details"
|
|
234
|
-
visible={Boolean(selectedItem)}
|
|
235
|
-
>
|
|
236
|
-
{selectedItem ? (
|
|
237
|
-
<Box gap={3} maxHeight={600} padding={3} scroll testID="ai-dataset-item-modal">
|
|
238
|
-
<Box gap={1}>
|
|
239
|
-
<Heading size="sm">Input</Heading>
|
|
240
|
-
<Box color="baseAlternate" padding={3} rounding="md">
|
|
241
|
-
<Text>{formatFullJson(selectedItem.input)}</Text>
|
|
242
|
-
</Box>
|
|
243
|
-
</Box>
|
|
244
|
-
<Box gap={1}>
|
|
245
|
-
<Heading size="sm">Expected output</Heading>
|
|
246
|
-
<Box color="baseAlternate" padding={3} rounding="md">
|
|
247
|
-
<Text>{formatFullJson(selectedItem.expectedOutput)}</Text>
|
|
248
|
-
</Box>
|
|
249
|
-
</Box>
|
|
250
|
-
<Box gap={1}>
|
|
251
|
-
<Heading size="sm">Item details</Heading>
|
|
252
|
-
<Text>ID: {selectedItem.id}</Text>
|
|
253
|
-
<Text>Dataset ID: {selectedItem.datasetId}</Text>
|
|
254
|
-
<Text>Origin: {selectedItem.origin}</Text>
|
|
255
|
-
<Text>Proofread: {selectedItem.proofread ? "Yes" : "No"}</Text>
|
|
256
|
-
<Text>Outcome: {selectedItem.outcomeClass ?? "—"}</Text>
|
|
257
|
-
<Text>Source trace: {selectedItem.sourceTraceId ?? "—"}</Text>
|
|
258
|
-
<Text>Annotated by: {selectedItem.annotatedBy?.label ?? "—"}</Text>
|
|
259
|
-
<Text>Annotator user: {selectedItem.annotatedBy?.userId ?? "—"}</Text>
|
|
260
|
-
<Text>Review item: {selectedItem.annotatedBy?.reviewItemId ?? "—"}</Text>
|
|
261
|
-
<Text>Tags: {selectedItem.tags.join(", ") || "—"}</Text>
|
|
262
|
-
<Text>Created: {formatTimestamp(selectedItem.created)}</Text>
|
|
263
|
-
<Text>Updated: {formatTimestamp(selectedItem.updated)}</Text>
|
|
264
|
-
</Box>
|
|
265
|
-
<Box gap={1}>
|
|
266
|
-
<Heading size="sm">Metadata</Heading>
|
|
267
|
-
<Box color="baseAlternate" padding={3} rounding="md">
|
|
268
|
-
<Text>{formatFullJson(selectedItem.metadata)}</Text>
|
|
269
|
-
</Box>
|
|
270
|
-
</Box>
|
|
271
|
-
{selectedItem.sourceTraceId && onOpenTrace ? (
|
|
272
|
-
<Button
|
|
273
|
-
onClick={handleOpenSelectedTrace}
|
|
274
|
-
testID="ai-dataset-item-open-trace"
|
|
275
|
-
text="Open source trace"
|
|
276
|
-
variant="secondary"
|
|
277
|
-
/>
|
|
278
|
-
) : undefined}
|
|
279
|
-
</Box>
|
|
280
|
-
) : undefined}
|
|
281
|
-
</Modal>
|
|
282
|
-
<Modal
|
|
283
|
-
onDismiss={() => {
|
|
284
|
-
setAddOpen(false);
|
|
285
|
-
}}
|
|
286
|
-
title="Add dataset item"
|
|
287
|
-
visible={addOpen}
|
|
288
|
-
>
|
|
289
|
-
<Box gap={3} padding={3}>
|
|
290
|
-
<TextArea onChange={setInputText} rows={4} title="Input (JSON)" value={inputText} />
|
|
291
|
-
<TextArea
|
|
292
|
-
onChange={setExpectedText}
|
|
293
|
-
rows={4}
|
|
294
|
-
title="Expected output (JSON)"
|
|
295
|
-
value={expectedText}
|
|
296
|
-
/>
|
|
297
|
-
<Button loading={isAdding} onClick={handleAddItem} text="Add item" />
|
|
298
|
-
{addError ? (
|
|
299
|
-
<Text color="error" testID="ai-dataset-add-item-error">
|
|
300
|
-
{addError}
|
|
301
|
-
</Text>
|
|
302
|
-
) : undefined}
|
|
303
|
-
</Box>
|
|
304
|
-
</Modal>
|
|
305
181
|
</Box>
|
|
306
182
|
);
|
|
307
183
|
};
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
import {Box, Spinner, Text} from "@terreno/ui";
|
|
2
|
+
import {router, useLocalSearchParams} from "expo-router";
|
|
3
|
+
import React, {useCallback, useEffect, useMemo, useState} from "react";
|
|
4
|
+
import type {AdminScreenWidgetProps} from "../../../types";
|
|
5
|
+
import {useAiObservabilityPromptsApi} from "../prompts/useAiObservabilityPromptsApi";
|
|
6
|
+
import {AiObservabilityChrome} from "../shell/AiObservabilityChrome";
|
|
7
|
+
import {unwrapObservabilityStatus} from "../shell/aiObservabilityNav";
|
|
8
|
+
import {observabilityActionAllowed} from "../shell/observabilityPermissions";
|
|
9
|
+
import {AiDatasetItemView} from "./AiDatasetItemView";
|
|
10
|
+
import {nextNeedsReviewItemId, unwrapDatasetItems, unwrapDatasetRecord} from "./datasetTypes";
|
|
11
|
+
import {emptyLike, type JsonSchema, prepareFormValue} from "./expectedOutputModel";
|
|
12
|
+
import {useAiObservabilityDatasetsApi} from "./useAiObservabilityDatasetsApi";
|
|
13
|
+
|
|
14
|
+
const firstParam = (value: string | string[] | undefined): string => {
|
|
15
|
+
if (Array.isArray(value)) {
|
|
16
|
+
return value[0] ?? "";
|
|
17
|
+
}
|
|
18
|
+
return value ?? "";
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
type OutcomeClass = "fn" | "fp" | "tn" | "tp";
|
|
22
|
+
|
|
23
|
+
const outcomeOrNull = (value: string): OutcomeClass | null => {
|
|
24
|
+
if (value === "tp" || value === "fp" || value === "tn" || value === "fn") {
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
return null;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const saveErrorMessage = (error: unknown): string => {
|
|
31
|
+
if (error && typeof error === "object" && "data" in error) {
|
|
32
|
+
const title = (error as {data?: {title?: string}}).data?.title;
|
|
33
|
+
if (title) {
|
|
34
|
+
return title;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return "Could not save this dataset item.";
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export const AiDatasetItemScreenWidget: React.FC<AdminScreenWidgetProps> = (props) => {
|
|
41
|
+
const {routeBase} = props;
|
|
42
|
+
const params = useLocalSearchParams<{
|
|
43
|
+
datasetId?: string | string[];
|
|
44
|
+
itemId?: string | string[];
|
|
45
|
+
}>();
|
|
46
|
+
const datasetId = firstParam(params.datasetId);
|
|
47
|
+
const itemId = firstParam(params.itemId);
|
|
48
|
+
const isNew = itemId.length === 0;
|
|
49
|
+
const {useCreateItemMutation, useDetailQuery, useItemsQuery, useUpdateItemMutation} =
|
|
50
|
+
useAiObservabilityDatasetsApi();
|
|
51
|
+
const {useStatusQuery} = useAiObservabilityPromptsApi();
|
|
52
|
+
const statusQuery = useStatusQuery();
|
|
53
|
+
const canSave = observabilityActionAllowed({
|
|
54
|
+
action: "update",
|
|
55
|
+
resource: "aiDataset",
|
|
56
|
+
status: unwrapObservabilityStatus(statusQuery.data),
|
|
57
|
+
statusError: statusQuery.isError,
|
|
58
|
+
statusLoading: statusQuery.isLoading,
|
|
59
|
+
});
|
|
60
|
+
const {data, isError, isLoading} = useDetailQuery(datasetId, {skip: datasetId.length === 0});
|
|
61
|
+
const {
|
|
62
|
+
data: itemsRaw,
|
|
63
|
+
isLoading: isItemsLoading,
|
|
64
|
+
refetch: refetchItems,
|
|
65
|
+
} = useItemsQuery(datasetId, {
|
|
66
|
+
skip: datasetId.length === 0,
|
|
67
|
+
});
|
|
68
|
+
const [createItem] = useCreateItemMutation();
|
|
69
|
+
const [updateItem] = useUpdateItemMutation();
|
|
70
|
+
const dataset = useMemo(() => unwrapDatasetRecord(data), [data]);
|
|
71
|
+
const items = useMemo(() => unwrapDatasetItems(itemsRaw), [itemsRaw]);
|
|
72
|
+
const item = useMemo(() => items.find((entry) => entry.id === itemId), [itemId, items]);
|
|
73
|
+
const schema = dataset?.expectedOutputSchema as JsonSchema | undefined;
|
|
74
|
+
const prefix = (routeBase ?? "").replace(/\/$/, "");
|
|
75
|
+
const backHref = `${prefix}/ai-dataset-detail?id=${encodeURIComponent(datasetId)}`;
|
|
76
|
+
const reviewQueueHref = `${backHref}&tab=needsReview`;
|
|
77
|
+
const reviewing = !isNew && item !== undefined && !item.proofread;
|
|
78
|
+
const nextReviewItem = reviewing ? nextNeedsReviewItemId(items, itemId) : undefined;
|
|
79
|
+
|
|
80
|
+
const formKey = `${datasetId}:${itemId}`;
|
|
81
|
+
const [filledKey, setFilledKey] = useState<string | undefined>(undefined);
|
|
82
|
+
const formReady = filledKey === formKey;
|
|
83
|
+
const [expected, setExpected] = useState<unknown>("");
|
|
84
|
+
const [input, setInput] = useState<unknown>("");
|
|
85
|
+
const [outcomeClass, setOutcomeClass] = useState("");
|
|
86
|
+
const [proofread, setProofread] = useState(true);
|
|
87
|
+
const [isSaving, setIsSaving] = useState(false);
|
|
88
|
+
const [error, setError] = useState("");
|
|
89
|
+
|
|
90
|
+
// Fill the form once per dataset and item, after both have loaded, so a param change refills it.
|
|
91
|
+
useEffect(() => {
|
|
92
|
+
if (formReady || isLoading || isItemsLoading) {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
setError("");
|
|
96
|
+
setIsSaving(false);
|
|
97
|
+
if (!dataset) {
|
|
98
|
+
setFilledKey(formKey);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
if (isNew) {
|
|
102
|
+
const sample = items[0];
|
|
103
|
+
setInput(sample ? emptyLike(sample.input) : "");
|
|
104
|
+
setExpected(
|
|
105
|
+
schema
|
|
106
|
+
? prepareFormValue(schema, undefined)
|
|
107
|
+
: sample && sample.expectedOutput !== undefined
|
|
108
|
+
? emptyLike(sample.expectedOutput)
|
|
109
|
+
: ""
|
|
110
|
+
);
|
|
111
|
+
setOutcomeClass("");
|
|
112
|
+
setProofread(true);
|
|
113
|
+
setFilledKey(formKey);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
if (!item) {
|
|
117
|
+
setFilledKey(formKey);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
setInput(item.input);
|
|
121
|
+
setExpected(prepareFormValue(schema, item.expectedOutput));
|
|
122
|
+
setOutcomeClass(item.outcomeClass ?? "");
|
|
123
|
+
setProofread(item.proofread);
|
|
124
|
+
setFilledKey(formKey);
|
|
125
|
+
}, [dataset, formKey, formReady, isItemsLoading, isLoading, isNew, item, items, schema]);
|
|
126
|
+
|
|
127
|
+
const handleOpenTrace = useCallback((): void => {
|
|
128
|
+
if (!item?.sourceTraceId) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
router.push(`${prefix}/ai-trace-detail?id=${encodeURIComponent(item.sourceTraceId)}`);
|
|
132
|
+
}, [item?.sourceTraceId, prefix]);
|
|
133
|
+
|
|
134
|
+
// Reads the list after the save so an item saved just now is no longer offered as needing review.
|
|
135
|
+
const findNextReviewItem = useCallback(async (): Promise<string | undefined> => {
|
|
136
|
+
const refreshed = await refetchItems();
|
|
137
|
+
const queue =
|
|
138
|
+
refreshed.isError || refreshed.data === undefined
|
|
139
|
+
? items
|
|
140
|
+
: unwrapDatasetItems(refreshed.data);
|
|
141
|
+
return nextNeedsReviewItemId(queue, itemId);
|
|
142
|
+
}, [itemId, items, refetchItems]);
|
|
143
|
+
|
|
144
|
+
const handleSave = useCallback(async (): Promise<void> => {
|
|
145
|
+
if (!datasetId) {
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
setError("");
|
|
149
|
+
setIsSaving(true);
|
|
150
|
+
const outcome = outcomeOrNull(outcomeClass);
|
|
151
|
+
try {
|
|
152
|
+
if (isNew) {
|
|
153
|
+
await createItem({
|
|
154
|
+
body: {
|
|
155
|
+
expectedOutput: expected,
|
|
156
|
+
input,
|
|
157
|
+
origin: "manual",
|
|
158
|
+
outcomeClass: outcome,
|
|
159
|
+
proofread,
|
|
160
|
+
},
|
|
161
|
+
datasetId,
|
|
162
|
+
}).unwrap();
|
|
163
|
+
} else {
|
|
164
|
+
await updateItem({
|
|
165
|
+
body: {
|
|
166
|
+
expectedOutput: expected,
|
|
167
|
+
outcomeClass: outcome,
|
|
168
|
+
proofread,
|
|
169
|
+
},
|
|
170
|
+
datasetId,
|
|
171
|
+
itemId,
|
|
172
|
+
}).unwrap();
|
|
173
|
+
}
|
|
174
|
+
if (!reviewing) {
|
|
175
|
+
router.push(backHref);
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
const nextId = await findNextReviewItem();
|
|
179
|
+
if (nextId) {
|
|
180
|
+
router.replace(
|
|
181
|
+
`${prefix}/ai-dataset-item?datasetId=${encodeURIComponent(datasetId)}&itemId=${encodeURIComponent(nextId)}`
|
|
182
|
+
);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
router.push(reviewQueueHref);
|
|
186
|
+
} catch (saveError) {
|
|
187
|
+
setError(saveErrorMessage(saveError));
|
|
188
|
+
setIsSaving(false);
|
|
189
|
+
}
|
|
190
|
+
}, [
|
|
191
|
+
backHref,
|
|
192
|
+
createItem,
|
|
193
|
+
datasetId,
|
|
194
|
+
expected,
|
|
195
|
+
findNextReviewItem,
|
|
196
|
+
input,
|
|
197
|
+
isNew,
|
|
198
|
+
itemId,
|
|
199
|
+
outcomeClass,
|
|
200
|
+
prefix,
|
|
201
|
+
proofread,
|
|
202
|
+
reviewQueueHref,
|
|
203
|
+
reviewing,
|
|
204
|
+
updateItem,
|
|
205
|
+
]);
|
|
206
|
+
|
|
207
|
+
let saveLabel = "Save expected output";
|
|
208
|
+
if (isNew) {
|
|
209
|
+
saveLabel = "Add item";
|
|
210
|
+
} else if (reviewing) {
|
|
211
|
+
saveLabel = nextReviewItem ? "Save and next" : "Save and finish review";
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if (datasetId.length === 0) {
|
|
215
|
+
return (
|
|
216
|
+
<AiObservabilityChrome {...props} backHref={prefix} screenName="ai-dataset-item">
|
|
217
|
+
<Box padding={4}>
|
|
218
|
+
<Text>Missing dataset id.</Text>
|
|
219
|
+
</Box>
|
|
220
|
+
</AiObservabilityChrome>
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (isLoading || isItemsLoading || !formReady) {
|
|
225
|
+
return (
|
|
226
|
+
<AiObservabilityChrome {...props} backHref={backHref} hideHeader screenName="ai-dataset-item">
|
|
227
|
+
<Box alignItems="center" padding={4} testID="ai-dataset-item-loading">
|
|
228
|
+
<Spinner />
|
|
229
|
+
</Box>
|
|
230
|
+
</AiObservabilityChrome>
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (isError || !dataset || (!isNew && !item)) {
|
|
235
|
+
return (
|
|
236
|
+
<AiObservabilityChrome {...props} backHref={backHref} screenName="ai-dataset-item">
|
|
237
|
+
<Box padding={4}>
|
|
238
|
+
<Text color="error">
|
|
239
|
+
{dataset ? "Failed to load dataset item." : "Failed to load dataset."}
|
|
240
|
+
</Text>
|
|
241
|
+
</Box>
|
|
242
|
+
</AiObservabilityChrome>
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
return (
|
|
247
|
+
<AiObservabilityChrome
|
|
248
|
+
{...props}
|
|
249
|
+
backHref={backHref}
|
|
250
|
+
description={
|
|
251
|
+
isNew
|
|
252
|
+
? `Add a golden row to ${dataset.name}.`
|
|
253
|
+
: `Set the golden expected output for ${dataset.name}.`
|
|
254
|
+
}
|
|
255
|
+
screenName="ai-dataset-item"
|
|
256
|
+
title={isNew ? "New dataset item" : "Edit expected output"}
|
|
257
|
+
>
|
|
258
|
+
<AiDatasetItemView
|
|
259
|
+
canSave={canSave}
|
|
260
|
+
error={error}
|
|
261
|
+
expected={expected}
|
|
262
|
+
expectedSchema={schema}
|
|
263
|
+
input={input}
|
|
264
|
+
inputEditable={isNew}
|
|
265
|
+
isSaving={isSaving}
|
|
266
|
+
onExpectedChange={setExpected}
|
|
267
|
+
onInputChange={setInput}
|
|
268
|
+
onOpenTrace={!isNew && item?.sourceTraceId ? handleOpenTrace : undefined}
|
|
269
|
+
onOutcomeChange={setOutcomeClass}
|
|
270
|
+
onProofreadChange={setProofread}
|
|
271
|
+
onSave={handleSave}
|
|
272
|
+
outcomeClass={outcomeClass}
|
|
273
|
+
proofread={proofread}
|
|
274
|
+
reviewing={reviewing}
|
|
275
|
+
saveLabel={saveLabel}
|
|
276
|
+
/>
|
|
277
|
+
</AiObservabilityChrome>
|
|
278
|
+
);
|
|
279
|
+
};
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import {describe, expect, it, mock} from "bun:test";
|
|
2
|
+
import {act, fireEvent} from "@testing-library/react-native";
|
|
3
|
+
import {assert} from "chai";
|
|
4
|
+
import React from "react";
|
|
5
|
+
import {renderWithTheme} from "../../../../../ui/src/test-utils";
|
|
6
|
+
import {AiDatasetItemView} from "./AiDatasetItemView";
|
|
7
|
+
import type {JsonSchema} from "./expectedOutputModel";
|
|
8
|
+
|
|
9
|
+
const CHAT_SAFETY_SCHEMA: JsonSchema = {
|
|
10
|
+
properties: {
|
|
11
|
+
crisis: {description: "Immediate danger to self", type: "boolean"},
|
|
12
|
+
route: {
|
|
13
|
+
description: "Where to send this message",
|
|
14
|
+
enum: ["none", "988", "care-team"],
|
|
15
|
+
type: "string",
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
required: ["crisis", "route"],
|
|
19
|
+
type: "object",
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
const baseProps = {
|
|
23
|
+
expected: "Terreno provides a full-stack framework.",
|
|
24
|
+
input: {text: "What is Terreno?"},
|
|
25
|
+
onExpectedChange: (): void => undefined,
|
|
26
|
+
onInputChange: (): void => undefined,
|
|
27
|
+
onOutcomeChange: (): void => undefined,
|
|
28
|
+
onProofreadChange: (): void => undefined,
|
|
29
|
+
onSave: (): void => undefined,
|
|
30
|
+
outcomeClass: "",
|
|
31
|
+
proofread: true,
|
|
32
|
+
saveLabel: "Save expected output",
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
describe("AiDatasetItemView", () => {
|
|
36
|
+
it("edits a string expected output as one text field", async () => {
|
|
37
|
+
const onExpectedChange = mock((_value: unknown): void => undefined);
|
|
38
|
+
const onSave = mock((): void => undefined);
|
|
39
|
+
const view = renderWithTheme(
|
|
40
|
+
<AiDatasetItemView {...baseProps} onExpectedChange={onExpectedChange} onSave={onSave} />
|
|
41
|
+
);
|
|
42
|
+
|
|
43
|
+
expect(view.getByTestId("ai-dataset-item-expected")).toBeTruthy();
|
|
44
|
+
fireEvent.changeText(view.getByTestId("ai-dataset-item-expected-root"), "A golden answer.");
|
|
45
|
+
assert.deepEqual(onExpectedChange.mock.calls[0], ["A golden answer."]);
|
|
46
|
+
|
|
47
|
+
await act(async () => {
|
|
48
|
+
fireEvent.press(view.getByTestId("ai-dataset-item-save"));
|
|
49
|
+
await Promise.resolve();
|
|
50
|
+
});
|
|
51
|
+
assert.equal(onSave.mock.calls.length, 1);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("builds checkboxes and selects from the expected output schema", () => {
|
|
55
|
+
const view = renderWithTheme(
|
|
56
|
+
<AiDatasetItemView
|
|
57
|
+
{...baseProps}
|
|
58
|
+
expected={{crisis: false, route: "988"}}
|
|
59
|
+
expectedSchema={CHAT_SAFETY_SCHEMA}
|
|
60
|
+
/>
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
expect(view.getByText("Crisis")).toBeTruthy();
|
|
64
|
+
expect(view.getByText("Immediate danger to self")).toBeTruthy();
|
|
65
|
+
expect(view.getByText("Route")).toBeTruthy();
|
|
66
|
+
expect(view.getByText("Where to send this message")).toBeTruthy();
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it("toggles proofread and opens the source trace", async () => {
|
|
70
|
+
const onProofreadChange = mock((_value: boolean): void => undefined);
|
|
71
|
+
const onOpenTrace = mock((): void => undefined);
|
|
72
|
+
const view = renderWithTheme(
|
|
73
|
+
<AiDatasetItemView
|
|
74
|
+
{...baseProps}
|
|
75
|
+
onOpenTrace={onOpenTrace}
|
|
76
|
+
onProofreadChange={onProofreadChange}
|
|
77
|
+
proofread
|
|
78
|
+
/>
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
await act(async () => {
|
|
82
|
+
fireEvent.press(view.getByTestId("ai-dataset-item-proofread-clickable"));
|
|
83
|
+
fireEvent.press(view.getByTestId("ai-dataset-item-open-trace"));
|
|
84
|
+
await Promise.resolve();
|
|
85
|
+
});
|
|
86
|
+
assert.deepEqual(onProofreadChange.mock.calls, [[false]]);
|
|
87
|
+
assert.equal(onOpenTrace.mock.calls.length, 1);
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
describe("AiDatasetItemView review guidance", () => {
|
|
92
|
+
it("explains the golden label on every item", () => {
|
|
93
|
+
const view = renderWithTheme(<AiDatasetItemView {...baseProps} />);
|
|
94
|
+
expect(view.getByText("Expected output (golden label)")).toBeTruthy();
|
|
95
|
+
expect(
|
|
96
|
+
view.getByText(/Experiments score the live feature against proofread golden labels/)
|
|
97
|
+
).toBeTruthy();
|
|
98
|
+
expect(view.queryByTestId("ai-dataset-item-review-steps")).toBeNull();
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it("lists the review steps when the item is in the review queue", () => {
|
|
102
|
+
const view = renderWithTheme(<AiDatasetItemView {...baseProps} proofread={false} reviewing />);
|
|
103
|
+
const steps = view.getByTestId("ai-dataset-item-review-steps");
|
|
104
|
+
assert.exists(steps);
|
|
105
|
+
expect(
|
|
106
|
+
view.getByText("1. Read the input. It is exactly what the model received.")
|
|
107
|
+
).toBeTruthy();
|
|
108
|
+
expect(view.getByText("5. Save and next. Unticked items stay in Needs review.")).toBeTruthy();
|
|
109
|
+
});
|
|
110
|
+
});
|