@elevasis/core 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +60 -103
- package/dist/index.js +162 -109
- package/dist/organization-model/index.d.ts +60 -103
- package/dist/organization-model/index.js +162 -109
- package/package.json +1 -1
- package/src/README.md +24 -17
- package/src/__tests__/template-foundations-compatibility.test.ts +28 -36
- package/src/auth/multi-tenancy/types.ts +4 -11
- package/src/auth/multi-tenancy/users/api-schemas.ts +1 -1
- package/src/business/base-entities.test.ts +481 -0
- package/src/business/base-entities.ts +241 -0
- package/src/business/delivery/types.ts +1 -1
- package/src/business/index.ts +3 -0
- package/src/execution/index.ts +3 -6
- package/src/index.ts +1 -1
- package/src/organization-model/README.md +25 -26
- package/src/organization-model/__tests__/graph.test.ts +103 -71
- package/src/organization-model/__tests__/resolve.test.ts +20 -29
- package/src/organization-model/contracts.ts +3 -0
- package/src/organization-model/defaults.ts +40 -6
- package/src/organization-model/domains/features.ts +19 -54
- package/src/organization-model/domains/navigation.ts +25 -16
- package/src/organization-model/domains/shared.ts +1 -10
- package/src/organization-model/foundation.ts +96 -0
- package/src/organization-model/graph/build.ts +34 -67
- package/src/organization-model/graph/schema.ts +2 -4
- package/src/organization-model/graph/types.ts +3 -15
- package/src/organization-model/index.ts +2 -0
- package/src/organization-model/organization-model.mdx +34 -36
- package/src/organization-model/published.ts +12 -3
- package/src/organization-model/schema.ts +38 -34
- package/src/organization-model/types.ts +5 -10
- package/src/platform/constants/versions.ts +1 -1
- package/src/platform/sse/events.ts +1 -34
- package/src/projects/api-schemas.ts +2 -1
- package/src/reference/_generated/contracts.md +10 -31
- package/src/reference/glossary.md +14 -18
- package/src/supabase/database.types.ts +0 -107
- package/src/test-utils/rls/RLSTestContext.ts +1 -31
- package/src/execution/calibration/__tests__/schemas.test.ts +0 -320
- package/src/execution/calibration/index.ts +0 -3
- package/src/execution/calibration/schemas.ts +0 -121
- package/src/execution/calibration/sse-events.ts +0 -125
- package/src/execution/calibration/types.ts +0 -190
|
@@ -13,7 +13,7 @@ This condensed version covers every ambiguity-prone term a template consumer or
|
|
|
13
13
|
|
|
14
14
|
## Terms
|
|
15
15
|
|
|
16
|
-
**
|
|
16
|
+
**featureId** (on FeatureModule) -- the `id` string a `FeatureModule` declares so the provider knows which entry in `OrganizationModel.features` to check for access gating. Required field on every `FeatureModule`. Must match the `id` of an entry in the features array (e.g. `'crm'`, `'projects'`). Distinct from the nav-item `featureKey` (see below). No aliasing layer -- IDs are direct matches.
|
|
17
17
|
|
|
18
18
|
**AdminGuard** -- route-level admin wrapper from `@elevasis/ui/features/auth`. Wraps routes restricted to admin members. Must nest inside `ProtectedRoute`. Does not replace `requiresAdmin` on nav entries -- use both when both route access and nav visibility need admin enforcement.
|
|
19
19
|
|
|
@@ -21,45 +21,41 @@ This condensed version covers every ambiguity-prone term a template consumer or
|
|
|
21
21
|
|
|
22
22
|
**DeploymentSpec** -- the complete resource collection for one organization (`workflows`, `agents`, `triggers`, `integrations`, `relationships`, `externalResources`, `humanCheckpoints`). Defined in `@repo/core`. The `operations/src/index.ts` file in each external project exports one `DeploymentSpec`.
|
|
23
23
|
|
|
24
|
-
**Domain** --
|
|
24
|
+
**Domain** -- removed concept. What was previously a `SemanticDomain` entry in `OrganizationModel.domains` is now expressed directly as a **Feature** in the `features` array. The `domains` field no longer exists on `OrganizationModel`. Semantic grouping (entity IDs, surface IDs, resource IDs, capability IDs) now lives on each `OrganizationModelFeature` object.
|
|
25
25
|
|
|
26
|
-
**
|
|
27
|
-
|
|
28
|
-
**FoundationLegacyFeatureKey** -- template-local union for feature keys defined in `foundations/config/organization-model.ts`. Distinct from the published `OrganizationModelFeatureKey` (seven-key union from `@elevasis/core`). Extension point for template-specific feature identity without modifying the published core union. `FoundationFeatureKey` combines both (`OrganizationModelFeatureKey | FoundationLegacyFeatureKey`). `FEATURE_KEY_ALIASES` maps between shell keys and org-model grouped keys at runtime.
|
|
29
|
-
|
|
30
|
-
**Feature** -- an overloaded term. Always qualify which layer is in scope:
|
|
26
|
+
**Feature** -- the unified concept replacing the former three-layer system (feature keys, semantic domains, feature modules). Always qualify which layer is in scope:
|
|
31
27
|
|
|
32
28
|
- **Platform capability** -- a product area (Execution Engine, Workflows, Agents, etc.). Not one-to-one with shell features.
|
|
33
|
-
- **
|
|
34
|
-
- **
|
|
29
|
+
- **Organization-model feature** (`OrganizationModelFeature`) -- a single entry in `OrganizationModel.features`. Combines access gating (`enabled`), semantic grouping (`entityIds`, `surfaceIds`, `resourceIds`, `capabilityIds`), and display metadata (`label`, `description`, `color`, `icon`). Seven defaults: `crm`, `lead-gen`, `projects`, `operations`, `monitoring`, `settings`, `seo`.
|
|
30
|
+
- **Shell FeatureModule** -- a manifest-backed UI feature registered with `ElevasisFeaturesProvider`. Declares `featureId` matching an org-model feature ID. Seven manifest-backed: `lead-gen`, `crm`, `projects`, `operations`, `monitoring`, `settings`, `seo`. Two utility (no manifest): `auth`, `dashboard`.
|
|
35
31
|
|
|
36
|
-
**featureKey** (nav-item) -- optional field on `FeatureNavEntry` and `FeatureNavLink` that gates a specific nav item's visibility independently of the module's `
|
|
32
|
+
**featureKey** (nav-item) -- optional field on `FeatureNavEntry` and `FeatureNavLink` that gates a specific nav item's visibility independently of the module's `featureId`. Must match a feature ID in the org model. May be more specific than the module's `featureId` when a nested link needs a narrower gate.
|
|
37
33
|
|
|
38
34
|
**FeatureGuard** -- route-level feature gate from `@elevasis/ui/features/auth`. Blocks access to a route when the resolved org model has the relevant feature key disabled. Must nest inside `ProtectedRoute`. Distinct from `AdminGuard` (admin membership) and `ProtectedRoute` (authentication).
|
|
39
35
|
|
|
40
|
-
**FeatureModule** -- the manifest contract each shell feature provides to `ElevasisFeaturesProvider`. Defined in `@repo/ui`, NOT `@repo/core`. Fields: `key`, `
|
|
36
|
+
**FeatureModule** -- the manifest contract each shell feature provides to `ElevasisFeaturesProvider`. Defined in `@repo/ui`, NOT `@repo/core`. Fields: `key`, `featureId` (required -- replaces the former `accessFeatureKey`), `capabilityIds`, `navEntry`, `sidebar`, `subshellRoutes`, `organizationGraph` (Operations-only). Template consumers build a local manifest array and pass it to `ElevasisFeaturesProvider` in `__root.tsx`.
|
|
41
37
|
|
|
42
38
|
**Foundations** -- the adapter layer in `foundations/` (two modules: `config/organization-model.ts` and `types/index.ts`). Source package with no build step. Depends only on `@elevasis/core` (npm) and `zod`. Never import `@repo/core` from foundations -- that would break standalone deployment.
|
|
43
39
|
|
|
44
40
|
**Manifest** -- a `FeatureModule` instance that declares what one shell feature contributes at runtime (nav, sidebar, subshell routes, access key, semantic references). The provider registers an array of manifests at startup and validates them against the resolved org model.
|
|
45
41
|
|
|
46
|
-
**MembershipFeatureConfig** -- per-member-per-org feature overrides stored in `org_memberships.config`.
|
|
42
|
+
**MembershipFeatureConfig** -- per-member-per-org feature overrides stored in `org_memberships.config`. The `features` field is now `Record<string, boolean>` -- any feature ID can be overridden per member. Previously hardcoded to five keys (`operations`, `monitoring`, `acquisition`, `delivery`, `seo`); now open to any feature ID in the org model.
|
|
47
43
|
|
|
48
44
|
**OrganizationModel** -- the top-level semantic contract for an organization. Published from `@elevasis/core/organization-model`. In the template, authored in `foundations/config/organization-model.ts` and exported as `canonicalOrganizationModel` (passed to `ElevasisFeaturesProvider`) and `organizationModel` (enriched shape for app-local use).
|
|
49
45
|
|
|
50
|
-
**
|
|
46
|
+
**OrganizationModelFeature** -- the TypeScript type (`z.infer<typeof FeatureSchema>`) for a single entry in `OrganizationModel.features`. Replaces the former `OrganizationModelFeatureKey` (closed enum) and `OrganizationModelSemanticDomain`. Each feature has `id`, `label`, `enabled`, and semantic grouping arrays (`entityIds`, `surfaceIds`, `resourceIds`, `capabilityIds`). Exported from `@elevasis/core/organization-model`.
|
|
51
47
|
|
|
52
48
|
**Provider / ElevasisFeaturesProvider** -- the runtime that registers manifests, resolves feature access against the org model, dispatches subshell routing via `FeatureShell`, and exposes resolved state through `useElevasisFeatures()`. Mounted in `__root.tsx`. Accepts `features`, optional `organizationModel`, and optional `appShellOverrides`.
|
|
53
49
|
|
|
54
|
-
**Resolved types (ResolvedFeatureModule, ResolvedShellNavItem)** -- provider output. `ResolvedFeatureModule` extends `FeatureModule` with `access` (enabled state) and `semantics` (merged
|
|
50
|
+
**Resolved types (ResolvedFeatureModule, ResolvedShellNavItem)** -- provider output. `ResolvedFeatureModule` extends `FeatureModule` with `access` (enabled state and `featureId`) and `semantics` (merged capability and surface IDs). `ResolvedShellNavItem` extends `FeatureNavEntry` with `placement`, `source`, and `featureId`. Both from `@elevasis/ui`.
|
|
55
51
|
|
|
56
52
|
**Resource** -- an entry in `OrganizationModel.resourceMappings` linking a deployable automation resource into the semantic model. At the registry layer, resources are `WorkflowDefinition` or `AgentDefinition` instances in a `DeploymentSpec`.
|
|
57
53
|
|
|
58
|
-
**Settings asymmetry** -- `settings` is a valid
|
|
54
|
+
**Settings asymmetry** -- `settings` is a valid feature ID (org-level, present in `OrganizationModel.features` with `enabled: true` by default). `MembershipFeatureConfig.features` is now `Record<string, boolean>`, so `settings` can technically be overridden per member, but the convention is still to gate settings visibility via `requiresAdmin` on the nav entry and `AdminGuard` on routes rather than per-member feature overrides.
|
|
59
55
|
|
|
60
56
|
**Subshell / Sidebar** -- the feature-scoped UI region rendered when the current route matches a manifest's `subshellRoutes`. Each `FeatureModule.sidebar` is a `ComponentType`. Consumers customize by composing the feature's published sidebar primitives (`CrmSidebar`, `CrmSidebarMiddle`, etc.) and assigning their component to `manifest.sidebar`.
|
|
61
57
|
|
|
62
|
-
**Surface** -- a navigable view in `OrganizationModel.navigation.surfaces`. Identified by a dotted `id` (e.g., `crm.pipeline`). Has `path`, `surfaceType`, optional `featureKey`
|
|
58
|
+
**Surface** -- a navigable view in `OrganizationModel.navigation.surfaces`. Identified by a dotted `id` (e.g., `crm.pipeline`). Has `path`, `surfaceType`, optional `featureId` gate (replaces the former `featureKey` field), and cross-reference arrays (`featureIds`, `entityIds`, `resourceIds`, `capabilityIds`). Distinct from "page": a surface is the org-model declaration; a page is the React component rendered at the route.
|
|
63
59
|
|
|
64
60
|
**Topology** -- resource relationships (`triggers`, `uses`, `approval`) declared in `DeploymentSpec.relationships` and `HumanCheckpointDefinition.routesTo`. Used for Command View graph edges.
|
|
65
61
|
|
|
@@ -69,7 +65,7 @@ This condensed version covers every ambiguity-prone term a template consumer or
|
|
|
69
65
|
|
|
70
66
|
**`@elevasis/core`** (published npm)
|
|
71
67
|
|
|
72
|
-
- `OrganizationModel`, `
|
|
68
|
+
- `OrganizationModel`, `OrganizationModelFeature`, `OrganizationModelSurface`, `OrganizationModelResourceMapping`
|
|
73
69
|
- `resolveOrganizationModel`, `defineOrganizationModel`, `DEFAULT_ORGANIZATION_MODEL`
|
|
74
70
|
- `MembershipFeatureConfig`
|
|
75
71
|
|
|
@@ -79,7 +75,7 @@ This condensed version covers every ambiguity-prone term a template consumer or
|
|
|
79
75
|
- `ResolvedFeatureModule`, `ResolvedShellNavItem`
|
|
80
76
|
- `FeatureGuard`, `AdminGuard`, `ProtectedRoute`
|
|
81
77
|
- `ElevasisFeaturesProvider`, `ElevasisCoreProvider`, `useElevasisFeatures`
|
|
82
|
-
- `
|
|
78
|
+
- `createFeatureAccessHook`
|
|
83
79
|
|
|
84
80
|
**`foundations/`** (local source package -- not published)
|
|
85
81
|
|
|
@@ -1117,113 +1117,6 @@ export type Database = {
|
|
|
1117
1117
|
},
|
|
1118
1118
|
]
|
|
1119
1119
|
}
|
|
1120
|
-
calibration_projects: {
|
|
1121
|
-
Row: {
|
|
1122
|
-
created_at: string | null
|
|
1123
|
-
description: string | null
|
|
1124
|
-
id: string
|
|
1125
|
-
name: string
|
|
1126
|
-
organization_id: string
|
|
1127
|
-
resource_id: string
|
|
1128
|
-
resource_type: string
|
|
1129
|
-
updated_at: string | null
|
|
1130
|
-
}
|
|
1131
|
-
Insert: {
|
|
1132
|
-
created_at?: string | null
|
|
1133
|
-
description?: string | null
|
|
1134
|
-
id?: string
|
|
1135
|
-
name: string
|
|
1136
|
-
organization_id: string
|
|
1137
|
-
resource_id: string
|
|
1138
|
-
resource_type: string
|
|
1139
|
-
updated_at?: string | null
|
|
1140
|
-
}
|
|
1141
|
-
Update: {
|
|
1142
|
-
created_at?: string | null
|
|
1143
|
-
description?: string | null
|
|
1144
|
-
id?: string
|
|
1145
|
-
name?: string
|
|
1146
|
-
organization_id?: string
|
|
1147
|
-
resource_id?: string
|
|
1148
|
-
resource_type?: string
|
|
1149
|
-
updated_at?: string | null
|
|
1150
|
-
}
|
|
1151
|
-
Relationships: [
|
|
1152
|
-
{
|
|
1153
|
-
foreignKeyName: "calibration_projects_organization_id_fkey"
|
|
1154
|
-
columns: ["organization_id"]
|
|
1155
|
-
isOneToOne: false
|
|
1156
|
-
referencedRelation: "organizations"
|
|
1157
|
-
referencedColumns: ["id"]
|
|
1158
|
-
},
|
|
1159
|
-
]
|
|
1160
|
-
}
|
|
1161
|
-
calibration_runs: {
|
|
1162
|
-
Row: {
|
|
1163
|
-
completed_at: string | null
|
|
1164
|
-
config_variants: Json
|
|
1165
|
-
created_at: string | null
|
|
1166
|
-
description: string | null
|
|
1167
|
-
execution_mode: string
|
|
1168
|
-
grader_model: string | null
|
|
1169
|
-
grading_rubric: Json | null
|
|
1170
|
-
id: string
|
|
1171
|
-
name: string
|
|
1172
|
-
organization_id: string
|
|
1173
|
-
project_id: string
|
|
1174
|
-
results: Json
|
|
1175
|
-
status: string
|
|
1176
|
-
test_inputs: Json
|
|
1177
|
-
}
|
|
1178
|
-
Insert: {
|
|
1179
|
-
completed_at?: string | null
|
|
1180
|
-
config_variants: Json
|
|
1181
|
-
created_at?: string | null
|
|
1182
|
-
description?: string | null
|
|
1183
|
-
execution_mode?: string
|
|
1184
|
-
grader_model?: string | null
|
|
1185
|
-
grading_rubric?: Json | null
|
|
1186
|
-
id?: string
|
|
1187
|
-
name: string
|
|
1188
|
-
organization_id: string
|
|
1189
|
-
project_id: string
|
|
1190
|
-
results?: Json
|
|
1191
|
-
status?: string
|
|
1192
|
-
test_inputs: Json
|
|
1193
|
-
}
|
|
1194
|
-
Update: {
|
|
1195
|
-
completed_at?: string | null
|
|
1196
|
-
config_variants?: Json
|
|
1197
|
-
created_at?: string | null
|
|
1198
|
-
description?: string | null
|
|
1199
|
-
execution_mode?: string
|
|
1200
|
-
grader_model?: string | null
|
|
1201
|
-
grading_rubric?: Json | null
|
|
1202
|
-
id?: string
|
|
1203
|
-
name?: string
|
|
1204
|
-
organization_id?: string
|
|
1205
|
-
project_id?: string
|
|
1206
|
-
results?: Json
|
|
1207
|
-
status?: string
|
|
1208
|
-
test_inputs?: Json
|
|
1209
|
-
}
|
|
1210
|
-
Relationships: [
|
|
1211
|
-
{
|
|
1212
|
-
foreignKeyName: "calibration_runs_organization_id_fkey"
|
|
1213
|
-
columns: ["organization_id"]
|
|
1214
|
-
isOneToOne: false
|
|
1215
|
-
referencedRelation: "organizations"
|
|
1216
|
-
referencedColumns: ["id"]
|
|
1217
|
-
},
|
|
1218
|
-
{
|
|
1219
|
-
foreignKeyName: "calibration_runs_project_id_fkey"
|
|
1220
|
-
columns: ["project_id"]
|
|
1221
|
-
isOneToOne: false
|
|
1222
|
-
referencedRelation: "calibration_projects"
|
|
1223
|
-
referencedColumns: ["id"]
|
|
1224
|
-
},
|
|
1225
|
-
]
|
|
1226
|
-
}
|
|
1227
1120
|
command_queue: {
|
|
1228
1121
|
Row: {
|
|
1229
1122
|
action_payload: Json | null
|
|
@@ -68,8 +68,6 @@ export class RLSTestContext {
|
|
|
68
68
|
notifications: string[]
|
|
69
69
|
credentials: string[]
|
|
70
70
|
activities: string[]
|
|
71
|
-
calibration_projects: string[]
|
|
72
|
-
calibration_runs: string[]
|
|
73
71
|
}
|
|
74
72
|
|
|
75
73
|
constructor() {
|
|
@@ -105,9 +103,7 @@ export class RLSTestContext {
|
|
|
105
103
|
executionMetrics: [],
|
|
106
104
|
notifications: [],
|
|
107
105
|
credentials: [],
|
|
108
|
-
activities: []
|
|
109
|
-
calibration_projects: [],
|
|
110
|
-
calibration_runs: []
|
|
106
|
+
activities: []
|
|
111
107
|
}
|
|
112
108
|
}
|
|
113
109
|
|
|
@@ -396,8 +392,6 @@ export class RLSTestContext {
|
|
|
396
392
|
* - credentials (FK: organizations, users)
|
|
397
393
|
* - sessions (FK: organizations, users)
|
|
398
394
|
* - activities (FK: organizations)
|
|
399
|
-
* - calibration_runs (FK: calibration_projects, organizations)
|
|
400
|
-
* - calibration_projects (FK: organizations)
|
|
401
395
|
*
|
|
402
396
|
* Level 3 (Organization/User dependencies):
|
|
403
397
|
* - invitations (FK: organizations, users)
|
|
@@ -505,30 +499,6 @@ export class RLSTestContext {
|
|
|
505
499
|
}
|
|
506
500
|
}
|
|
507
501
|
|
|
508
|
-
// Delete calibration_runs (FK: calibration_projects, organizations)
|
|
509
|
-
if (this.createdIds.calibration_runs.length > 0) {
|
|
510
|
-
const { error } = await this.adminClient
|
|
511
|
-
.from('calibration_runs')
|
|
512
|
-
.delete()
|
|
513
|
-
.in('id', this.createdIds.calibration_runs)
|
|
514
|
-
|
|
515
|
-
if (error) {
|
|
516
|
-
errors.push(`Failed to delete calibration_runs: ${error.message}`)
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
// Delete calibration_projects (FK: organizations)
|
|
521
|
-
if (this.createdIds.calibration_projects.length > 0) {
|
|
522
|
-
const { error } = await this.adminClient
|
|
523
|
-
.from('calibration_projects')
|
|
524
|
-
.delete()
|
|
525
|
-
.in('id', this.createdIds.calibration_projects)
|
|
526
|
-
|
|
527
|
-
if (error) {
|
|
528
|
-
errors.push(`Failed to delete calibration_projects: ${error.message}`)
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
|
-
|
|
532
502
|
// LEVEL 3: Delete organization/user relationship tables
|
|
533
503
|
|
|
534
504
|
// Delete invitations
|
|
@@ -1,320 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest'
|
|
2
|
-
import {
|
|
3
|
-
CreateCalibrationProjectSchema,
|
|
4
|
-
UpdateCalibrationProjectSchema,
|
|
5
|
-
CreateCalibrationRunSchema,
|
|
6
|
-
GradingRubricSchema,
|
|
7
|
-
GradingCriterionSchema,
|
|
8
|
-
ConfigVariantSchema
|
|
9
|
-
} from '../schemas'
|
|
10
|
-
|
|
11
|
-
describe('Calibration Schemas', () => {
|
|
12
|
-
describe('GradingCriterionSchema', () => {
|
|
13
|
-
it('validates a valid grading criterion', () => {
|
|
14
|
-
const valid = {
|
|
15
|
-
name: 'Accuracy',
|
|
16
|
-
weight: 0.5,
|
|
17
|
-
description: 'Measures factual correctness',
|
|
18
|
-
scoringGuide: 'Check all facts are correct'
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
const result = GradingCriterionSchema.safeParse(valid)
|
|
22
|
-
expect(result.success).toBe(true)
|
|
23
|
-
})
|
|
24
|
-
|
|
25
|
-
it('rejects invalid weight (> 1)', () => {
|
|
26
|
-
const invalid = {
|
|
27
|
-
name: 'Accuracy',
|
|
28
|
-
weight: 1.5,
|
|
29
|
-
description: 'Measures factual correctness',
|
|
30
|
-
scoringGuide: 'Check all facts are correct'
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
const result = GradingCriterionSchema.safeParse(invalid)
|
|
34
|
-
expect(result.success).toBe(false)
|
|
35
|
-
})
|
|
36
|
-
|
|
37
|
-
it('rejects invalid weight (< 0)', () => {
|
|
38
|
-
const invalid = {
|
|
39
|
-
name: 'Accuracy',
|
|
40
|
-
weight: -0.1,
|
|
41
|
-
description: 'Measures factual correctness',
|
|
42
|
-
scoringGuide: 'Check all facts are correct'
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
const result = GradingCriterionSchema.safeParse(invalid)
|
|
46
|
-
expect(result.success).toBe(false)
|
|
47
|
-
})
|
|
48
|
-
})
|
|
49
|
-
|
|
50
|
-
describe('GradingRubricSchema', () => {
|
|
51
|
-
it('validates a valid grading rubric', () => {
|
|
52
|
-
const valid = {
|
|
53
|
-
passingThreshold: 0.7,
|
|
54
|
-
criteria: [
|
|
55
|
-
{
|
|
56
|
-
name: 'Accuracy',
|
|
57
|
-
weight: 0.4,
|
|
58
|
-
description: 'Factual correctness',
|
|
59
|
-
scoringGuide: 'Verify claims'
|
|
60
|
-
},
|
|
61
|
-
{
|
|
62
|
-
name: 'Completeness',
|
|
63
|
-
weight: 0.6,
|
|
64
|
-
description: 'Covers all aspects',
|
|
65
|
-
scoringGuide: 'Check coverage'
|
|
66
|
-
}
|
|
67
|
-
]
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
const result = GradingRubricSchema.safeParse(valid)
|
|
71
|
-
expect(result.success).toBe(true)
|
|
72
|
-
})
|
|
73
|
-
|
|
74
|
-
it('rejects empty criteria array', () => {
|
|
75
|
-
const invalid = {
|
|
76
|
-
passingThreshold: 0.7,
|
|
77
|
-
criteria: []
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
const result = GradingRubricSchema.safeParse(invalid)
|
|
81
|
-
expect(result.success).toBe(false)
|
|
82
|
-
})
|
|
83
|
-
})
|
|
84
|
-
|
|
85
|
-
describe('ConfigVariantSchema', () => {
|
|
86
|
-
it('validates a variant with overrides', () => {
|
|
87
|
-
const valid = {
|
|
88
|
-
variantName: 'GPT-5 High Reasoning',
|
|
89
|
-
definitionOverrides: {
|
|
90
|
-
modelConfig: { model: 'gpt-5', temperature: 1 }
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
const result = ConfigVariantSchema.safeParse(valid)
|
|
95
|
-
expect(result.success).toBe(true)
|
|
96
|
-
})
|
|
97
|
-
|
|
98
|
-
it('validates a variant without overrides', () => {
|
|
99
|
-
const valid = {
|
|
100
|
-
variantName: 'Baseline'
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
const result = ConfigVariantSchema.safeParse(valid)
|
|
104
|
-
expect(result.success).toBe(true)
|
|
105
|
-
})
|
|
106
|
-
})
|
|
107
|
-
|
|
108
|
-
describe('CreateCalibrationProjectSchema', () => {
|
|
109
|
-
it('validates a valid project creation', () => {
|
|
110
|
-
const valid = {
|
|
111
|
-
resourceId: 'research-agent',
|
|
112
|
-
resourceType: 'agent' as const,
|
|
113
|
-
name: 'Q1 2025 Model Selection',
|
|
114
|
-
description: 'Finding optimal model'
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
const result = CreateCalibrationProjectSchema.safeParse(valid)
|
|
118
|
-
expect(result.success).toBe(true)
|
|
119
|
-
})
|
|
120
|
-
|
|
121
|
-
it('validates project without description', () => {
|
|
122
|
-
const valid = {
|
|
123
|
-
resourceId: 'research-agent',
|
|
124
|
-
resourceType: 'workflow' as const,
|
|
125
|
-
name: 'Q1 2025 Model Selection'
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
const result = CreateCalibrationProjectSchema.safeParse(valid)
|
|
129
|
-
expect(result.success).toBe(true)
|
|
130
|
-
})
|
|
131
|
-
|
|
132
|
-
it('rejects invalid resourceType', () => {
|
|
133
|
-
const invalid = {
|
|
134
|
-
resourceId: 'research-agent',
|
|
135
|
-
resourceType: 'invalid',
|
|
136
|
-
name: 'Q1 2025 Model Selection'
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
const result = CreateCalibrationProjectSchema.safeParse(invalid)
|
|
140
|
-
expect(result.success).toBe(false)
|
|
141
|
-
})
|
|
142
|
-
})
|
|
143
|
-
|
|
144
|
-
describe('UpdateCalibrationProjectSchema', () => {
|
|
145
|
-
it('validates name update', () => {
|
|
146
|
-
const valid = {
|
|
147
|
-
name: 'Updated Name'
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
const result = UpdateCalibrationProjectSchema.safeParse(valid)
|
|
151
|
-
expect(result.success).toBe(true)
|
|
152
|
-
})
|
|
153
|
-
|
|
154
|
-
it('validates description update', () => {
|
|
155
|
-
const valid = {
|
|
156
|
-
description: 'Updated description'
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
const result = UpdateCalibrationProjectSchema.safeParse(valid)
|
|
160
|
-
expect(result.success).toBe(true)
|
|
161
|
-
})
|
|
162
|
-
|
|
163
|
-
it('validates both fields', () => {
|
|
164
|
-
const valid = {
|
|
165
|
-
name: 'Updated Name',
|
|
166
|
-
description: 'Updated description'
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
const result = UpdateCalibrationProjectSchema.safeParse(valid)
|
|
170
|
-
expect(result.success).toBe(true)
|
|
171
|
-
})
|
|
172
|
-
})
|
|
173
|
-
|
|
174
|
-
describe('CreateCalibrationRunSchema', () => {
|
|
175
|
-
it('validates a valid calibration run', () => {
|
|
176
|
-
const valid = {
|
|
177
|
-
projectId: '123e4567-e89b-12d3-a456-426614174000',
|
|
178
|
-
name: 'Model comparison',
|
|
179
|
-
executionMode: 'single' as const,
|
|
180
|
-
testInputs: [{ query: 'Test query' }],
|
|
181
|
-
configVariants: [
|
|
182
|
-
{
|
|
183
|
-
variantName: 'GPT-5',
|
|
184
|
-
definitionOverrides: { modelConfig: { model: 'gpt-5' } }
|
|
185
|
-
}
|
|
186
|
-
]
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
const result = CreateCalibrationRunSchema.safeParse(valid)
|
|
190
|
-
expect(result.success).toBe(true)
|
|
191
|
-
})
|
|
192
|
-
|
|
193
|
-
it('validates run with grading rubric and model', () => {
|
|
194
|
-
const valid = {
|
|
195
|
-
projectId: '123e4567-e89b-12d3-a456-426614174000',
|
|
196
|
-
name: 'Model comparison',
|
|
197
|
-
testInputs: [{ query: 'Test query' }],
|
|
198
|
-
configVariants: [
|
|
199
|
-
{
|
|
200
|
-
variantName: 'GPT-5'
|
|
201
|
-
}
|
|
202
|
-
],
|
|
203
|
-
gradingRubric: {
|
|
204
|
-
passingThreshold: 0.7,
|
|
205
|
-
criteria: [
|
|
206
|
-
{
|
|
207
|
-
name: 'Accuracy',
|
|
208
|
-
weight: 1,
|
|
209
|
-
description: 'Factual correctness',
|
|
210
|
-
scoringGuide: 'Verify claims'
|
|
211
|
-
}
|
|
212
|
-
]
|
|
213
|
-
},
|
|
214
|
-
graderModel: 'gpt-5' as const
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
const result = CreateCalibrationRunSchema.safeParse(valid)
|
|
218
|
-
expect(result.success).toBe(true)
|
|
219
|
-
})
|
|
220
|
-
|
|
221
|
-
it('rejects grading rubric without grader model', () => {
|
|
222
|
-
const invalid = {
|
|
223
|
-
projectId: '123e4567-e89b-12d3-a456-426614174000',
|
|
224
|
-
name: 'Model comparison',
|
|
225
|
-
testInputs: [{ query: 'Test query' }],
|
|
226
|
-
configVariants: [
|
|
227
|
-
{
|
|
228
|
-
variantName: 'GPT-5'
|
|
229
|
-
}
|
|
230
|
-
],
|
|
231
|
-
gradingRubric: {
|
|
232
|
-
passingThreshold: 0.7,
|
|
233
|
-
criteria: [
|
|
234
|
-
{
|
|
235
|
-
name: 'Accuracy',
|
|
236
|
-
weight: 1,
|
|
237
|
-
description: 'Factual correctness',
|
|
238
|
-
scoringGuide: 'Verify claims'
|
|
239
|
-
}
|
|
240
|
-
]
|
|
241
|
-
}
|
|
242
|
-
// Missing graderModel
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
const result = CreateCalibrationRunSchema.safeParse(invalid)
|
|
246
|
-
expect(result.success).toBe(false)
|
|
247
|
-
})
|
|
248
|
-
|
|
249
|
-
it('rejects empty testInputs array', () => {
|
|
250
|
-
const invalid = {
|
|
251
|
-
projectId: '123e4567-e89b-12d3-a456-426614174000',
|
|
252
|
-
name: 'Model comparison',
|
|
253
|
-
testInputs: [],
|
|
254
|
-
configVariants: [
|
|
255
|
-
{
|
|
256
|
-
variantName: 'GPT-5'
|
|
257
|
-
}
|
|
258
|
-
]
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
const result = CreateCalibrationRunSchema.safeParse(invalid)
|
|
262
|
-
expect(result.success).toBe(false)
|
|
263
|
-
})
|
|
264
|
-
|
|
265
|
-
it('rejects more than 50 testInputs', () => {
|
|
266
|
-
const invalid = {
|
|
267
|
-
projectId: '123e4567-e89b-12d3-a456-426614174000',
|
|
268
|
-
name: 'Model comparison',
|
|
269
|
-
testInputs: Array(51).fill({ query: 'Test' }),
|
|
270
|
-
configVariants: [
|
|
271
|
-
{
|
|
272
|
-
variantName: 'GPT-5'
|
|
273
|
-
}
|
|
274
|
-
]
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
const result = CreateCalibrationRunSchema.safeParse(invalid)
|
|
278
|
-
expect(result.success).toBe(false)
|
|
279
|
-
})
|
|
280
|
-
|
|
281
|
-
it('rejects empty configVariants array', () => {
|
|
282
|
-
const invalid = {
|
|
283
|
-
projectId: '123e4567-e89b-12d3-a456-426614174000',
|
|
284
|
-
name: 'Model comparison',
|
|
285
|
-
testInputs: [{ query: 'Test' }],
|
|
286
|
-
configVariants: []
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
const result = CreateCalibrationRunSchema.safeParse(invalid)
|
|
290
|
-
expect(result.success).toBe(false)
|
|
291
|
-
})
|
|
292
|
-
|
|
293
|
-
it('rejects more than 10 configVariants', () => {
|
|
294
|
-
const invalid = {
|
|
295
|
-
projectId: '123e4567-e89b-12d3-a456-426614174000',
|
|
296
|
-
name: 'Model comparison',
|
|
297
|
-
testInputs: [{ query: 'Test' }],
|
|
298
|
-
configVariants: Array(11).fill({ variantName: 'Test' })
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
const result = CreateCalibrationRunSchema.safeParse(invalid)
|
|
302
|
-
expect(result.success).toBe(false)
|
|
303
|
-
})
|
|
304
|
-
|
|
305
|
-
it('applies default executionMode', () => {
|
|
306
|
-
const input = {
|
|
307
|
-
projectId: '123e4567-e89b-12d3-a456-426614174000',
|
|
308
|
-
name: 'Model comparison',
|
|
309
|
-
testInputs: [{ query: 'Test' }],
|
|
310
|
-
configVariants: [{ variantName: 'GPT-5' }]
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
const result = CreateCalibrationRunSchema.safeParse(input)
|
|
314
|
-
expect(result.success).toBe(true)
|
|
315
|
-
if (result.success) {
|
|
316
|
-
expect(result.data.executionMode).toBe('single')
|
|
317
|
-
}
|
|
318
|
-
})
|
|
319
|
-
})
|
|
320
|
-
})
|