@frontera-sdk/cli 1.52.45 → 1.52.47

package/src/local/local-sync-portable.ts

@@ -13,8 +13,12 @@ import { deepEqual } from './local-sync-merge'
  *   a model the workspace lacks   which model an Agent runs on is the workspace's choice: the Agent keeps
  *                                 the workspace's model selection (its modes and default), a new Agent
  *                                 takes the workspace's default, and the local model stays this computer's.
- *   a multi-turn evaluation case  `frontera local test` runs a conversation of several turns; the workspace's
- *                                 evaluation datasets hold single questions, so those cases stay in the file here.
+ *   a multi-turn evaluation case  `frontera local test` runs a conversation of several turns. A workspace whose
+ *                                 evaluation datasets hold conversations (`evaluation-conversations`, L3) takes
+ *                                 them, except one that waits before a turn (`waitSeconds`), which runs on this
+ *                                 computer only; an older workspace holds single questions, so those stay here.
+ *   an Action's `attachment`      how the local runtime deploys an Action that takes a file (L6): in the
+ *                                 workspace the person who deploys it chooses, so it stays in the file here.
  *
  * `toWorkspace` is the file as the workspace has it: what Propose sends, and
  * what status and push compare with the workspace. `toLocal` is the
@@ -29,6 +33,7 @@ export type ModelMap = Record<string, string>
 const AGENT_FILE = /^agents\/[^/]+\/agent\.json$/
 const WORKFLOW_FILE = /^workflows\/[^/]+\.ya?ml$/
 const EVAL_FILE = /^agents\/[^/]+\/evals\/[^/]+\.ya?ml$/
+const ACTION_FILE = /^blueprint\/actions\/[^/]+\.ya?ml$/
 
 const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 
@@ -62,16 +67,21 @@ interface CaseNode {
   toJSON(): unknown
 }
 
-/** An evaluation file's cases, each with whether it is a conversation of more than one turn. */
-function evaluationCases(doc: Document): Array<{ node: CaseNode; multiTurn: boolean; name: string | null; question: string | null }> {
+/**
+ * An evaluation file's cases, each with whether it stays here: a
+ * conversation of more than one turn, unless the workspace holds
+ * conversations (`conversations`) and none of its turns waits.
+ */
+function evaluationCases(doc: Document, conversations = false): Array<{ node: CaseNode; multiTurn: boolean; name: string | null; question: string | null }> {
   const cases = doc.get('cases', true)
   if (!isSeq(cases)) return []
   return cases.items.map((node) => {
     const value = (isMap(node) ? node.toJSON() : null) as { name?: unknown; conversation?: unknown; scenario?: unknown } | null
-    const turns = Array.isArray(value?.conversation) ? (value!.conversation as Array<{ user?: unknown }>) : []
+    const turns = Array.isArray(value?.conversation) ? (value!.conversation as Array<{ user?: unknown; waitSeconds?: unknown } | null>) : []
+    const waits = turns.some((t) => t?.waitSeconds !== undefined)
     return {
       node: node as CaseNode,
-      multiTurn: value?.scenario === undefined && turns.length > 1,
+      multiTurn: value?.scenario === undefined && turns.length > 1 && (!conversations || waits),
       name: typeof value?.name === 'string' ? value.name : null,
       question: typeof turns[0]?.user === 'string' ? (turns[0]!.user as string).trim() : null,
     }
@@ -84,6 +94,8 @@ export interface Portable {
   models: Array<[string, string]>
   /** Evaluation cases that stay here (more than one turn), by name. */
   keptHere: string[]
+  /** An Action file's `attachment`, its deployment choice for the local runtime, stays here (L6). */
+  attachmentKept?: boolean
   /** Models the workspace lacks that the file names here; the workspace keeps its own selection (`upstream`: its models, none for a new Agent). */
   localModels?: { slugs: string[]; upstream: string[] }
 }
@@ -93,6 +105,8 @@ export interface ModelContext {
   map?: ModelMap
   /** The workspace's slugs; null or absent: unknown, so nothing is judged missing. */
   workspace?: ReadonlySet<string> | null
+  /** The workspace's evaluation datasets hold conversations (L3), so a multi-turn case goes unless it waits. */
+  conversations?: boolean
 }
 
 interface AgentDoc {
@@ -131,18 +145,25 @@ function agentAsWorkspace(text: string, workspace: ReadonlySet<string>, upstream
   let modelConfigs: Array<Record<string, unknown>>
   if (theirs) {
     modelConfigs = theirs.modelConfigs ?? []
-    if (theirs.config && 'defaultMode' in theirs.config) config.defaultMode = theirs.config.defaultMode
-    else delete config.defaultMode
+    // The default and the fallback name modes: both are the workspace's selection.
+    for (const key of ['defaultMode', 'fallbackMode'] as const) {
+      if (theirs.config && key in theirs.config) config[key] = theirs.config[key]
+      else delete config[key]
+    }
   } else {
     modelConfigs = (doc.modelConfigs ?? []).filter((m) => !(typeof m?.modelSlug === 'string' && missing.includes(m.modelSlug)))
     if (!modelConfigs.some((m) => m.modeKey === config.defaultMode)) config.defaultMode = (modelConfigs[0]?.modeKey as string | undefined) ?? null
+    // Gone with its model, or now the default: either way no longer a fallback.
+    if (config.fallbackMode && (config.fallbackMode === config.defaultMode || !modelConfigs.some((m) => m.modeKey === config.fallbackMode))) {
+      config.fallbackMode = null
+    }
   }
   return { text: agentText({ ...doc, config, modelConfigs }), slugs: missing, upstream: slugsOf({ modelConfigs }) }
 }
 
 /** The file as the workspace has it. Unchanged text when nothing in it is this computer's. */
 export function toWorkspace(path: string, text: string, models: ModelMap | ModelContext = {}, upstream?: string | null): Portable {
-  const ctx: ModelContext = 'map' in models || 'workspace' in models ? (models as ModelContext) : { map: models as ModelMap }
+  const ctx: ModelContext = 'map' in models || 'workspace' in models || 'conversations' in models ? (models as ModelContext) : { map: models as ModelMap }
   const map = ctx.map ?? {}
   if (EVAL_FILE.test(path)) {
     let doc: Document
@@ -152,13 +173,20 @@ export function toWorkspace(path: string, text: string, models: ModelMap | Model
       return { text, models: [], keptHere: [] }
     }
     if (doc.errors.length) return { text, models: [], keptHere: [] }
-    const cases = evaluationCases(doc)
+    const cases = evaluationCases(doc, ctx.conversations)
     const kept = cases.filter((c) => c.multiTurn)
     if (!kept.length) return { text, models: [], keptHere: [] }
     const seq = doc.get('cases', true) as { items: unknown[] }
     seq.items = seq.items.filter((_, i) => !cases[i]!.multiTurn)
     return { text: doc.toString(), models: [], keptHere: kept.map((c) => c.name ?? `case ${cases.indexOf(c) + 1}`) }
   }
+  // An Action's `attachment` is how this project's runtime deploys it; in the workspace the person deploying it chooses (L6).
+  if (ACTION_FILE.test(path)) {
+    const doc = parseDocument(text)
+    if (doc.errors.length || !doc.has('attachment')) return { text, models: [], keptHere: [] }
+    doc.delete('attachment')
+    return { text: doc.toString(), models: [], keptHere: [], attachmentKept: true }
+  }
   const { text: swapped, swapped: done } = swapModels(path, text, map)
   if (AGENT_FILE.test(path) && ctx.workspace) {
     const local = agentAsWorkspace(swapped, ctx.workspace, upstream)
@@ -175,7 +203,7 @@ export function toWorkspace(path: string, text: string, models: ModelMap | Model
  * as the last pull left it (`base`); one the workspace changed comes here.
  */
 export function toLocal(path: string, remote: string, local: string | null, models: ModelMap | ModelContext = {}, base?: string | null): string {
-  const ctx: ModelContext = 'map' in models || 'workspace' in models ? (models as ModelContext) : { map: models as ModelMap }
+  const ctx: ModelContext = 'map' in models || 'workspace' in models || 'conversations' in models ? (models as ModelContext) : { map: models as ModelMap }
   const map = ctx.map ?? {}
   if (EVAL_FILE.test(path)) {
     if (local === null) return remote
@@ -188,7 +216,7 @@ export function toLocal(path: string, remote: string, local: string | null, mode
       return remote
     }
     if (ours.errors.length || theirs.errors.length) return remote
-    const kept = evaluationCases(ours).filter((c) => c.multiTurn)
+    const kept = evaluationCases(ours, ctx.conversations).filter((c) => c.multiTurn)
     if (!kept.length) return remote
     const present = evaluationCases(theirs)
     const seq = theirs.get('cases', true)
@@ -200,6 +228,15 @@ export function toLocal(path: string, remote: string, local: string | null, mode
     }
     return theirs.toString()
   }
+  if (ACTION_FILE.test(path)) {
+    // The pulled Action with this project's `attachment` put back.
+    if (local === null) return remote
+    const ours = parseDocument(local)
+    const theirs = parseDocument(remote)
+    if (ours.errors.length || theirs.errors.length || !ours.has('attachment') || theirs.has('attachment')) return remote
+    theirs.set('attachment', ours.get('attachment', true))
+    return theirs.toString()
+  }
   const written = swapModels(path, remote, reversed(map)).text
   // An Agent this computer runs on a model the workspace lacks keeps it, while the workspace's selection is as it was.
   if (AGENT_FILE.test(path) && local !== null && base && ctx.workspace) {
@@ -207,8 +244,16 @@ export function toLocal(path: string, remote: string, local: string | null, mode
     const theirs = parseAgent(written)
     const before = parseAgent(base)
     const localOnly = agentAsWorkspace(swapModels(path, local, map).text, ctx.workspace, base)
-    if (ours && theirs && before && localOnly && deepEqual(before.modelConfigs ?? [], parseAgent(remote)?.modelConfigs ?? []) && deepEqual(before.config?.defaultMode ?? null, parseAgent(remote)?.config?.defaultMode ?? null)) {
-      return agentText({ ...theirs, config: { ...(theirs.config ?? {}), defaultMode: ours.config?.defaultMode ?? null }, modelConfigs: ours.modelConfigs ?? [] })
+    if (ours && theirs && before && localOnly && deepEqual(before.modelConfigs ?? [], parseAgent(remote)?.modelConfigs ?? []) && deepEqual(before.config?.defaultMode ?? null, parseAgent(remote)?.config?.defaultMode ?? null) && deepEqual(before.config?.fallbackMode ?? null, parseAgent(remote)?.config?.fallbackMode ?? null)) {
+      return agentText({
+        ...theirs,
+        config: {
+          ...(theirs.config ?? {}),
+          defaultMode: ours.config?.defaultMode ?? null,
+          ...(ours.config && 'fallbackMode' in ours.config ? { fallbackMode: ours.config.fallbackMode } : {}),
+        },
+        modelConfigs: ours.modelConfigs ?? [],
+      })
     }
   }
   return written
@@ -251,9 +296,13 @@ export function portableNotes(files: Record<string, string>, models: ModelMap |
       const there = p.localModels.upstream.map((s) => names[s] ?? s)
       notes.push(`Runs on ${here} here, a model the workspace doesn’t have: ${upstream[path] ? `the workspace keeps its own model selection${there.length ? ` (${there.join(', ')})` : ''}` : there.length ? `the workspace runs it on ${there.join(', ')}` : 'the workspace runs it on its default model'}`)
     }
+    if (p.attachmentKept) notes.push('Its attachment stays in this project: it is how the local runtime deploys the Action, and in the workspace the person who deploys it chooses (`frontera action deploy --attach … --file-input …`)')
     if (p.keptHere.length) {
       const n = p.keptHere.length
-      notes.push(`${n} multi-turn case${n === 1 ? '' : 's'} stay${n === 1 ? 's' : ''} here (${p.keptHere.slice(0, 3).join(', ')}${n > 3 ? ', …' : ''}): the workspace’s evaluation datasets hold single questions, so \`frontera local test\` runs ${n === 1 ? 'it' : 'them'} on this computer only`)
+      const why = (models as ModelContext).conversations
+        ? `${n === 1 ? 'it waits' : 'they wait'} before a turn, and the workspace’s runner sends each turn as soon as the reply before it ends`
+        : 'the workspace’s evaluation datasets hold single questions'
+      notes.push(`${n} multi-turn case${n === 1 ? '' : 's'} stay${n === 1 ? 's' : ''} here (${p.keptHere.slice(0, 3).join(', ')}${n > 3 ? ', …' : ''}): ${why}, so \`frontera local test\` runs ${n === 1 ? 'it' : 'them'} on this computer only`)
     }
   }
   for (const [from, to] of swapped) notes.push(`Uses ${from} here; the workspace has the same model as ${names[to] ?? to}`)

package/src/local/local-sync-precheck.ts

@@ -69,12 +69,41 @@ export function fixFor(check: Check): string | null {
   const model = /the model (\S+) is not in this workspace/.exec(check.message)
   if (model) return `choose the workspace model it goes as (\`frontera local sync map-model ${model[1]} <workspace model>\`)`
   if (check.code === 'unresolved') return 'propose what it names too, or create that in the workspace first'
+  if (check.code === 'object_set_unresolved') return 'propose the set’s file (blueprint/object-sets/<name>.yaml) in this request too, or create the saved object set in the workspace first'
+  if (check.code === 'name_taken_in_organization') return 'rename the object type here to a more specific API name, or ask a Blueprint Steward to grant the workspace the organization’s type'
   return null
 }
 
+/** One step of Apply's plan, as the check returns it (N3): in Apply's order, with what it needs. */
+export interface ApplyStep {
+  key: string
+  phase: string
+  ref: string
+  label: string
+  /** `afterApply`: Apply goes ahead and leaves the step for a person afterwards (N10). */
+  needs: Array<{ code: string; message: string; fix: string; met: boolean; afterApply?: boolean }>
+}
+
+/** The plan from the check's answer; empty from a platform that sends none. */
+export function planOf(answer: unknown): ApplyStep[] {
+  const plan = (answer as { plan?: unknown } | null)?.plan
+  return Array.isArray(plan) ? (plan as ApplyStep[]).filter((s) => s && typeof s.key === 'string' && Array.isArray(s.needs)) : []
+}
+
+/**
+ * The steps Apply would meet that the project's credentials can't take:
+ * a grant a Blueprint Steward makes, an Action's deployment decision. They
+ * don't stop a proposal; they are what a person does before Apply, or, for
+ * a new Agent's grants, after it (`after`), so Propose shows them under
+ * **Needs a person**.
+ */
+export function needsAPerson(plan: ApplyStep[]): Array<{ step: string; ref: string; code: string; message: string; fix: string; after?: true }> {
+  return plan.flatMap((s) => s.needs.filter((n) => !n.met).map((n) => ({ step: s.label, ref: s.ref, code: n.code, message: n.message, fix: n.fix, ...(n.afterApply ? { after: true as const } : {}) })))
+}
+
 /** The message without the file it names, and the file: a row says why first. */
 export function splitCheckMessage(message: string): { why: string; file: string | null } {
-  const m = /^((?:agents|workflows|functions|skills|knowledge|blueprint|mcp|chat-apps|apps)\/[^:\s]+):\s+(.*)$/s.exec(message)
+  const m = /^((?:agents|workflows|functions|skills|knowledge|blueprint|datasets|mcp|chat-apps|apps)\/[^:\s]+):\s+(.*)$/s.exec(message)
   if (!m) return { why: message, file: null }
   return { why: m[2]!.charAt(0).toUpperCase() + m[2]!.slice(1), file: m[1]! }
 }

package/src/local/local-sync-push.ts

@@ -31,7 +31,9 @@ import { dropBase, SYNC_DIR, writeBase, type RemoteSnapshot } from './local-sync
  *
  *   what is pushed     items changed here (a merge or a resolve leaves them so) and items only here
  *   what is refused    an item the workspace changed too (pull and merge first), one with conflicts,
- *                      a deletion, and kinds a request does not carry (plugins, evaluations, datasets)
+ *                      a deletion, and kinds a request does not carry (the brand, Library installations)
+ *   datasets           a blank dataset's columns, listed when an object type the push proposes reads it
+ *                      or when named, never its rows
  *   each item          the lock's id, the revision its base is (the Blueprint's for a Blueprint file),
  *                      every file of the item, and for a Function its `function deploy` build
  *   the request        `.frontera/sync/requests.json` (not committed): the current one, so the next
@@ -42,18 +44,18 @@ import { dropBase, SYNC_DIR, writeBase, type RemoteSnapshot } from './local-sync
  */
 
 /** The kinds a change request carries. */
-export const PUSH_KINDS: readonly SyncKind[] = ['blueprint', 'skill', 'knowledge', 'plugin', 'function', 'workflow', 'agent', 'evaluation', 'chat-app', 'lessons']
+export const PUSH_KINDS: readonly SyncKind[] = ['blueprint', 'dataset', 'skill', 'knowledge', 'plugin', 'function', 'workflow', 'agent', 'evaluation', 'chat-app', 'lessons', 'app']
 
 /** Why the other kinds are not proposed, in the reader's words. */
 export const NOT_PROPOSED: Partial<Record<SyncKind, string>> = {
-  dataset: 'Datasets aren’t proposed; upload them on the platform',
   brand: 'The organization’s brand is set on the platform, in Settings; it isn’t proposed from a project',
   library: 'Library installations are made from the library on the platform; this file only records them',
-  app: 'An App’s code ships with its own project, and the Functions it may start are set on the platform',
 }
 
 /** Kinds the platform cannot delete, and what to do instead. */
 const NOT_DELETED: Partial<Record<SyncKind, string>> = {
+  app: 'A Custom App is archived on the platform, in its settings; a change request sets its name, icon and the Functions it may start',
+  dataset: 'A dataset holds the organization’s records and other workspaces may read it: archive it on the platform (Blueprint, Datasets)',
   function: 'A Function can’t be deleted in a workspace: switch it off on the platform, or keep its file',
   lessons: 'An Agent’s rules are retired on the platform, where they are reviewed: a change request only proposes new ones',
 }
@@ -130,7 +132,7 @@ export interface PushItem {
   documents?: PushDocuments
 }
 
-const BLUEPRINT_DIRS = ['semantic-types', 'shared-fields', 'object-types', 'link-types', 'metrics', 'actions'] as const
+const BLUEPRINT_DIRS = ['semantic-types', 'shared-fields', 'object-types', 'link-types', 'metrics', 'actions', 'object-sets'] as const
 
 const BLUEPRINT_WORD: Record<string, string> = {
   'semantic-types': 'semantic type',
@@ -139,6 +141,7 @@ const BLUEPRINT_WORD: Record<string, string> = {
   'link-types': 'link type',
   metrics: 'metric',
   actions: 'Action',
+  'object-sets': 'saved object set',
 }
 
 const SKIP_DIRS = new Set(['node_modules', '.frontera', '.git', 'dist', 'build', 'coverage'])
@@ -238,11 +241,23 @@ export function itemFiles(root: string, kind: SyncKind, name: string, entry?: Lo
     case 'blueprint':
       paths = [`blueprint/${name}.yaml`, `blueprint/${name}.yml`, `blueprint/${name}.json`]
       break
+    case 'dataset': {
+      // Its columns, never its rows (X6): a request carries a blank dataset's declaration.
+      const declared = datasetDeclaration(root, name)
+      if ('problem' in declared) return { files: {}, notes, problem: declared.problem }
+      notes.push(...declared.notes)
+      return { files: { [`datasets/${name}.json`]: declared.text }, notes }
+    }
+    case 'app':
+      // Its settings (X6): its code is its own project, deployed from apps/<slug>/, and a release goes with `--release`.
+      paths = [`apps/${name}.json`]
+      notes.push('Proposes its name, description, icon and the Functions it may start; its code is deployed from its own folder, and `--release` adds the deployment to go live')
+      break
     default:
       return { files: {}, notes }
   }
   // What the last pull wrote for it comes along when it is still here (an agent's files are fixed; the rest may not be).
-  if (kind !== 'agent' && kind !== 'knowledge') for (const p of Object.keys(entry?.files ?? {})) if (!paths.includes(p)) paths.push(p)
+  if (kind !== 'agent' && kind !== 'knowledge' && kind !== 'app') for (const p of Object.keys(entry?.files ?? {})) if (!paths.includes(p)) paths.push(p)
   const files: Record<string, string> = {}
   const here: Record<string, string> = {}
   const models = projectModels(root)
@@ -294,7 +309,10 @@ export function projectRefs(root: string): Map<string, { kind: SyncKind; name: s
   // The workspace's own grants beyond read: one Blueprint item, `blueprint/workspace-grants`.
   if (existsSync(join(root, WORKSPACE_GRANTS_FILE))) add('blueprint', WORKSPACE_GRANTS_NAME)
   for (const f of dirs('mcp')) if (f.endsWith('.plugin.json') && f !== '.plugin.json') add('plugin', f.slice(0, -'.plugin.json'.length))
+  for (const f of dirs('datasets')) if (f.endsWith('.json') && f !== '.json') add('dataset', f.slice(0, -'.json'.length))
   for (const f of dirs('chat-apps')) if (f.endsWith('.json') && f !== '.json') add('chat-app', f.slice(0, -'.json'.length))
+  // A Custom App's settings file (X6); its code is the folder beside it, and its release a request's own file.
+  for (const f of dirs('apps')) if (/^[a-z0-9][a-z0-9-]*\.json$/.test(f) && !f.endsWith('.release.json')) add('app', f.slice(0, -'.json'.length))
   for (const slug of dirs('agents')) if (existsSync(join(root, lessonsPath(slug)))) add('lessons', slug)
   for (const slug of dirs('agents')) {
     for (const f of dirs(`agents/${slug}/evals`)) {
@@ -312,6 +330,8 @@ function titleFor(kind: SyncKind, name: string, files: Record<string, string>, k
 
 export interface CandidateInput {
   root: string
+  /** The items the person named; a dataset is listed when named, or when an object type the push proposes reads it. */
+  named?: readonly string[]
   plan: PlanItem[]
   lock: SyncLock | null
   snapshot: RemoteSnapshot | null
@@ -473,6 +493,8 @@ export function pushCandidates(input: CandidateInput): PushItem[] {
     else if (status === 'unresolved') block('conflicts', BLOCK_REASON.conflicts)
     else if (deleted && NOT_DELETED[kind]) block('shape', NOT_DELETED[kind]!)
     // Never proposed as a deletion: the platform refuses one, since deleting the file changes no grant.
+    // A saved object set is saved by Apply, never removed: what else names it would stop resolving (X6).
+    else if (deleted && ref.startsWith('blueprint/object-sets/')) block('shape', 'A change request doesn’t remove a saved object set: remove it on the platform, or put its file back')
     else if (deleted && ref === WORKSPACE_GRANTS_REF) block('shape', `Deleting ${WORKSPACE_GRANTS_FILE} changes no grant: to take read_sensitive away, put the file back and list the type with \`permissions: [read]\``)
     else if (status === 'deleted-remote') block('workspace-deleted', BLOCK_REASON['workspace-deleted'])
     else if (deleted) {
@@ -497,12 +519,63 @@ export function pushCandidates(input: CandidateInput): PushItem[] {
     }
     out.push(item)
   }
-  return out
+  // A dataset goes with what reads it: listed every push otherwise, since no pull records it (X6).
+  // A platform without `change-request-datasets` refuses the kind, so none is listed for it.
+  const takesDatasets = !!snapshot?.capabilities.includes('change-request-datasets')
+  const read = new Set(out.filter((i) => i.proposable && i.ref.startsWith('blueprint/object-types/')).flatMap((i) => Object.values(i.files).map(backingDataset)).filter((d): d is string => !!d))
+  return out.filter((i) => i.kind !== 'dataset' || (takesDatasets && (read.has(i.name) || input.named?.includes(i.ref))))
+}
+
+/** The dataset an object type's file reads (`backing.dataset`), when it names one. */
+function backingDataset(text: string): string | null {
+  try {
+    const dataset = (parseYaml(text) as { backing?: { dataset?: unknown } } | null)?.backing?.dataset
+    return typeof dataset === 'string' ? dataset : null
+  } catch {
+    return null
+  }
+}
+
+/**
+ * `datasets/<apiName>.json` as a request carries it: a blank declaration of
+ * its columns and key (X6). A blank dataset goes as written. A file dataset
+ * goes as its columns when the file declares every included column's type
+ * and whether it may be empty, so its contract is in the file and not only in
+ * the CSV; its rows stay here. A Source dataset is made on the platform.
+ */
+export function datasetDeclaration(root: string, name: string): { text: string; notes: string[] } | { problem: string } {
+  const path = `datasets/${name}.json`
+  const text = readLocal(root, path)
+  if (text === null) return { problem: `${path}: no file` }
+  let doc: Record<string, unknown>
+  try {
+    doc = JSON.parse(text) as Record<string, unknown>
+  } catch (err) {
+    return { problem: `${path} could not be read: ${(err as Error).message}` }
+  }
+  const kind = doc.kind ?? 'blank'
+  const common = {
+    apiName: name,
+    ...(typeof doc.displayName === 'string' ? { displayName: doc.displayName } : {}),
+    ...(typeof doc.description === 'string' ? { description: doc.description } : {}),
+    kind: 'blank',
+  }
+  const key = Array.isArray(doc.keyColumns) && doc.keyColumns.length ? { keyColumns: doc.keyColumns, deterministicKeyConfirmed: doc.deterministicKeyConfirmed === true } : {}
+  if (kind === 'blank') return { text: `${JSON.stringify({ ...common, columns: doc.columns, ...key }, null, 2)}\n`, notes: ['Proposes its columns: Apply creates the dataset, with no rows, when the organization has none of this name'] }
+  if (kind === 'source') return { problem: 'A Source dataset reads a connected Source: create it on the platform (Blueprint, Datasets)' }
+  const included = Array.isArray(doc.includedColumns) ? doc.includedColumns.filter((c): c is string => typeof c === 'string') : []
+  const declared = new Map((Array.isArray(doc.columns) ? doc.columns : []).map((c) => [(c as { name?: string }).name, c as { databaseType?: string; nullable?: boolean }]))
+  const open = included.filter((c) => typeof declared.get(c)?.databaseType !== 'string' || typeof declared.get(c)?.nullable !== 'boolean')
+  if (!included.length || open.length) {
+    return { problem: `Its columns come from the CSV${open.length ? ` (${open.slice(0, 5).join(', ')}${open.length > 5 ? '…' : ''})` : ''}: to propose the dataset, give each included column its databaseType and nullable under columns, so the file states the whole contract` }
+  }
+  const columns = included.map((c) => ({ name: c, databaseType: declared.get(c)!.databaseType, nullable: declared.get(c)!.nullable }))
+  return { text: `${JSON.stringify({ ...common, columns, ...key }, null, 2)}\n`, notes: ['Proposes its columns as a blank dataset: its rows (the CSV) stay here, and Apply creates the dataset, empty, when the organization has none of this name'] }
 }
 
 /** Whether pushing these needs the organization key at Apply: any Blueprint item does. */
 export function needsOrganizationKey(items: Array<{ kind: string; proposable?: boolean }>): boolean {
-  return items.some((i) => i.kind === 'blueprint' && i.proposable !== false)
+  return items.some((i) => (i.kind === 'blueprint' || i.kind === 'dataset') && i.proposable !== false)
 }
 
 // ── What the platform takes ─────────────────────────────────────────────────

package/src/local/local-sync-remote.ts

@@ -298,7 +298,7 @@ function stripRow(row: Record<string, unknown>): Record<string, unknown> {
  * evaluator settings, and its work-item statuses and labels.
  */
 const AGENT_CONFIG_KEYS = [
-  'name', 'description', 'avatarSeed', 'avatarConfig', 'defaultMode', 'utilityMode', 'utilityOverrides', 'agentType', 'agentModeEnabled', 'disableWebBrowsing',
+  'name', 'description', 'avatarSeed', 'avatarConfig', 'defaultMode', 'fallbackMode', 'utilityMode', 'utilityOverrides', 'agentType', 'agentModeEnabled', 'disableWebBrowsing',
   'plugins', 'features', 'memory', 'voice', 'compression', 'streaming', 'upload', 'planning', 'delegation', 'modelRouting', 'toolPhasing', 'stopConditions',
   'maxCostPerRun', 'permissions', 'functionGuardrails', 'audit', 'harness', 'statuses', 'defaultStatus', 'defaultLabels',
 ]
@@ -1453,7 +1453,7 @@ async function readEvalDatasets(ws: SyncClient, first: unknown): Promise<EvalDat
 }
 
 /** The check types the local runner grades in code, as the platform's expectation grading does. */
-const LOCAL_CHECKS = new Set(['contains', 'regex', 'tool_called', 'tool_not_called'])
+const LOCAL_CHECKS = new Set(['contains', 'not_contains', 'regex', 'tool_called', 'tool_not_called'])
 
 /**
  * A scenario's simulated results and expected actions name Functions and
@@ -1482,7 +1482,7 @@ function scenarioNames(value: unknown, byId: Map<string, { kind: RefKind; name:
 
 export interface EvalCaseOut {
   name: string
-  conversation?: Array<{ user: string }>
+  conversation?: Array<{ user: string } | { assistant: string }>
   expected?: string
   expectedContains?: string[]
   checks?: Array<Record<string, unknown>>
@@ -1525,15 +1525,18 @@ export function evalCase(item: EvalItem, name: string, byId: Map<string, { kind:
   for (const e of expectations) {
     const type = String(e.check?.type ?? e.check?.kind ?? '')
     if (e.check && LOCAL_CHECKS.has(type)) {
-      if (type === 'contains' && typeof e.check.value === 'string' && e.check.value) contains.push(e.check.value)
-      else if (type !== 'contains') checks.push({ ...(typeof e.text === 'string' && e.text ? { text: e.text } : {}), ...e.check, type })
+      // A phrase on the last reply is `expectedContains`; one on a named turn stays a check, with its turn (L3).
+      if (type === 'contains' && typeof e.check.value === 'string' && e.check.value && e.check.turn === undefined) contains.push(e.check.value)
+      else if (type !== 'contains' || e.check.turn !== undefined) checks.push({ ...(typeof e.text === 'string' && e.text ? { text: e.text } : {}), ...e.check, type })
     } else if (typeof e.text === 'string' && e.text.trim()) judged.push(e.text.trim())
   }
   const expected = textOf(item.expectedOutput)
+  // A conversation item (L3): its turns in order, as `local test` sends them.
+  const turns = Array.isArray(input?.turns) ? (input!.turns as Array<{ role?: unknown; text?: unknown }>).flatMap((t) => (typeof t?.text === 'string' && t.text.trim() ? [t.role === 'assistant' ? { assistant: t.text } : { user: t.text }] : [])) : []
   return {
     case: {
       name,
-      conversation: [{ user: question }],
+      conversation: turns.filter((t) => 'user' in t).length > 1 ? turns : [{ user: question }],
       ...(contains.length ? { expectedContains: contains } : expected ? { expected } : {}),
       ...(checks.length ? { checks } : {}),
       ...(judged.length ? { judged } : {}),
@@ -1616,7 +1619,7 @@ async function readEvaluations(source: SyncSource, lists: Lists, directory: Dire
       })
       if (conversations) notes.push(`${conversations} case${conversations === 1 ? ' replays' : 's replay'} a real conversation from the workspace, so ${conversations === 1 ? 'it stays' : 'they stay'} on the platform`)
       if (noQuestion) notes.push(`${noQuestion} item${noQuestion === 1 ? '' : 's'} had no question the local runner can send`)
-      if (judged) notes.push(`${judged} expectation${judged === 1 ? ' is' : 's are'} graded by the platform’s judge; ${judged === 1 ? 'it is' : 'they are'} kept in the file (\`judged\`) and not graded locally`)
+      if (judged) notes.push(`${judged} expectation${judged === 1 ? ' is' : 's are'} graded by a judge: ${judged === 1 ? 'it is' : 'they are'} kept in the file (\`judged\`), and \`local test\` grades ${judged === 1 ? 'it' : 'them'} with the local runtime’s grading model`)
       if (gold) notes.push(`${gold} gold case${gold === 1 ? '' : 's'} (human-certified)`)
       if (unnamed.size) notes.push(`A scenario names ${unnamed.size} Function${unnamed.size === 1 ? ' or workflow' : 's or workflows'} by a platform id the workspace no longer has`)
       if (ds.totalItems > ds.items.length) notes.push(`The first ${ds.items.length} of ${ds.totalItems} items were read`)

package/src/local/local-test-models.ts

@@ -0,0 +1,173 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { PlatformApi } from '../api/platform-api'
+import { CliError } from '../errors'
+import type { LocalProject } from './local-project'
+import type { LocalCredentials } from './local-state'
+import { caseKey, compareWithLastRun, type LastRun, type RunComparison } from './local-test-baseline'
+import type { CaseResult } from './local-test-runner'
+import { acquireLock } from './local-writers'
+
+/**
+ * Comparing models on the local runtime (Forge improvement plan L1, T9).
+ *
+ * `frontera local test --model <slug>` runs the cases with each Agent in
+ * scope on that model: its draft's models are swapped for the run and put
+ * back after it, whatever happened, so the files and the published versions
+ * never change. Every run is kept under `.frontera/local/test-runs/`, with
+ * the model it ran on, and `--baseline <run>` compares a run with one kept
+ * earlier: which cases the other model made fail, and which it fixed.
+ */
+
+export interface SavedRun {
+  id: string
+  at: string
+  /** The model `--model` swapped in; null for the Agents' own. */
+  model: string | null
+  cases: LastRun
+}
+
+const DIR = 'test-runs'
+const KEEP = 20
+
+/** The draft's models, every one on `slug`; null when the draft names none (it runs on the runtime's default). */
+export function withModel(snapshot: Record<string, unknown>, slug: string): Array<Record<string, unknown>> | null {
+  const configs = Array.isArray(snapshot.modelConfigs) ? (snapshot.modelConfigs as Array<Record<string, unknown>>) : []
+  if (!configs.length) return null
+  return configs.map((c) => ({ ...c, modelSlug: slug }))
+}
+
+export interface DraftAccess {
+  draft(agentId: string): Promise<{ snapshot: Record<string, unknown> } | null>
+  patch(agentId: string, patch: Record<string, unknown>): Promise<unknown>
+}
+
+/**
+ * Put each Agent's draft on `model`, and return how to put it back. An Agent
+ * whose draft names no model is left as it is, and said.
+ */
+export async function swapModels(access: DraftAccess, agents: Array<{ slug: string; id: string }>, model: string): Promise<{ swapped: string[]; unchanged: string[]; restore: () => Promise<string[]> }> {
+  const before: Array<{ slug: string; id: string; modelConfigs: unknown }> = []
+  const unchanged: string[] = []
+  try {
+    for (const a of agents) {
+      const draft = await access.draft(a.id)
+      const next = draft ? withModel(draft.snapshot, model) : null
+      if (!draft || !next) {
+        unchanged.push(a.slug)
+        continue
+      }
+      await access.patch(a.id, { modelConfigs: next })
+      before.push({ slug: a.slug, id: a.id, modelConfigs: draft.snapshot.modelConfigs })
+    }
+  } catch (err) {
+    // What was swapped before the failure is put back before it is said.
+    for (const b of before.reverse()) await access.patch(b.id, { modelConfigs: b.modelConfigs }).catch(() => {})
+    throw err
+  }
+  return {
+    swapped: before.map((b) => b.slug),
+    unchanged,
+    // Each one tried, so one that fails doesn't keep the rest on the other model; the ones that failed are returned.
+    restore: async () => {
+      const failed: string[] = []
+      for (const b of [...before].reverse()) await access.patch(b.id, { modelConfigs: b.modelConfigs }).catch(() => failed.push(b.slug))
+      return failed
+    },
+  }
+}
+
+const runDir = (project: LocalProject) => join(project.stateDir, DIR)
+
+/** Keep this run, newest last; the oldest beyond twenty go. */
+export function saveTestRun(project: LocalProject, results: CaseResult[], model: string | null, now = new Date()): SavedRun {
+  const at = now.toISOString()
+  const id = `${at.replace(/[:.]/g, '-').replace(/Z$/, '')}${model ? `-${model.replace(/[^A-Za-z0-9._-]/g, '_')}` : ''}`
+  const cases: LastRun = {}
+  for (const r of results) if (!r.notRun) cases[caseKey(r)] = { passed: r.passed, at, ms: r.ms, ...(r.passed || !r.reason ? {} : { reason: r.reason.slice(0, 300) }) }
+  const run: SavedRun = { id, at, model, cases }
+  mkdirSync(runDir(project), { recursive: true })
+  writeFileSync(join(runDir(project), `${id}.json`), `${JSON.stringify(run, null, 2)}\n`)
+  for (const old of listTestRuns(project).slice(0, -KEEP)) rmSync(join(runDir(project), `${old}.json`), { force: true })
+  return run
+}
+
+/** The kept runs' ids, oldest first. */
+export function listTestRuns(project: LocalProject): string[] {
+  if (!existsSync(runDir(project))) return []
+  return readdirSync(runDir(project)).filter((f) => f.endsWith('.json')).map((f) => f.slice(0, -'.json'.length)).sort()
+}
+
+/** A kept run by its id, a prefix of it, `last` (the newest), or `last-<model>`; null when none matches. */
+export function readTestRun(project: LocalProject, ref: string): SavedRun | null {
+  const ids = listTestRuns(project)
+  const read = (id: string | undefined) => {
+    if (!id) return null
+    try {
+      return JSON.parse(readFileSync(join(runDir(project), `${id}.json`), 'utf8')) as SavedRun
+    } catch {
+      return null
+    }
+  }
+  if (ref === 'last') return read(ids[ids.length - 1])
+  if (ref === 'last-own') return [...ids].reverse().map(read).find((r) => r && r.model === null) ?? null
+  if (ref.startsWith('last-')) return [...ids].reverse().map(read).find((r) => r?.model === ref.slice('last-'.length)) ?? null
+  const matches = ids.filter((id) => id.startsWith(ref))
+  return matches.length === 1 ? read(matches[0]) : null
+}
+
+/** This run against a kept one: the cases the change of model (or of anything else) made fail, fixed or left failing. */
+export function compareWithRun(results: CaseResult[], baseline: SavedRun): RunComparison {
+  return compareWithLastRun(results, baseline.cases)
+}
+
+export function comparisonWords(c: RunComparison, baseline: SavedRun): string {
+  const on = baseline.model ? `on ${baseline.model}` : 'on the Agents’ own models'
+  return `against run ${baseline.id} (${on}): ${c.newlyFailing.length} newly failing, ${c.fixed.length} fixed, ${c.stillFailing.length} still failing${c.firstRun.length ? `, ${c.firstRun.length} not in that run` : ''}`
+}
+
+/**
+ * Run `run` with these Agents on `model`, holding the runtime (X9) so no
+ * other run tests the swapped drafts, and put each back after, whatever
+ * happened. `stuck` names any that could not be put back.
+ */
+export async function runOnModel<T>(
+  project: LocalProject,
+  link: { serviceUrl: string; creds: LocalCredentials },
+  slugs: readonly string[],
+  model: string,
+  run: (on: { swapped: string[]; unchanged: string[] }) => Promise<T>,
+  deps: { access?: DraftAccess; agentIds?: () => Promise<Array<{ id?: string; agentId?: string }>> } = {},
+): Promise<{ value: T; swapped: string[]; unchanged: string[]; stuck: string[] }> {
+  const release = acquireLock(project.stateDir, 'local test --model')
+  try {
+    const platform = new PlatformApi(link.serviceUrl, link.creds.workspaceKey ?? link.creds.orgKey, link.creds.workspaceId)
+    const access = deps.access ?? { draft: (id: string) => platform.agentDraft(id), patch: (id: string, patch: Record<string, unknown>) => platform.patchAgentDraft(id, patch) }
+    let swap: Awaited<ReturnType<typeof swapModels>>
+    try {
+      const listed = await (deps.agentIds ?? (() => platform.allAgents() as Promise<Array<{ id?: string; agentId?: string }>>))()
+      const agents = [...new Set(slugs)].flatMap((slug) => {
+        const row = listed.find((a) => a.agentId === slug)
+        return row?.id ? [{ slug, id: row.id }] : []
+      })
+      swap = await swapModels(access, agents, model)
+    } catch (err) {
+      throw new CliError(`the Agents could not be put on ${model} for the run: ${(err as Error).message}`, { code: 'LOCAL_TEST_MODEL', hint: '`frontera local models` lists the models this runtime has' })
+    }
+    let stuck: string[] = []
+    try {
+      const value = await run({ swapped: swap.swapped, unchanged: swap.unchanged })
+      stuck = await swap.restore()
+      return { value, swapped: swap.swapped, unchanged: swap.unchanged, stuck }
+    } catch (err) {
+      await swap.restore()
+      throw err
+    }
+  } finally {
+    release()
+  }
+}
+
+export const stuckWords = (stuck: readonly string[], model: string) =>
+  `${stuck.join(', ')} ${stuck.length === 1 ? 'is' : 'are'} still on ${model}: run \`frontera local dev --once --reapply\` to put the files’ models back`