@frontera-sdk/cli 1.52.28 → 1.52.30

package/src/local/local-portability.ts

@@ -0,0 +1,346 @@
+import { parse as parseYaml } from 'yaml'
+
+import { unnamedObjectBindings } from './local-objects'
+import { governedActionIds, referencesIn } from './local-sync-refs'
+
+/**
+ * Whether an item can run on the local runtime, and how (design §8). Computed
+ * from two declarations, never from error handling: what the item **needs**
+ * (governed Actions, a built-in connection, the platform's media files, the
+ * Agent's computer, calculated fields, markings), read from its files, and
+ * what the local runtime **can do**. The fetch stores the result beside each
+ * item in the snapshot; `local dev` lists the same adaptations in the build;
+ * the Blueprint outline (`local-blueprint-outline.ts`) and `resolveBlueprint`
+ * read the object-type rule from here. One set of rules, said once.
+ *
+ *   portable       applies as written
+ *   adapted        applies after listed adaptations; the files stay as they are
+ *   stubbed        depends on something platform-only; the runtime gets a stand-in
+ *   platform-only  cannot run here; named, so nothing silently refers to it
+ */
+
+export type PortabilityClass = 'portable' | 'adapted' | 'stubbed' | 'platform-only'
+
+/** What the local runtime can do. */
+export interface RuntimeCapabilities {
+  /**
+   * An Agent's computer. The service starts one only on Daytona
+   * (`agentComputerConfigured`: `DAYTONA_API_KEY` and `DAYTONA_API_URL`), which
+   * the local runtime's environment never carries; its Agents run without one
+   * and attachments go to the model.
+   */
+  computer: boolean
+  /**
+   * The platform's media files (a `media_reference` property). A type that
+   * reads a Media Set's catalog gets an empty Media Set of the same name
+   * instead (Stubbed); one that reads its extracted text or passages is left off.
+   */
+  media: boolean
+  /** Built-in connections signed in with OAuth or a service account (Google Sheets, BigQuery…). */
+  connections: boolean
+  /** Calculated properties (a location from latitude and longitude); `blueprint apply` cannot create one yet. */
+  calculatedFields: boolean
+  /** Governed Actions: `local` deploys those that write the Blueprint's own records (D3). */
+  governedActions: 'local' | 'none'
+}
+
+export const LOCAL_RUNTIME: RuntimeCapabilities = {
+  computer: false,
+  media: false,
+  connections: false,
+  calculatedFields: false,
+  governedActions: 'local',
+}
+
+/** One thing an item needs that the runtime may not have. */
+export type Need =
+  | { kind: 'action'; name: string }
+  /** A governed Action named only by the platform's id: reading its name takes an organization key. */
+  | { kind: 'action-unnamed'; count: number }
+  | { kind: 'connection'; name: string }
+  /** The workspace's Documentation plugin: served here from the runtime's documentation. */
+  | { kind: 'documentation'; name: string }
+  /** The workspace's analytics Plugin: it reads the workspace's analytics, which a local runtime does not have. */
+  | { kind: 'analytics'; name: string }
+  | { kind: 'computer' }
+  | { kind: 'media' }
+  /** A media-backed object type over a Media Set's catalog: an empty Media Set of that name stands in. */
+  | { kind: 'media-stub'; set: string }
+  | { kind: 'calculated'; fields: string[] }
+  | { kind: 'markings'; dataset: string }
+  /** A governed Action whose effect reaches outside the runtime (a transition, a command, another system). */
+  | { kind: 'external-effect' }
+  /** An Agent's Context and data the pull could not read (`bindings.json` `blueprintUnread`). */
+  | { kind: 'context-unread' }
+  /** A Chat App: published only with published Agents, and every local Agent is a draft. */
+  | { kind: 'app-draft' }
+  /** A Custom App: its code is its own project, deployed on its own; `external` when it is hosted on its own domain. */
+  | { kind: 'app-code'; external?: boolean }
+  /** An Agent's object tools whose object definition the pull couldn't name (an older platform). */
+  | { kind: 'objects-unnamed'; count: number }
+
+export interface Portability {
+  class: PortabilityClass
+  /** Each adaptation in plain words, most important first. */
+  adaptations: string[]
+  needs: Need[]
+}
+
+const RANK: Record<PortabilityClass, number> = { portable: 0, adapted: 1, stubbed: 2, 'platform-only': 3 }
+
+/** How the local runtime runs governed Actions it holds, by apiName: deployed, or switched off. */
+export type ActionFits = ReadonlyMap<string, ActionFit>
+
+export interface Context {
+  caps?: RuntimeCapabilities
+  /** The governed Actions known locally (the project's `blueprint/actions/`), and whether each runs here. */
+  actions?: ActionFits
+  /** Built-in connections as people know them ("Google Sheets"), by install name. */
+  titles?: ReadonlyMap<string, string>
+}
+
+/** One need against the runtime: its class and what it means, in words. Null when the runtime has it. */
+export function judge(need: Need, ctx: Context = {}): { class: PortabilityClass; text: string } | null {
+  const caps = ctx.caps ?? LOCAL_RUNTIME
+  switch (need.kind) {
+    case 'action': {
+      if (caps.governedActions === 'none') return { class: 'platform-only', text: `governed Action ${need.name} runs on the platform only` }
+      const fit = ctx.actions?.get(need.name)
+      // Unknown here: what the build says about it decides; at fetch it comes down with the Blueprint.
+      if (!fit || fit.runsLocally) return null
+      return { class: 'adapted', text: `governed Action ${need.name} is switched off here: ${fit.reason}` }
+    }
+    case 'action-unnamed':
+      return { class: 'platform-only', text: `runs ${need.count === 1 ? 'a governed Action' : `${need.count} governed Actions`} this key can’t name: add an organization key to bring ${need.count === 1 ? 'it' : 'them'}` }
+    case 'connection':
+      return caps.connections ? null : { class: 'stubbed', text: `${ctx.titles?.get(need.name) ?? need.name} isn’t connected here: a stand-in answers its tools with “not connected”` }
+    case 'documentation':
+      return null
+    case 'analytics':
+      return { class: 'adapted', text: 'analytics tools stay on the platform: a local runtime has no analytics' }
+    case 'computer':
+      return caps.computer ? null : { class: 'adapted', text: 'runs without its computer: attachments go to the model' }
+    case 'media':
+      return caps.media ? null : { class: 'platform-only', text: 'reads the platform’s media files, so the local runtime leaves it off' }
+    case 'media-stub':
+      return caps.media ? null : { class: 'stubbed', text: `its files stay on the platform: an empty Media Set, ${need.set}, stands in for them here` }
+    case 'calculated':
+      return caps.calculatedFields
+        ? null
+        : { class: 'adapted', text: `${need.fields.join(', ')} ${need.fields.length === 1 ? 'is' : 'are'} left off: calculated fields aren’t applied locally yet` }
+    case 'markings':
+      return { class: 'adapted', text: `reads every row: its markings come from the platform’s dataset, and ${need.dataset} has none` }
+    case 'external-effect':
+      return { class: 'stubbed', text: 'switched off here: it changes a system outside the local runtime' }
+    case 'app-draft':
+      return { class: 'adapted', text: 'saved as the owner’s draft here and opened as its preview, its Agents running their drafts: a Chat App publishes only published Agents, and the local runtime’s Agents are drafts' }
+    case 'app-code':
+      return { class: 'adapted', text: `saved here with the Functions it may start, without its code: an App’s code is its own project, deployed to this runtime on its own${need.external ? '; it is hosted on its own domain, whose sign-in stays on the platform' : ''}` }
+    case 'objects-unnamed':
+      return { class: 'adapted', text: `goes without ${need.count === 1 ? 'one of its object tools' : `${need.count} of its object tools`} here: the pull couldn’t read ${need.count === 1 ? 'its object definition' : 'their object definitions'} on the platform` }
+    case 'context-unread':
+      return { class: 'adapted', text: 'its Context and data couldn’t be read on the platform: read access to the pulled object types is assumed here until it can be read' }
+  }
+}
+
+/** The class an item gets from its needs: the most limiting one wins. */
+export function classify(needs: Need[], ctx: Context = {}): Portability {
+  let cls: PortabilityClass = 'portable'
+  const judged = needs.flatMap((n) => {
+    const j = judge(n, ctx)
+    return j ? [j] : []
+  })
+  judged.sort((a, b) => RANK[b.class] - RANK[a.class])
+  for (const j of judged) if (RANK[j.class] > RANK[cls]) cls = j.class
+  return { class: cls, adaptations: [...new Set(judged.map((j) => j.text))], needs }
+}
+
+/** "Runs locally without its computer; Google Sheets isn’t connected here." */
+export function portabilitySentence(p: Portability): string {
+  if (!p.adaptations.length) return 'Runs locally as written'
+  const [first, ...rest] = p.adaptations
+  return `${first!.charAt(0).toUpperCase()}${first!.slice(1)}${rest.length ? `; ${rest.join('; ')}` : ''}`
+}
+
+// ── Object types ─────────────────────────────────────────────────────────────
+
+type Doc = Record<string, any>
+
+/** What the local runtime does with an object type. */
+export interface ObjectTypeFit {
+  /** It reads the platform's media files (a `media_reference` property) other than a Media Set's catalog: left off the runtime. */
+  leftOff: boolean
+  /** It reads a Media Set's catalog: the runtime holds it over an empty Media Set of this API name. */
+  mediaStub?: string | null
+  /** Calculated properties, by apiName: the runtime holds the type without them. */
+  calculated: string[]
+  /** Its rows are read by markings from the platform's dataset: here it reads every row. */
+  readsEveryRow: boolean
+}
+
+/**
+ * The rule `resolveBlueprint` applies to one object type. `hasLocalDataset`
+ * says whether the project has the dataset's local file (`datasets/<name>.json`).
+ */
+/** The columns a Media Set's catalog publishes (the service's `MEDIA_CATALOG_COLUMNS`). */
+const MEDIA_CATALOG_COLUMNS = new Set(['media_item_id', 'path', 'mime_type', 'size_bytes', 'created_at', 'version', 'is_latest', 'media_reference', '_security_markings'])
+
+/**
+ * The Media Set whose catalog an object type reads, by its API name: the
+ * catalog's dataset is `media_<set>`, and every column the type reads is one
+ * the catalog publishes. Null for a type over a set's extracted text or
+ * passages (other columns), or a dataset named otherwise.
+ */
+export function mediaCatalogSet(doc: Doc): string | null {
+  const m = typeof doc.backing?.dataset === 'string' ? /^media_([A-Za-z][A-Za-z0-9_]*)$/.exec(doc.backing.dataset) : null
+  if (!m) return null
+  const mapping = (doc.backing?.mapping ?? {}) as Record<string, unknown>
+  const properties = (Array.isArray(doc.properties) ? doc.properties : []) as Array<{ apiName?: string; column?: unknown }>
+  const columns = properties.map((p) => (typeof p.column === 'string' ? p.column : typeof mapping[p.apiName ?? ''] === 'string' ? (mapping[p.apiName ?? ''] as string) : null))
+  return columns.every((c) => c !== null && MEDIA_CATALOG_COLUMNS.has(c)) ? m[1]! : null
+}
+
+export function objectTypeFit(doc: Doc, hasLocalDataset: (dataset: string) => boolean): ObjectTypeFit {
+  const properties = (Array.isArray(doc.properties) ? doc.properties : []) as Array<{ apiName?: string; valueType?: string; calculation?: unknown }>
+  if (properties.some((p) => p.valueType === 'media_reference')) {
+    const set = mediaCatalogSet(doc)
+    return set ? { leftOff: false, mediaStub: set, calculated: [], readsEveryRow: false } : { leftOff: true, calculated: [], readsEveryRow: false }
+  }
+  const calculated = properties.filter((p) => p.calculation).map((p) => String(p.apiName ?? ''))
+  const dataset = doc.backing?.dataset
+  const readsEveryRow = doc.rowAccess === 'markings' && !doc.markingsColumn && typeof dataset === 'string' && hasLocalDataset(dataset)
+  return { leftOff: false, calculated, readsEveryRow }
+}
+
+export function objectTypeNeeds(doc: Doc, hasLocalDataset: (dataset: string) => boolean): Need[] {
+  const fit = objectTypeFit(doc, hasLocalDataset)
+  if (fit.leftOff) return [{ kind: 'media' }]
+  if (fit.mediaStub) return [{ kind: 'media-stub', set: fit.mediaStub }]
+  const needs: Need[] = []
+  if (fit.calculated.length) {
+    const properties = (Array.isArray(doc.properties) ? doc.properties : []) as Array<{ apiName?: string; displayName?: string }>
+    needs.push({ kind: 'calculated', fields: fit.calculated.map((api) => properties.find((p) => p.apiName === api)?.displayName ?? api) })
+  }
+  if (fit.readsEveryRow) needs.push({ kind: 'markings', dataset: String(doc.backing?.dataset) })
+  return needs
+}
+
+// ── Governed Actions ─────────────────────────────────────────────────────────
+
+export interface ActionFit {
+  /** It writes only the Blueprint's own records, so the local runtime deploys it. */
+  runsLocally: boolean
+  /** Why not, in words. */
+  reason?: string
+  /** The object type it acts on. */
+  subject: string | null
+}
+
+const ONTOLOGY_EDITS = new Set(['create', 'update', 'delete'])
+
+/**
+ * What a governed Action does when it runs, from its file. An Action whose
+ * effect creates, updates or deletes the subject's properties is an edit over
+ * the Blueprint's records: locally that writes over the runtime's own
+ * datasets and reaches nothing else, so it is deployed (D3). A transition, a
+ * command, or an edit with nothing to set would be bound to a connector on the
+ * platform, which calls a real system: here it is published switched off.
+ */
+export function actionFit(doc: Doc): ActionFit {
+  const subject = typeof doc.subject?.objectType === 'string' ? doc.subject.objectType : null
+  const effect = typeof doc.effect?.kind === 'string' ? doc.effect.kind : null
+  if (!effect || !ONTOLOGY_EDITS.has(effect)) return { runsLocally: false, reason: 'it changes a system outside the local runtime', subject }
+  const changes = (Array.isArray(doc.businessOutcomes) ? doc.businessOutcomes : []).flatMap((o: Doc) => (Array.isArray(o?.expectedChanges) ? o.expectedChanges : []))
+  const sets = changes.some((c: Doc) => c?.kind === 'property_set' && c?.value?.kind === 'input')
+  if (effect !== 'delete' && !sets) return { runsLocally: false, reason: 'it sets no field from its inputs, so there is nothing to write locally', subject }
+  return { runsLocally: true, subject }
+}
+
+export function actionNeeds(doc: Doc): Need[] {
+  return actionFit(doc).runsLocally ? [] : [{ kind: 'external-effect' }]
+}
+
+// ── Agents and workflows ─────────────────────────────────────────────────────
+
+const COMPUTER_ON = new Set(['on-demand', 'eager'])
+
+function parse(path: string, text: string): unknown {
+  try {
+    return /\.json$/.test(path) ? JSON.parse(text) : parseYaml(text)
+  } catch {
+    return null
+  }
+}
+
+/**
+ * What an Agent's or workflow's files need: the governed Actions they name
+ * (and any still named by the platform's id), the built-in connections among
+ * the Plugins they name (`connections`, install names), and for an Agent its
+ * computer when its availability asks for one.
+ */
+/** What a built-in Plugin is, when it is not a connection to another system. */
+export type BuiltInKind = 'documentation' | 'analytics'
+
+export function itemNeeds(kind: string, files: Record<string, string>, connections: ReadonlySet<string> = new Set(), builtIns: ReadonlyMap<string, BuiltInKind> = new Map()): Need[] {
+  const needs: Need[] = []
+  const actions = new Set<string>()
+  const plugins = new Set<string>()
+  let unnamed = 0
+  for (const [path, text] of Object.entries(files)) {
+    if (!/\.(json|ya?ml)$/.test(path)) continue
+    const doc = parse(path, text)
+    if (!doc) continue
+    for (const ref of referencesIn(doc)) {
+      if (ref.kind === 'action') actions.add(ref.name)
+      if (ref.kind === 'plugin' && connections.has(ref.name)) plugins.add(ref.name)
+    }
+    unnamed += governedActionIds(doc).length
+    if (kind === 'agent' && /\/agent\.json$/.test(path)) {
+      const availability = (doc as { config?: { features?: { computerAvailability?: unknown } } }).config?.features?.computerAvailability
+      if (typeof availability === 'string' && COMPUTER_ON.has(availability)) needs.push({ kind: 'computer' })
+    }
+    if (kind === 'agent' && /\/agent\.json$/.test(path)) {
+      const unnamed = unnamedObjectBindings(doc)
+      if (unnamed) needs.push({ kind: 'objects-unnamed', count: unnamed })
+    }
+    if (kind === 'agent' && /\/bindings\.json$/.test(path) && (doc as { blueprintUnread?: unknown }).blueprintUnread === true && !Array.isArray((doc as { blueprint?: unknown }).blueprint)) needs.push({ kind: 'context-unread' })
+  }
+  if (kind === 'chat-app') needs.push({ kind: 'app-draft' })
+  if (kind === 'app') {
+    const doc = Object.values(files)[0]
+    let external = false
+    try {
+      external = !!doc && (JSON.parse(doc) as { hosting?: unknown }).hosting === 'external'
+    } catch {
+      // A file that does not parse is applied as it is; the apply says why it fails.
+    }
+    needs.push({ kind: 'app-code', ...(external ? { external } : {}) })
+  }
+  for (const name of [...actions].sort()) needs.push({ kind: 'action', name })
+  if (unnamed) needs.push({ kind: 'action-unnamed', count: unnamed })
+  for (const name of [...plugins].sort()) needs.push({ kind: builtIns.get(name) ?? 'connection', name })
+  return needs
+}
+
+/**
+ * An item from the workspace, by its files: Blueprint files by their own
+ * rule, everything else by what it names. `hasLocalDataset` answers for the
+ * project the item lands in.
+ */
+export function portabilityOf(
+  kind: string,
+  files: Record<string, string>,
+  opts: { connections?: ReadonlySet<string>; builtIns?: ReadonlyMap<string, BuiltInKind>; hasLocalDataset?: (dataset: string) => boolean; context?: Context } = {},
+): Portability {
+  if (kind === 'blueprint') {
+    const needs = Object.entries(files).flatMap(([path, text]): Need[] => {
+      const doc = parse(path, text) as Doc | null
+      if (!doc || typeof doc !== 'object') return []
+      if (path.startsWith('blueprint/object-types/')) return objectTypeNeeds(doc, opts.hasLocalDataset ?? (() => true))
+      if (path.startsWith('blueprint/actions/')) return actionNeeds(doc)
+      return []
+    })
+    return classify(needs, opts.context)
+  }
+  return classify(itemNeeds(kind, files, opts.connections, opts.builtIns), opts.context)
+}

package/src/local/local-project.ts

@@ -1,5 +1,6 @@
 import { createHash } from 'node:crypto'
 import { existsSync } from 'node:fs'
+import { homedir } from 'node:os'
 import { basename, dirname, join, resolve } from 'node:path'
 
 /**
@@ -27,10 +28,13 @@ export interface LocalProject {
 export const PROJECT_FILES = ['frontera.yaml', 'frontera.solution.yaml'] as const
 const MARKERS = [...PROJECT_FILES, '.frontera'] as const
 
-export function findSolutionRoot(cwd: string): string {
+export function findSolutionRoot(cwd: string, home: string = homedir()): string {
   let dir = resolve(cwd)
   for (;;) {
-    if (MARKERS.some((m) => existsSync(join(dir, m)))) return dir
+    // `~/.frontera` is the CLI's own settings, not a project: in the home folder only a project file counts,
+    // or every new folder under it (`~/Frontera/acme`) would be taken for part of a project rooted at home.
+    const markers = dir === resolve(home) ? PROJECT_FILES : MARKERS
+    if (markers.some((m) => existsSync(join(dir, m)))) return dir
     const parent = dirname(dir)
     if (parent === dir) return resolve(cwd)
     dir = parent

package/src/local/local-quality-checks.ts

@@ -0,0 +1,171 @@
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'
+
+/**
+ * A knowledge base's quality checks (the platform's knowledge eval
+ * fixtures): the questions its retrieval should answer, each with the
+ * documents it should find, the words it should return and the answer the
+ * returned passages should support. They come down with the knowledge base,
+ * in `knowledge/<name>/quality-checks.yaml` beside `knowledge.json`, so a
+ * quality check runs against the local runtime's copy. Their runs and scores
+ * stay on the platform (sync v2 design §8, records).
+ *
+ * A check names its documents by path in the knowledge base's folder
+ * (`Policies/Returns/terms.md`), not by the platform's ids: the local
+ * runtime's copy of a document has its own. A document the runtime doesn't
+ * hold (documents are brought only when asked) is left off that check
+ * locally, said once.
+ */
+
+export const QUALITY_CHECKS_FILE = 'quality-checks.yaml'
+
+export const qualityChecksPath = (kb: string) => `knowledge/${kb}/${QUALITY_CHECKS_FILE}`
+
+const HEADER = '# Quality checks for this knowledge base, from the workspace (frontera local sync pull).\n# Each question names the documents its search should find; frontera local dev gives them to the local runtime. Runs stay on the platform.\n'
+
+export interface QualityCheck {
+  question: string
+  /** Documents the search should find, by path in the knowledge base's folder. */
+  documents?: string[]
+  /** Words the returned passages should contain. */
+  keywords?: string[]
+  /** The answer the returned passages should support. */
+  answer?: string
+  /** Generated from the documents on the platform, not written by a person. */
+  generated?: true
+}
+
+/** A fixture as `GET /v1/workspace-knowledge/:id/eval/fixtures` lists it. */
+export interface FixtureRow {
+  id: string
+  query: string
+  expectedSourceIds?: string[] | null
+  expectedKeywords?: string[] | null
+  groundTruthAnswer?: string | null
+  source?: string | null
+}
+
+/** A document as `GET …/sources` lists it. */
+export interface SourceRow {
+  id: string
+  fileName?: string | null
+  folderId?: string | null
+}
+
+const rowsOf = <T>(body: unknown, key: string): T[] => {
+  const data = (body as { data?: unknown } | null)?.data ?? body
+  const list = Array.isArray(data) ? data : (data as Record<string, unknown> | null)?.[key]
+  return (Array.isArray(list) ? list : []) as T[]
+}
+
+export const fixtureRows = (body: unknown) => rowsOf<FixtureRow>(body, 'fixtures')
+export const sourceRows = (body: unknown) => rowsOf<SourceRow>(body, 'sources')
+
+/** A document's path under its knowledge base's folder (as `local-knowledge-documents.ts` writes it). */
+const documentPath = (folder: string | null | undefined, fileName: string) => (folder ? `${folder}/${fileName}` : fileName)
+
+/** Each document's path by its id, as the knowledge base's folders file it. */
+export function sourcePaths(sources: SourceRow[], folders: ReadonlyMap<string, string>): Map<string, string> {
+  return new Map(sources.flatMap((s) => (s.id && s.fileName ? [[s.id, documentPath(s.folderId ? folders.get(s.folderId) : null, s.fileName)] as const] : [])))
+}
+
+/** The platform's fixtures as the file holds them; `unnamed` counts the documents no path could be found for. */
+export function checksFromFixtures(fixtures: FixtureRow[], paths: ReadonlyMap<string, string>): { checks: QualityCheck[]; unnamed: number } {
+  let unnamed = 0
+  const checks = fixtures.map((f): QualityCheck => {
+    const documents = (f.expectedSourceIds ?? []).flatMap((id) => {
+      const path = paths.get(id)
+      if (!path) unnamed += 1
+      return path ? [path] : []
+    })
+    return {
+      question: f.query,
+      ...(documents.length ? { documents } : {}),
+      ...(f.expectedKeywords?.length ? { keywords: [...f.expectedKeywords] } : {}),
+      ...(f.groundTruthAnswer ? { answer: f.groundTruthAnswer } : {}),
+      ...(f.source === 'synthetic' ? { generated: true as const } : {}),
+    }
+  })
+  // In the order of their questions: the platform lists them newest first, which a file need not follow.
+  return { checks: checks.sort((a, b) => a.question.localeCompare(b.question)), unnamed }
+}
+
+export function qualityChecksText(checks: QualityCheck[]): string {
+  return HEADER + stringifyYaml({ checks }, { lineWidth: 0 })
+}
+
+/** The checks a file holds, in order; a question once. Throws on a file that doesn't parse. */
+export function readQualityChecks(text: string): QualityCheck[] {
+  const doc = parseYaml(text) as { checks?: unknown } | null
+  const out: QualityCheck[] = []
+  const strings = (v: unknown) => (Array.isArray(v) ? v.filter((x): x is string => typeof x === 'string' && !!x.trim()).map((x) => x.trim()) : [])
+  for (const c of Array.isArray(doc?.checks) ? doc!.checks : []) {
+    const row = (c ?? {}) as Record<string, unknown>
+    const question = typeof row.question === 'string' ? row.question.trim() : ''
+    if (!question || out.some((x) => x.question === question)) continue
+    const documents = strings(row.documents)
+    const keywords = strings(row.keywords)
+    out.push({
+      question,
+      ...(documents.length ? { documents } : {}),
+      ...(keywords.length ? { keywords } : {}),
+      ...(typeof row.answer === 'string' && row.answer.trim() ? { answer: row.answer.trim() } : {}),
+      ...(row.generated === true ? { generated: true as const } : {}),
+    })
+  }
+  return out
+}
+
+/** What a check is as the fixture routes take it, its documents by this runtime's ids. */
+function fixtureBody(check: QualityCheck, ids: ReadonlyMap<string, string>): { query: string; expectedSourceIds: string[]; expectedKeywords: string[]; groundTruthAnswer: string } {
+  return {
+    query: check.question,
+    expectedSourceIds: (check.documents ?? []).flatMap((p) => (ids.has(p) ? [ids.get(p)!] : [])),
+    expectedKeywords: check.keywords ?? [],
+    groundTruthAnswer: check.answer ?? '',
+  }
+}
+
+const sameList = (a: readonly string[] | null | undefined, b: readonly string[]) => JSON.stringify([...(a ?? [])].sort()) === JSON.stringify([...b].sort())
+
+type Call = (method: string, path: string, body?: unknown) => Promise<{ status: number; body: unknown }>
+
+/**
+ * Give a knowledge base on the runtime the file's quality checks: a question
+ * the runtime lacks is added, one whose documents, words or answer differ is
+ * updated, and one the file no longer has is removed. `documentIds` is the
+ * runtime's documents by path. Returns the lines for the build.
+ */
+export async function applyQualityChecks(kbId: string, kbName: string, checks: QualityCheck[], documentIds: ReadonlyMap<string, string>, call: Call): Promise<{ ok: boolean; lines: string[] }> {
+  const base = `/v1/workspace-knowledge/${encodeURIComponent(kbId)}/eval/fixtures`
+  const listed = await call('GET', base)
+  if (listed.status >= 400) return { ok: false, lines: [`  warning: Knowledge base ${kbName}’s quality checks weren’t applied: the runtime answered ${listed.status}`] }
+  const here = fixtureRows(listed.body)
+  const lines: string[] = []
+  let added = 0
+  let changed = 0
+  let removed = 0
+  const missing = new Set<string>()
+  for (const check of checks) {
+    for (const p of check.documents ?? []) if (!documentIds.has(p)) missing.add(p)
+    const body = fixtureBody(check, documentIds)
+    const found = here.find((f) => f.query === check.question)
+    if (!found) {
+      const made = await call('POST', base, body)
+      if (made.status >= 400) lines.push(`  warning: Knowledge base ${kbName}: the quality check “${check.question}” wasn’t added (${made.status})`)
+      else added += 1
+      continue
+    }
+    if (sameList(found.expectedSourceIds, body.expectedSourceIds) && sameList(found.expectedKeywords, body.expectedKeywords) && (found.groundTruthAnswer ?? '') === body.groundTruthAnswer) continue
+    const put = await call('PUT', `${base}/${encodeURIComponent(found.id)}`, body)
+    if (put.status >= 400) lines.push(`  warning: Knowledge base ${kbName}: the quality check “${check.question}” wasn’t updated (${put.status})`)
+    else changed += 1
+  }
+  for (const f of here.filter((x) => !checks.some((c) => c.question === x.query))) {
+    const gone = await call('DELETE', `${base}/${encodeURIComponent(f.id)}`)
+    if (gone.status < 400) removed += 1
+  }
+  const counts = [added ? `${added} added` : '', changed ? `${changed} updated` : '', removed ? `${removed} removed` : ''].filter(Boolean).join(', ')
+  lines.unshift(`  ${checks.length} quality check${checks.length === 1 ? '' : 's'} for knowledge base ${kbName}${counts ? ` (${counts})` : ''}`)
+  if (missing.size) lines.push(`  warning: Knowledge base ${kbName}’s quality checks name ${missing.size} document${missing.size === 1 ? '' : 's'} this runtime doesn’t have, so they check without ${missing.size === 1 ? 'it' : 'them'}: bring the documents to check them here`)
+  return { ok: true, lines }
+}

package/src/local/local-scaffold.ts

@@ -316,7 +316,8 @@ const GITIGNORE = `node_modules/
 .env
 `
 
-export function solutionScaffoldFiles(name: string, opts: { agentModel?: string } = {}): Record<string, string> {
+/** `samples: false` leaves out the example Agent, workflow, Function and object type: a project started from a workspace gets the workspace's own. */
+export function solutionScaffoldFiles(name: string, opts: { agentModel?: string; samples?: boolean } = {}): Record<string, string> {
   const sdk = vendored.files as Record<string, string>
   const out: Record<string, string> = {}
   for (const rel of AUTOMATION_SDK_FILES) {
@@ -331,15 +332,17 @@ export function solutionScaffoldFiles(name: string, opts: { agentModel?: string
   out['.gitignore'] = GITIGNORE
   out['.mcp.json'] = DOCS_MCP
   out['.cursor/mcp.json'] = DOCS_MCP
-  out['agents/assistant/agent.json'] = AGENT(name, opts.agentModel)
-  out['agents/assistant/operating.json'] = OPERATING
-  out['agents/assistant/evals/regression.yaml'] = EVALS
-  out['workflows/hello.yaml'] = WORKFLOW
-  out['functions/word-count.ts'] = FUNCTION
-  out['functions/word-count.test.ts'] = FUNCTION_TEST
-  out['datasets/tickets.json'] = DATASET
-  out['datasets/tickets.csv'] = DATASET_ROWS
-  out['blueprint/object-types/Ticket.yaml'] = OBJECT_TYPE
+  if (opts.samples !== false) {
+    out['agents/assistant/agent.json'] = AGENT(name, opts.agentModel)
+    out['agents/assistant/operating.json'] = OPERATING
+    out['agents/assistant/evals/regression.yaml'] = EVALS
+    out['workflows/hello.yaml'] = WORKFLOW
+    out['functions/word-count.ts'] = FUNCTION
+    out['functions/word-count.test.ts'] = FUNCTION_TEST
+    out['datasets/tickets.json'] = DATASET
+    out['datasets/tickets.csv'] = DATASET_ROWS
+    out['blueprint/object-types/Ticket.yaml'] = OBJECT_TYPE
+  }
   out['package.json'] =
     JSON.stringify(
       {
@@ -386,7 +389,7 @@ export function solutionScaffoldFiles(name: string, opts: { agentModel?: string
  * are. An AGENTS.md that is still the CLI's generic one is replaced anyway:
  * nobody has changed it, and the project's version contains it.
  */
-export function writeSolutionScaffold(target: string, name: string, force: boolean, opts: { agentModel?: string } = {}): { written: string[]; skipped: string[] } {
+export function writeSolutionScaffold(target: string, name: string, force: boolean, opts: { agentModel?: string; samples?: boolean } = {}): { written: string[]; skipped: string[] } {
   const written: string[] = []
   const skipped: string[] = []
   for (const [rel, content] of Object.entries(solutionScaffoldFiles(name, opts))) {

package/src/local/local-schemas.ts

@@ -70,13 +70,44 @@ const EVALS_SCHEMA: JsonSchema = {
             items: { type: 'string', minLength: 1 },
             description: 'Phrases the last reply must contain, ignoring case. When present, expected is not checked.',
           },
+          checks: {
+            type: 'array',
+            description: 'Checks on the last reply and the tools the Agent called, graded as the platform grades them; every one must pass.',
+            items: {
+              type: 'object',
+              properties: {
+                type: { enum: ['contains', 'regex', 'tool_called', 'tool_not_called'] },
+                value: { type: 'string', description: 'contains: the phrase; regex: the pattern.' },
+                flags: { type: 'string', description: 'regex flags; i when absent.' },
+                tool: { type: 'string', description: 'tool_called, tool_not_called: the tool’s name.' },
+                argsSubset: { type: 'object', description: 'The arguments the call must carry.' },
+                mode: { enum: ['exact', 'ignore', 'subset', 'superset'], description: 'How argsSubset is compared; superset when absent.' },
+                text: { type: 'string', description: 'What the check means, in words.' },
+              },
+              required: ['type'],
+            },
+          },
+          judged: { type: 'array', items: { type: 'string' }, description: 'Expectations only the platform’s judge grades; kept, not graded locally.' },
+          gold: { type: 'boolean', description: 'Human-certified on the platform.' },
+          scenario: {
+            type: 'object',
+            description: 'A conversation with a simulated customer (persona, goal, opening, maxTurns, verified, results, note), run on the local runtime’s evaluation runner.',
+            required: ['goal', 'opening'],
+          },
+          expect: { type: 'object', description: 'What a scenario expects: outcome, actions, noActions.' },
           savedFrom: { type: 'string', description: 'Where the case was saved from; Studio writes it.' },
         },
-        required: ['name', 'conversation'],
-        anyOf: [{ required: ['expected'] }, { required: ['expectedContains'] }],
+        required: ['name'],
+        anyOf: [
+          { required: ['conversation', 'expected'] },
+          { required: ['conversation', 'expectedContains'] },
+          { required: ['conversation', 'checks'] },
+          { required: ['scenario'] },
+        ],
         additionalProperties: false,
       },
     },
+    regression: { type: 'boolean', description: 'This file is the Agent’s regression dataset in the workspace; `local test --regression` runs only these.' },
   },
   required: ['cases'],
   additionalProperties: false,