@frontera-sdk/cli 1.53.23 → 1.53.24

package/src/local/local-knowledge-base-apply.ts

@@ -0,0 +1,714 @@
+import { createHash } from 'node:crypto'
+import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { basename, join, relative, sep } from 'node:path'
+
+import { isIgnoredName } from './local-dev-sync'
+import {
+  KNOWLEDGE_BASE_DIR,
+  KNOWLEDGE_BASE_DOCUMENTS_DIR,
+  KNOWLEDGE_BASE_FILE,
+  KNOWLEDGE_BASE_SETTINGS_FILE,
+  KNOWLEDGE_BASE_SOURCES_DIR,
+  type KnowledgeBaseFile,
+  type KnowledgeBaseSettingsFile,
+  type KnowledgeBaseSourceFile,
+} from './local-knowledge-base-files'
+import { classify, knowledgeBaseNeeds, type Need } from './local-portability'
+import { hasSecretValue, needsValueLine, type SecretPresence } from './local-secrets'
+import type { ComponentState, LocalCredentials } from './local-state'
+
+/**
+ * `frontera local dev` for `knowledge-base/`: each base folder and the
+ * organization's Knowledge settings made on the local runtime as the files
+ * say, through the Knowledge app's routes, as the runtime's owner.
+ *
+ *   knowledge-base/<name>/knowledge-base.json   the base: created, or renamed, by the id this runtime gave it
+ *   knowledge-base/<name>/sources/<source>.json each API source: created, changed or removed
+ *   knowledge-base/<name>/documents/**          each file uploaded once, by its sha256
+ *   knowledge-base/settings.json                the assistant, the search model and the source policy
+ *   agents/<slug>/agent.json knowledgeBases     the Agent's picks, after the Agent is applied
+ *
+ * Each step reads what the runtime holds first and writes only what differs,
+ * so applying unchanged files again makes no write. The ids this runtime gave
+ * each base, source and uploaded document are kept in
+ * `.frontera/local/knowledge-bases.json`; a base whose id is gone (a reset) is
+ * found again by its name, or made again. A base this runtime made is
+ * removed once its folder is deleted (`removeDeletedBases`).
+ */
+
+const API = '/v1/knowledge-app'
+
+/** Where the owner stands for a call: in the project's workspace, or at the organization level (no workspace). */
+export type Place = 'workspace' | 'organization'
+
+export type KnowledgeCall = (place: Place, method: string, path: string, body?: unknown) => Promise<{ status: number; body: unknown }>
+
+/** Calls as the runtime's owner (the session the runtime keeps); a `FormData` body is sent as it is. */
+export function knowledgeBaseCalls(serviceUrl: string, creds: Pick<LocalCredentials, 'session' | 'organizationId' | 'workspaceId'>, fetcher: typeof fetch = fetch): KnowledgeCall {
+  return async (place, method, path, body) => {
+    const form = body instanceof FormData
+    const res = await fetcher(new URL(path, serviceUrl), {
+      method,
+      headers: {
+        authorization: `Bearer ${creds.session}`,
+        'x-org-id': creds.organizationId,
+        ...(place === 'workspace' ? { 'x-workspace-id': creds.workspaceId } : {}),
+        ...(body === undefined || form ? {} : { 'content-type': 'application/json' }),
+      },
+      body: body === undefined ? undefined : form ? body : JSON.stringify(body),
+    })
+    const text = await res.text()
+    let parsed: unknown = text
+    try {
+      parsed = text ? JSON.parse(text) : null
+    } catch {
+      // Kept as text for the message.
+    }
+    return { status: res.status, body: parsed }
+  }
+}
+
+const dataOf = <T>(body: unknown): T => ((body as { data?: unknown } | null)?.data ?? body) as T
+const messageOf = (r: { status: number; body: unknown }) => {
+  const b = r.body as { message?: unknown } | null
+  return `${r.status} ${String(typeof b?.message === 'string' ? b.message : JSON.stringify(b)).replace(/\s+/g, ' ').slice(0, 300)}`
+}
+const isOk = (r: { status: number }) => r.status >= 200 && r.status < 300
+
+export interface RunResult {
+  ok: boolean
+  output: string
+}
+
+// ── What this runtime gave each base ────────────────────────────────────────
+
+export interface AppliedBase {
+  id: string
+  /** Where this runtime made the base; absent for one it found by its name, which a deleted folder never removes. */
+  made?: Place
+  /** Source file stem (`sources/<stem>.json`) to the source's id here. */
+  sources: Record<string, string>
+  /** A document file's sha256 to the id of the document uploaded from it. */
+  documents: Record<string, string>
+  /** Source file stem to the secret it was last sent and when that secret was set (`credentialStamp`). */
+  credentials?: Record<string, string>
+}
+
+interface AppliedRecord {
+  organizationId: string
+  bases: Record<string, AppliedBase>
+}
+
+const RECORD_FILE = 'knowledge-bases.json'
+
+function readRecord(stateDir: string): AppliedRecord | null {
+  try {
+    const doc = JSON.parse(readFileSync(join(stateDir, RECORD_FILE), 'utf8')) as AppliedRecord
+    return doc && typeof doc.organizationId === 'string' && doc.bases && typeof doc.bases === 'object' ? doc : null
+  } catch {
+    return null
+  }
+}
+
+/** The bases applied to this runtime's organization, by folder; none for another organization (the runtime was reset). */
+export function appliedBases(stateDir: string, organizationId: string): Record<string, AppliedBase> {
+  const record = readRecord(stateDir)
+  return record?.organizationId === organizationId ? record.bases : {}
+}
+
+/** Each base's id here, by folder, whatever organization it was applied to: what an Agent's picks are set from. */
+export function appliedBaseIds(stateDir: string): Record<string, string> {
+  return Object.fromEntries(Object.entries(readRecord(stateDir)?.bases ?? {}).map(([folder, b]) => [folder, b.id]))
+}
+
+function saveBase(stateDir: string, organizationId: string, folder: string, base: AppliedBase | null): void {
+  const bases = { ...appliedBases(stateDir, organizationId) }
+  if (base) bases[folder] = base
+  else delete bases[folder]
+  mkdirSync(stateDir, { recursive: true })
+  writeFileSync(join(stateDir, RECORD_FILE), `${JSON.stringify({ organizationId, bases }, null, 2)}\n`)
+}
+
+// ── The search index ────────────────────────────────────────────────────────
+
+const RESTART = 'run `frontera local stop`, then `frontera local start`'
+
+/** Why the runtime cannot build knowledge bases, and what to run. */
+export interface KnowledgeOff {
+  why: string
+  next: string
+}
+
+/**
+ * Why the runtime cannot build knowledge bases, with the next step; null when
+ * it can. Read from how the runtime started its search index, then from the
+ * Knowledge app itself, which answers 404 when it does not run.
+ */
+export async function knowledgeBasesOff(components: readonly ComponentState[] | undefined, call: KnowledgeCall): Promise<KnowledgeOff | null> {
+  const index = components?.find((c) => c.name === 'search-index')
+  if (!index) return { why: 'the runtime started without its search index', next: RESTART }
+  if (index.status === 'off') {
+    if (index.detail === 'excluded') return { why: 'the runtime started with `--exclude search-index`', next: `${RESTART} without it` }
+    if (index.detail?.startsWith('no knowledge-base/ folder')) return { why: 'knowledge-base/ was added after the runtime started', next: RESTART }
+    return { why: index.detail ?? 'it is off', next: RESTART }
+  }
+  if (index.status !== 'running') return { why: `it is ${index.status === 'failed' ? 'down' : index.status}${index.detail ? ` (${index.detail})` : ''}`, next: `${RESTART}; \`frontera local logs search-index\` says why it stopped` }
+  // Not answering at all is the apply's to say, as for any item.
+  const me = await call('workspace', 'GET', `${API}/me`).catch(() => null)
+  if (me?.status === 404) return { why: "the runtime's service does not run the Knowledge app", next: RESTART }
+  return null
+}
+
+/** What `local dev` fails a knowledge base or the Knowledge settings with while the search index is off. */
+export function knowledgeOffOutput(off: KnowledgeOff): string {
+  return `error [LOCAL_KNOWLEDGE_OFF]: knowledge bases need the local runtime's search index, which is off: ${off.why}\n  hint: ${off.next}`
+}
+
+// ── Reading a base's folder ─────────────────────────────────────────────────
+
+interface SourceEntry {
+  stem: string
+  file: string
+  spec: KnowledgeBaseSourceFile
+}
+
+interface DocumentEntry {
+  file: string
+  sha256: string
+}
+
+interface BaseFiles {
+  base: KnowledgeBaseFile
+  sources: SourceEntry[]
+  documents: DocumentEntry[]
+}
+
+function readJson<T>(root: string, rel: string): T {
+  return JSON.parse(readFileSync(join(root, rel), 'utf8')) as T
+}
+
+function filesUnder(root: string, dir: string): string[] {
+  const out: string[] = []
+  const walk = (at: string) => {
+    let names: string[]
+    try {
+      names = readdirSync(at).sort()
+    } catch {
+      return
+    }
+    for (const name of names) {
+      if (isIgnoredName(name)) continue
+      const path = join(at, name)
+      const stat = lstatSync(path)
+      if (stat.isDirectory()) walk(path)
+      else if (stat.isFile()) out.push(relative(root, path).split(sep).join('/'))
+    }
+  }
+  walk(join(root, dir))
+  return out
+}
+
+function readBaseFiles(root: string, folder: string): BaseFiles {
+  const dir = `${KNOWLEDGE_BASE_DIR}/${folder}`
+  const base = readJson<KnowledgeBaseFile>(root, `${dir}/${KNOWLEDGE_BASE_FILE}`)
+  const sources = filesUnder(root, `${dir}/${KNOWLEDGE_BASE_SOURCES_DIR}`)
+    .filter((f) => f.endsWith('.json') && f.split('/').length === 4)
+    .map((file) => ({ stem: basename(file, '.json'), file, spec: readJson<KnowledgeBaseSourceFile>(root, file) }))
+  const documents = filesUnder(root, `${dir}/${KNOWLEDGE_BASE_DOCUMENTS_DIR}`).map((file) => ({
+    file,
+    sha256: createHash('sha256').update(readFileSync(join(root, file))).digest('hex'),
+  }))
+  return { base, sources, documents }
+}
+
+const adaptationLines = (needs: Need[]) => classify(needs).adaptations.map((a) => `  warning: ${a}`)
+
+// ── A source's credential ───────────────────────────────────────────────────
+
+/**
+ * The credential step for a source that signs in; `applySources` is its one
+ * caller. The runtime never gives a secret's value back, so the source is sent
+ * the secret's name (`credentialSecret`), which the local runtime's service
+ * reads and seals. While the runtime has no value for it, the source is left
+ * out with the command that sets one. A runtime that cannot say what it holds
+ * is sent the name, and refuses an unknown one with that command.
+ */
+export function sourceCredential(file: string, secret: string, secrets: SecretPresence | null): { send: { credentialSecret: string }; stamp: string } | { leftOut: string[] } {
+  if (secrets && !hasSecretValue(secrets, secret)) return { leftOut: [needsValueLine(secret, file)] }
+  return { send: { credentialSecret: secret }, stamp: credentialStamp(secret, secrets) }
+}
+
+/** The secret a source signs in with and when it was set: a source is sent the name again when this moves. */
+const credentialStamp = (secret: string, secrets: SecretPresence | null) => `${secret} ${secrets?.updatedAt?.get(secret) ?? ''}`
+
+/**
+ * The secrets a base's sources sign in with that were set, or lost their
+ * value, on the runtime since the base was applied: its files are unchanged,
+ * but `local dev` applies it again so each source signs in with the value now.
+ * `secrets` is asked only for a base with such a source.
+ */
+export async function movedCredentials(project: { root: string; stateDir: string }, folder: string, organizationId: string, secrets: () => Promise<SecretPresence | null>): Promise<string[]> {
+  let sources: SourceEntry[]
+  try {
+    sources = readBaseFiles(project.root, folder).sources.filter((s) => s.spec.credentialSecret)
+  } catch {
+    return []
+  }
+  if (!sources.length) return []
+  const presence = await secrets()
+  if (!presence) return []
+  const sent = appliedBases(project.stateDir, organizationId)[folder]?.credentials ?? {}
+  const now = (secret: string) => (hasSecretValue(presence, secret) ? credentialStamp(secret, presence) : undefined)
+  return [...new Set(sources.filter((s) => now(s.spec.credentialSecret!) !== sent[s.stem]).map((s) => s.spec.credentialSecret!))]
+}
+
+// ── Applying a base ─────────────────────────────────────────────────────────
+
+interface BaseRow {
+  id: string
+  name: string
+  description: string | null
+  ownerWorkspaceId: string | null
+  canManage?: boolean
+  sharing: { allWorkspaces: boolean; workspaceIds: string[] } | null
+}
+
+interface SourceRow {
+  id: string
+  name: string
+  state: string
+  schedule?: string
+  keepRemovedDays?: number
+  config?: unknown
+  auth?: unknown
+  /** When its credential was set; null for one that has none. */
+  credential?: { setAt: string } | null
+}
+
+export interface ApplyDeps {
+  call: KnowledgeCall
+  organizationId: string
+  /** The runtime's workspace: the project's. */
+  workspaceId: string
+  /** The secrets the runtime holds, by name; null when it could not say. */
+  secrets: SecretPresence | null
+}
+
+async function must(call: Promise<{ status: number; body: unknown }>, what: string): Promise<unknown> {
+  const r = await call
+  if (!isOk(r)) throw new Error(`${what}: ${messageOf(r)}`)
+  return r.body
+}
+
+const ownedHere = (row: BaseRow, place: Place, workspaceId: string) => (place === 'workspace' ? row.ownerWorkspaceId === workspaceId : row.ownerWorkspaceId === null)
+
+/** The base the folder is here: by the id this runtime gave it, else by its name; null when there is none. A recorded base in the other scope is refused, as the platform refuses it. */
+async function findBase(call: KnowledgeCall, place: Place, name: string, workspaceId: string, recordedId: string | undefined, folder: string): Promise<BaseRow | null> {
+  if (recordedId) {
+    for (const at of [place, place === 'workspace' ? 'organization' : 'workspace'] as const) {
+      const got = await call(at, 'GET', `${API}/bases/${recordedId}`)
+      if (!isOk(got)) continue
+      const row = dataOf<{ base: BaseRow }>(got.body).base
+      if (ownedHere(row, place, workspaceId)) return row
+      const its: Place = row.ownerWorkspaceId === null ? 'organization' : 'workspace'
+      throw new Error(
+        `${KNOWLEDGE_BASE_DIR}/${folder}/${KNOWLEDGE_BASE_FILE}: scope is ${place}, but its knowledge base ${row.name} is ${its === 'workspace' ? 'a workspace' : 'an organization'} base here; a knowledge base keeps its layer, so make a new one for the other layer` +
+          `\n  hint: put back \`"scope": "${its}"\`, or make a new folder under ${KNOWLEDGE_BASE_DIR}/ for the ${place} base`,
+      )
+    }
+  }
+  const listed = dataOf<{ bases: BaseRow[] }>(await must(call(place, 'GET', `${API}/bases`), 'the knowledge bases could not be listed')).bases
+  return listed.find((b) => b.name === name && ownedHere(b, place, workspaceId) && b.canManage !== false) ?? null
+}
+
+/** The whole folder as the runtime should hold it; each step writes only what differs. */
+export async function applyKnowledgeBase(project: { root: string; stateDir: string }, folder: string, deps: ApplyDeps): Promise<RunResult> {
+  const { call, workspaceId } = deps
+  const lines: string[] = []
+  let files: BaseFiles
+  try {
+    files = readBaseFiles(project.root, folder)
+  } catch (err) {
+    return { ok: false, output: `error [USAGE]: ${KNOWLEDGE_BASE_DIR}/${folder} could not be read: ${(err as Error).message}` }
+  }
+  const { base: file } = files
+  const place: Place = file.scope === 'workspace' ? 'workspace' : 'organization'
+  const name = file.name ?? folder
+  const prior = appliedBases(project.stateDir, deps.organizationId)[folder]
+  const applied: AppliedBase = { id: '', made: prior?.made, sources: { ...prior?.sources }, documents: { ...prior?.documents }, credentials: { ...prior?.credentials } }
+  try {
+    let row = await findBase(call, place, name, workspaceId, prior?.id, folder)
+    if (!row) {
+      const made = await must(call(place, 'POST', `${API}/bases`, { name, ...(file.description ? { description: file.description } : {}) }), `knowledge base ${name} could not be created`)
+      row = dataOf<{ base: BaseRow }>(made).base
+      Object.assign(applied, { made: place, sources: {}, documents: {}, credentials: {} })
+      lines.push(`  created ${place === 'workspace' ? 'workspace' : 'organization'} knowledge base ${name}`)
+    } else {
+      if (row.id !== prior?.id) Object.assign(applied, { made: undefined, sources: {}, documents: {}, credentials: {} })
+      const change: Record<string, string> = {}
+      if (row.name !== name) change.name = name
+      if (file.description !== undefined && (row.description ?? '') !== file.description) change.description = file.description
+      if (Object.keys(change).length) {
+        const was = row.name
+        await must(call(place, 'PATCH', `${API}/bases/${row.id}`, change), `knowledge base ${name} could not be changed`)
+        lines.push(change.name ? `  renamed knowledge base ${was} to ${name}` : `  updated knowledge base ${name}’s description`)
+      }
+    }
+    applied.id = row.id
+    // Saved now: a step below that fails leaves the base found by its id next time.
+    saveBase(project.stateDir, deps.organizationId, folder, applied)
+    if (file.scope === 'organization') lines.push(...(await applySharing(call, row, file.sharing, workspaceId)))
+    lines.push(...adaptationLines(knowledgeBaseNeeds(file, files.sources)))
+    lines.push(...(await applySources(call, place, row.id, files.sources, applied, deps.secrets, folder)))
+    saveBase(project.stateDir, deps.organizationId, folder, applied)
+    lines.push(...(await applyDocuments(project.root, call, place, row.id, files.documents, applied)))
+    saveBase(project.stateDir, deps.organizationId, folder, applied)
+  } catch (err) {
+    if (applied.id) saveBase(project.stateDir, deps.organizationId, folder, applied)
+    return { ok: false, output: [...lines, `error [KNOWLEDGE_BASE_APPLY]: ${(err as Error).message}`].join('\n') }
+  }
+  return { ok: true, output: lines.join('\n') }
+}
+
+/**
+ * The bases whose `knowledge-base/<name>/` folder is gone, after `local dev`
+ * applied the folders there are: one this runtime made is removed here, with
+ * a line, and forgotten; one it found by its name (made in Studio, say) is
+ * only forgotten. A folder renamed keeps its base when its file carries
+ * `name` (a pull always writes it): the new folder found the base by that
+ * name, so it is handed the old folder's record instead. A file without `name`
+ * takes its folder's, so the renamed folder made a new base, and the old one
+ * is removed. A base that a folder still here names (its scope and name), or
+ * any base while a folder's file cannot be read, is never removed: that folder
+ * may be the same base renamed, not applied yet. It is listed in `kept`, to
+ * look again once that folder applies. Nothing is removed while the search
+ * index is off, and a removal the runtime refuses keeps the record, to try
+ * again. `removed` lists the folders forgotten.
+ */
+export async function removeDeletedBases(
+  project: { root: string; stateDir: string },
+  organizationId: string,
+  call: KnowledgeCall,
+  components: readonly ComponentState[] | undefined,
+): Promise<{ lines: string[]; removed: string[]; kept: string[] }> {
+  const record = appliedBases(project.stateDir, organizationId)
+  const gone = (folder: string) => !existsSync(join(project.root, KNOWLEDGE_BASE_DIR, folder))
+  const claimedBy = new Map(Object.entries(record).filter(([folder]) => !gone(folder)).map(([folder, base]) => [base.id, folder]))
+  const lines: string[] = []
+  const removed: string[] = []
+  const kept: string[] = []
+  let off: boolean | undefined
+  let here: ReturnType<typeof namedHere> | undefined
+  for (const [folder, base] of Object.entries(record)) {
+    if (!gone(folder)) continue
+    const claimant = claimedBy.get(base.id)
+    if (base.made && claimant) saveBase(project.stateDir, organizationId, claimant, { ...appliedBases(project.stateDir, organizationId)[claimant]!, made: base.made })
+    else if (base.made) {
+      off ??= (await knowledgeBasesOff(components, call)) !== null
+      if (off) continue
+      here ??= namedHere(project.root)
+      if (here.unread) {
+        lines.push(`  ${KNOWLEDGE_BASE_DIR}/${folder} was deleted, and its knowledge base stays while ${KNOWLEDGE_BASE_DIR}/${here.unread}/${KNOWLEDGE_BASE_FILE} cannot be read: that folder may be the same base renamed`)
+        kept.push(folder)
+        continue
+      }
+      if (here.names.size) {
+        const got = await call(base.made, 'GET', `${API}/bases/${base.id}`)
+        if (!isOk(got) && got.status !== 404) {
+          lines.push(`  warning: ${KNOWLEDGE_BASE_DIR}/${folder} was deleted, and its knowledge base could not be read here: ${messageOf(got)}`)
+          continue
+        }
+        const name = isOk(got) ? dataOf<{ base: BaseRow }>(got.body).base.name : null
+        const holder = name === null ? undefined : here.names.get(`${base.made} ${name}`)
+        if (holder) {
+          lines.push(`  ${KNOWLEDGE_BASE_DIR}/${folder} was deleted, and ${KNOWLEDGE_BASE_DIR}/${holder} names its knowledge base ${name}, so it stays for that folder`)
+          kept.push(folder)
+          continue
+        }
+      }
+      const r = await call(base.made, 'DELETE', `${API}/bases/${base.id}`)
+      if (!isOk(r) && r.status !== 404) {
+        lines.push(`  warning: ${KNOWLEDGE_BASE_DIR}/${folder} was deleted, and its knowledge base could not be removed here: ${messageOf(r)}`)
+        continue
+      }
+      lines.push(`  ✓ ${KNOWLEDGE_BASE_DIR}/${folder} was deleted: removed the knowledge base it made here`)
+    } else if (!claimant) lines.push(`  ${KNOWLEDGE_BASE_DIR}/${folder} was deleted: its knowledge base was not made by this project here, so it stays`)
+    saveBase(project.stateDir, organizationId, folder, null)
+    removed.push(folder)
+  }
+  return { lines, removed, kept }
+}
+
+/** The bases the folders here name, by scope and name, to their folder; `unread` is a folder whose file cannot be read, so it may name any. */
+function namedHere(root: string): { names: Map<string, string>; unread?: string } {
+  const names = new Map<string, string>()
+  let folders: string[]
+  try {
+    folders = readdirSync(join(root, KNOWLEDGE_BASE_DIR)).sort()
+  } catch {
+    return { names }
+  }
+  for (const folder of folders) {
+    const rel = `${KNOWLEDGE_BASE_DIR}/${folder}/${KNOWLEDGE_BASE_FILE}`
+    if (!existsSync(join(root, rel))) continue
+    try {
+      const file = readJson<KnowledgeBaseFile>(root, rel)
+      names.set(`${file.scope === 'workspace' ? 'workspace' : 'organization'} ${file.name ?? folder}`, folder)
+    } catch {
+      return { names, unread: folder }
+    }
+  }
+  return { names }
+}
+
+/** An organization base is shared with every workspace here, or with this runtime's one workspace, the project's. */
+async function applySharing(call: KnowledgeCall, row: BaseRow, sharing: Extract<KnowledgeBaseFile, { scope: 'organization' }>['sharing'], workspaceId: string): Promise<string[]> {
+  const want = 'allWorkspaces' in sharing ? { allWorkspaces: true, workspaceIds: [] as string[] } : { allWorkspaces: false, workspaceIds: [workspaceId] }
+  const have = row.sharing ?? { allWorkspaces: false, workspaceIds: [] }
+  if (have.allWorkspaces === want.allWorkspaces && (want.allWorkspaces || [...have.workspaceIds].sort().join() === want.workspaceIds.join())) return []
+  await must(call('organization', 'PUT', `${API}/bases/${row.id}/shares`, want.allWorkspaces ? { allWorkspaces: true } : want), `knowledge base ${row.name} could not be shared`)
+  return [`  shared knowledge base ${row.name} with ${want.allWorkspaces ? 'every workspace' : 'the project’s workspace'}`]
+}
+
+const SPEC_KEYS = ['name', 'schedule', 'keepRemovedDays', 'config', 'auth'] as const
+
+/** JSON with each object's keys in order: two specs the same up to key order read the same. */
+function canonical(value: unknown): string {
+  const sort = (v: unknown): unknown =>
+    Array.isArray(v) ? v.map(sort) : v && typeof v === 'object' ? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([k, x]) => [k, sort(x)])) : v
+  return JSON.stringify(sort(value))
+}
+
+async function applySources(call: KnowledgeCall, place: Place, baseId: string, sources: SourceEntry[], applied: AppliedBase, secrets: SecretPresence | null, folder: string): Promise<string[]> {
+  if (!sources.length && !Object.keys(applied.sources).length) return []
+  const lines: string[] = []
+  const listed = dataOf<{ sources: SourceRow[]; sourcesAllowed: boolean }>(await must(call(place, 'GET', `${API}/bases/${baseId}/sources`), 'the base’s sources could not be read'))
+  const live = listed.sources.filter((s) => s.state !== 'removing')
+  const byId = new Map(live.map((s) => [s.id, s]))
+  const wanted = new Set<string>()
+  const sentCredentials = (applied.credentials ??= {})
+  for (const { stem, file, spec } of sources) {
+    const credential = spec.credentialSecret ? sourceCredential(file, spec.credentialSecret, secrets) : null
+    if (credential && 'leftOut' in credential) {
+      lines.push(...credential.leftOut)
+      delete sentCredentials[stem]
+      continue
+    }
+    const recorded = applied.sources[stem]
+    const claimed = new Set(Object.values(applied.sources))
+    const existing = (recorded ? byId.get(recorded) : undefined) ?? live.find((s) => s.name === spec.name && !claimed.has(s.id))
+    if (!listed.sourcesAllowed) {
+      if (existing) wanted.add(stem)
+      // Decided for this value of the secret: not applied again for it until it is set again.
+      if (credential) sentCredentials[stem] = credential.stamp
+      lines.push(`  warning: goes without its source ${stem} here: API sources are off for this base on the runtime; allow them with \`"sources": {"apiSources": "${place === 'organization' ? 'organization' : 'all_workspaces'}"}\` in ${KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_SETTINGS_FILE}`)
+      continue
+    }
+    wanted.add(stem)
+    const want: Record<string, unknown> = { name: spec.name, schedule: spec.schedule, keepRemovedDays: spec.keepRemovedDays, config: spec.config, auth: spec.auth }
+    if (!existing) {
+      const made = await must(call(place, 'POST', `${API}/bases/${baseId}/sources`, { kind: spec.kind, ...want, ...credential?.send }), `source ${stem} could not be added`)
+      applied.sources[stem] = dataOf<{ source: SourceRow }>(made).source.id
+      if (credential) sentCredentials[stem] = credential.stamp
+      else delete sentCredentials[stem]
+      lines.push(`  added source ${stem} to ${KNOWLEDGE_BASE_DIR}/${folder}`)
+      continue
+    }
+    applied.sources[stem] = existing.id
+    const patch: Record<string, unknown> = Object.fromEntries(SPEC_KEYS.filter((k) => canonical(existing[k]) !== canonical(want[k])).map((k) => [k, want[k]]))
+    // The name again when the secret was set since, the source has no credential, or where it is sent moves (which the service refuses without one).
+    if (credential && (sentCredentials[stem] !== credential.stamp || existing.credential === null || 'config' in patch || 'auth' in patch)) Object.assign(patch, credential.send)
+    if (Object.keys(patch).length) {
+      await must(call(place, 'PATCH', `${API}/sources/${existing.id}`, patch), `source ${stem} could not be changed`)
+      lines.push(`  changed source ${stem}: ${Object.keys(patch).map((k) => (k === 'credentialSecret' ? 'credential' : k)).join(', ')}`)
+    }
+    if (credential) sentCredentials[stem] = credential.stamp
+    else delete sentCredentials[stem]
+  }
+  for (const [stem, id] of Object.entries(applied.sources)) {
+    if (wanted.has(stem)) continue
+    delete applied.sources[stem]
+    delete sentCredentials[stem]
+    if (!byId.has(id)) continue
+    await must(call(place, 'DELETE', `${API}/sources/${id}`), `source ${stem} could not be removed`)
+    lines.push(`  removed source ${stem}: its file is gone, or it is left out here`)
+  }
+  return lines
+}
+
+/** A refusal of the file itself (its kind or size): said, and the file left out; any other refusal fails the base. */
+const FILE_REFUSALS = new Set(['file_type', 'file_too_large'])
+
+async function applyDocuments(root: string, call: KnowledgeCall, place: Place, baseId: string, documents: DocumentEntry[], applied: AppliedBase): Promise<string[]> {
+  if (!documents.length && !Object.keys(applied.documents).length) return []
+  const lines: string[] = []
+  const listed = dataOf<{ documents: Array<{ id: string }> }>(await must(call(place, 'GET', `${API}/bases/${baseId}/documents`), 'the base’s documents could not be read'))
+  const here = new Set(listed.documents.map((d) => d.id))
+  const wanted = new Set(documents.map((d) => d.sha256))
+  let uploaded = 0
+  for (const doc of documents) {
+    const id = applied.documents[doc.sha256]
+    if (id && here.has(id)) continue
+    const form = new FormData()
+    form.append('file', new File([readFileSync(join(root, doc.file))], basename(doc.file)))
+    const r = await call(place, 'POST', `${API}/bases/${baseId}/documents`, form)
+    if (!isOk(r)) {
+      if (FILE_REFUSALS.has(String((r.body as { reason?: unknown } | null)?.reason))) {
+        lines.push(`  warning: ${doc.file} is not uploaded: ${messageOf(r).replace(/^\d+ /, '')}`)
+        continue
+      }
+      throw new Error(`${doc.file} could not be uploaded: ${messageOf(r)}`)
+    }
+    applied.documents[doc.sha256] = dataOf<{ document: { id: string } }>(r.body).document.id
+    here.add(applied.documents[doc.sha256]!)
+    uploaded += 1
+  }
+  for (const [sha, id] of Object.entries(applied.documents)) {
+    if (wanted.has(sha)) continue
+    delete applied.documents[sha]
+    if (!here.has(id)) continue
+    const r = await call(place, 'DELETE', `${API}/documents/${id}`)
+    if (!isOk(r) && r.status !== 404) throw new Error(`a document whose file is gone could not be removed: ${messageOf(r)}`)
+    lines.push('  removed a document whose file is gone or changed')
+  }
+  if (uploaded) lines.push(`  uploaded ${uploaded} document${uploaded === 1 ? '' : 's'}: each is read in the background, then searchable`)
+  return lines
+}
+
+// ── Knowledge settings ──────────────────────────────────────────────────────
+
+interface AssistantModel {
+  provider: string
+  modelId: string
+}
+
+interface AssistantSetup {
+  enabled: boolean
+  model: AssistantModel | null
+  name: string | null
+  instructions: string | null
+  options: AssistantModel[]
+}
+
+const blankToNull = (v: string | null) => (v?.trim() ? v.trim() : null)
+const sameModel = (a: AssistantModel | null, b: AssistantModel | null) => (a === null || b === null ? a === b : a.provider === b.provider && a.modelId === b.modelId)
+
+/** `knowledge-base/settings.json` on the runtime: each key the file has, as it says, with what the runtime lacks swapped. */
+export async function applyKnowledgeSettings(root: string, deps: { call: KnowledgeCall; workspaceId: string; preferredModel?: string | null }): Promise<RunResult> {
+  const { call } = deps
+  const rel = `${KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_SETTINGS_FILE}`
+  let file: KnowledgeBaseSettingsFile
+  try {
+    file = readJson<KnowledgeBaseSettingsFile>(root, rel)
+  } catch (err) {
+    return { ok: false, output: `error [USAGE]: ${rel} could not be read: ${(err as Error).message}` }
+  }
+  const lines: string[] = []
+  try {
+    if (file.assistant) lines.push(...(await applyAssistant(call, file.assistant, deps.preferredModel ?? null)))
+    if (file.searchModel !== undefined) lines.push(...(await applySearchModel(call, file.searchModel)))
+    if (file.sources) lines.push(...(await applySourcePolicy(call, file.sources, deps.workspaceId)))
+  } catch (err) {
+    return { ok: false, output: [...lines, `error [KNOWLEDGE_SETTINGS_APPLY]: ${(err as Error).message}`].join('\n') }
+  }
+  return { ok: true, output: lines.join('\n') }
+}
+
+async function applyAssistant(call: KnowledgeCall, assistant: NonNullable<KnowledgeBaseSettingsFile['assistant']>, preferred: string | null): Promise<string[]> {
+  const setup = dataOf<{ setup: AssistantSetup | null }>(await must(call('organization', 'GET', `${API}/assistant`), 'the Knowledge assistant could not be read')).setup
+  if (!setup) throw new Error('the runtime’s owner may not set up the Knowledge assistant')
+  const lines: string[] = []
+  let { model, enabled } = assistant
+  if (model && !setup.options.some((o) => sameModel(o, model))) {
+    const swap = setup.options.find((o) => preferred === `${o.provider}/${o.modelId}` || preferred === o.modelId) ?? setup.options[0] ?? null
+    lines.push(swap
+      ? `  warning: the Knowledge assistant names the model ${model.provider}/${model.modelId}, which this runtime does not have; it uses ${swap.provider}/${swap.modelId} here`
+      : `  warning: the Knowledge assistant names the model ${model.provider}/${model.modelId}, and this runtime has no model it can answer with: its answers stay off here`)
+    model = swap
+    if (!swap) enabled = false
+  }
+  const want = { enabled, model, name: blankToNull(assistant.name), instructions: blankToNull(assistant.instructions) }
+  if (want.enabled === setup.enabled && sameModel(want.model, setup.model) && want.name === setup.name && want.instructions === setup.instructions) return lines
+  await must(call('organization', 'PUT', `${API}/assistant`, want), 'the Knowledge assistant could not be set up')
+  return [...lines, `  set up the Knowledge assistant (answers ${want.enabled ? 'on' : 'off'})`]
+}
+
+async function applySearchModel(call: KnowledgeCall, searchModel: KnowledgeBaseSettingsFile['searchModel']): Promise<string[]> {
+  const setting = dataOf<{ chosen: { provider: string; model: string } | null; models: Array<{ provider: string; model: string }> }>(
+    await must(call('organization', 'GET', `${API}/settings/embedding`), 'the Knowledge search model could not be read'),
+  )
+  const lines: string[] = []
+  let want = searchModel ?? null
+  if (want && !setting.models.some((m) => m.provider === want!.provider && m.model === want!.model)) {
+    lines.push(`  warning: the knowledge bases search by meaning with ${want.provider}/${want.model}, which this runtime does not have; they use the organization’s embedding model here`)
+    want = null
+  }
+  const have = setting.chosen
+  if (want === null ? have === null : have?.provider === want.provider && have.model === want.model) return lines
+  await must(call('organization', 'PUT', `${API}/settings/embedding`, { choice: want }), 'the Knowledge search model could not be set')
+  return [...lines, want ? `  set the Knowledge search model to ${want.provider}/${want.model}: every base’s index is built again` : '  set the Knowledge search model to follow the organization’s embedding model']
+}
+
+async function applySourcePolicy(call: KnowledgeCall, sources: NonNullable<KnowledgeBaseSettingsFile['sources']>, workspaceId: string): Promise<string[]> {
+  const policy = dataOf<{ policy: { apiSources: string; workspaceIds: string[] } }>(await must(call('organization', 'GET', `${API}/source-policy`), 'the source policy could not be read')).policy
+  const lines: string[] = []
+  const want = { apiSources: sources.apiSources, workspaceIds: sources.apiSources === 'chosen_workspaces' ? [workspaceId] : [] }
+  if (sources.apiSources === 'chosen_workspaces') lines.push(`  warning: API sources are allowed for ${sources.workspaces.join(', ')} on the platform; here, for this runtime’s one workspace, the project’s`)
+  if (policy.apiSources === want.apiSources && [...policy.workspaceIds].sort().join() === want.workspaceIds.join()) return lines
+  await must(call('organization', 'PUT', `${API}/source-policy`, want.apiSources === 'chosen_workspaces' ? want : { apiSources: want.apiSources }), 'the source policy could not be set')
+  return [...lines, `  set which knowledge bases may have API sources: ${want.apiSources}`]
+}
+
+// ── An Agent's picks ────────────────────────────────────────────────────────
+
+/** The base folder a pick names (`knowledge-base/<name>`), or null. */
+export const pickedFolder = (ref: unknown): string | null => (typeof ref === 'string' && ref.startsWith(`${KNOWLEDGE_BASE_DIR}/`) ? ref.slice(KNOWLEDGE_BASE_DIR.length + 1) || null : null)
+
+/** agent.json's `knowledgeBases`, or null when the key is left out (the picks stay as they are). */
+export function agentPicks(doc: unknown): string[] | null {
+  const picks = (doc as { knowledgeBases?: unknown } | null)?.knowledgeBases
+  return Array.isArray(picks) ? picks.filter((p): p is string => typeof p === 'string') : null
+}
+
+/**
+ * Set the Agent's picks from its file, after the Agent is applied: the whole
+ * list when `knowledgeBases` is there (an empty one drops them all), nothing
+ * when it is left out. A pick whose base is not on the runtime is left out
+ * and said. Returns the lines to say.
+ */
+export async function applyAgentKnowledgeBases(project: { root: string; stateDir: string }, agentFile: string, agentId: string | null, call: KnowledgeCall): Promise<string[]> {
+  let picks: string[] | null
+  try {
+    picks = agentPicks(readJson<unknown>(project.root, agentFile))
+  } catch {
+    return []
+  }
+  if (picks === null) return []
+  if (!agentId) return [`  warning: ${agentFile} knowledgeBases: the Agent is not on the runtime, so its knowledge bases are not set`]
+  const ids = appliedBaseIds(project.stateDir)
+  const lines: string[] = []
+  const want: string[] = []
+  for (const ref of picks) {
+    const id = ids[pickedFolder(ref) ?? '']
+    if (id) want.push(id)
+    else lines.push(`  warning: goes without ${ref} here: that knowledge base is not on this runtime`)
+  }
+  const path = `${API}/agents/${agentId}/bases`
+  const got = await call('workspace', 'GET', path).catch((err: Error) => ({ status: 0, body: { message: err.message } }))
+  if (got.status === 404 && (got.body as { reason?: unknown } | null)?.reason === 'knowledge_off') {
+    return [`  warning: goes without its knowledge bases here: the runtime’s search index is off`]
+  }
+  if (!isOk(got)) return [...lines, `  warning: ${agentFile} knowledgeBases could not be read on the runtime: ${messageOf(got)}`]
+  const have = dataOf<{ bases: Array<{ id: string; picked: boolean }> }>(got.body).bases.filter((b) => b.picked).map((b) => b.id).sort()
+  const wanted = [...new Set(want)].sort()
+  if (have.join() === wanted.join()) return lines
+  const put = await call('workspace', 'PUT', path, { knowledgeBaseIds: wanted }).catch((err: Error) => ({ status: 0, body: { message: err.message } }))
+  if (!isOk(put)) return [...lines, `  warning: ${agentFile} knowledgeBases could not be set on the runtime: ${messageOf(put)}`]
+  return [...lines, wanted.length ? `  set the Agent’s knowledge bases: ${picks.filter((p) => ids[pickedFolder(p) ?? '']).join(', ')}` : '  dropped the Agent’s knowledge bases']
+}
+
+/** Whether a project has the folder a pick names. */
+export function hasKnowledgeBaseFolder(root: string, ref: string): boolean {
+  const folder = pickedFolder(ref)
+  return !!folder && existsSync(join(root, KNOWLEDGE_BASE_DIR, folder, KNOWLEDGE_BASE_FILE))
+}